From 99f06f28615042bade669777fca3896f92946fda Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Mon, 18 Aug 2025 16:36:46 -0700
Subject: [PATCH 01/23] [LFI] [AArch64] Add aarch64_lfi architecture

Adds the `aarch64_lfi`, `aarch64_lfi_stores` and `aarch64_lfi_jumps`
subarches for AArch64. Reserves the necessary registers and disables
compressed jump tables when targeting LFI.
---
 llvm/include/llvm/TargetParser/Triple.h       | 30 +++++++++++++++++++
 .../Target/AArch64/AArch64RegisterInfo.cpp    | 22 ++++++++++++++
 llvm/lib/Target/AArch64/AArch64Subtarget.h    |  1 +
 .../Target/AArch64/AArch64TargetMachine.cpp   |  5 ++++
 llvm/lib/TargetParser/Triple.cpp              | 18 +++++++++++
 5 files changed, 76 insertions(+)

diff --git a/llvm/include/llvm/TargetParser/Triple.h b/llvm/include/llvm/TargetParser/Triple.h
index 7d67966d17256..afecf2c39bf74 100644
--- a/llvm/include/llvm/TargetParser/Triple.h
+++ b/llvm/include/llvm/TargetParser/Triple.h
@@ -148,6 +148,10 @@ class Triple {
     AArch64SubArch_arm64e,
     AArch64SubArch_arm64ec,
 
+    AArch64SubArch_lfi,
+    AArch64SubArch_lfi_stores,
+    AArch64SubArch_lfi_jumps,
+
     KalimbaSubArch_v3,
     KalimbaSubArch_v4,
     KalimbaSubArch_v5,
@@ -676,6 +680,32 @@ class Triple {
            getSubArch() == Triple::AArch64SubArch_arm64ec;
   }
 
+  /// Checks if we're targeting any subarch of AArch64 LFI.
+  bool isAArch64LFI() const {
+    return getArch() == Triple::aarch64 &&
+           (getSubArch() == Triple::AArch64SubArch_lfi ||
+            getSubArch() == Triple::AArch64SubArch_lfi_stores ||
+            getSubArch() == Triple::AArch64SubArch_lfi_jumps);
+  }
+
+  /// Checks if we're targeting AArch64 LFI.
+  bool isAArch64LFIFull() const {
+    return getArch() == Triple::aarch64 &&
+           getSubArch() == Triple::AArch64SubArch_lfi;
+  }
+
+  /// Checks if we're targeting AArch64 LFI stores-only.
+  bool isAArch64LFIStores() const {
+    return getArch() == Triple::aarch64 &&
+           getSubArch() == Triple::AArch64SubArch_lfi_stores;
+  }
+
+  /// Checks if we're targeting AArch64 LFI jumps-only.
+  bool isAArch64LFIJumps() const {
+    return getArch() == Triple::aarch64 &&
+           getSubArch() == Triple::AArch64SubArch_lfi_jumps;
+  }
+
   bool isWindowsCoreCLREnvironment() const {
     return isOSWindows() && getEnvironment() == Triple::CoreCLR;
   }
diff --git a/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp b/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
index 367f6b626b420..22f845e1fa798 100644
--- a/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
@@ -403,6 +403,18 @@ AArch64RegisterInfo::explainReservedReg(const MachineFunction &MF,
   if (hasBasePointer(MF) && MCRegisterInfo::regsOverlap(PhysReg, AArch64::X19))
     return std::string("X19 is used as the frame base pointer register.");
 
+  if (MF.getSubtarget<AArch64Subtarget>().isAArch64LFI()) {
+    bool warn = false;
+    if (MCRegisterInfo::regsOverlap(PhysReg, AArch64::X28) ||
+        MCRegisterInfo::regsOverlap(PhysReg, AArch64::X27) ||
+        MCRegisterInfo::regsOverlap(PhysReg, AArch64::X26))
+      warn = true;
+
+    if (warn)
+      return std::string(AArch64InstPrinter::getRegisterName(PhysReg)) +
+             " is used as a reserved register by LFI.";
+  }
+
   if (MF.getSubtarget<AArch64Subtarget>().isWindowsArm64EC()) {
     bool warn = false;
     if (MCRegisterInfo::regsOverlap(PhysReg, AArch64::X13) ||
@@ -448,6 +460,16 @@ AArch64RegisterInfo::getStrictlyReservedRegs(const MachineFunction &MF) const {
       markSuperRegs(Reserved, i);
   }
 
+  if (MF.getSubtarget<AArch64Subtarget>().isAArch64LFI()) {
+    markSuperRegs(Reserved, AArch64::W28);
+    markSuperRegs(Reserved, AArch64::W27);
+    markSuperRegs(Reserved, AArch64::W26);
+    if (!MF.getProperties().hasProperty(MachineFunctionProperties::Property::NoVRegs)) {
+      markSuperRegs(Reserved, AArch64::LR);
+      markSuperRegs(Reserved, AArch64::W30);
+    }
+  }
+
   for (size_t i = 0; i < AArch64::GPR32commonRegClass.getNumRegs(); ++i) {
     if (MF.getSubtarget<AArch64Subtarget>().isXRegisterReserved(i))
       markSuperRegs(Reserved, AArch64::GPR32commonRegClass.getRegister(i));
diff --git a/llvm/lib/Target/AArch64/AArch64Subtarget.h b/llvm/lib/Target/AArch64/AArch64Subtarget.h
index c6eb77e3bc3ba..cbc60a2e52567 100644
--- a/llvm/lib/Target/AArch64/AArch64Subtarget.h
+++ b/llvm/lib/Target/AArch64/AArch64Subtarget.h
@@ -288,6 +288,7 @@ class AArch64Subtarget final : public AArch64GenSubtargetInfo {
   bool isTargetAndroid() const { return TargetTriple.isAndroid(); }
   bool isTargetFuchsia() const { return TargetTriple.isOSFuchsia(); }
   bool isWindowsArm64EC() const { return TargetTriple.isWindowsArm64EC(); }
+  bool isAArch64LFI() const { return TargetTriple.isAArch64LFI(); }
 
   bool isTargetCOFF() const { return TargetTriple.isOSBinFormatCOFF(); }
   bool isTargetELF() const { return TargetTriple.isOSBinFormatELF(); }
diff --git a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
index 07f072446081a..846892b79cd64 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
@@ -369,6 +369,11 @@ AArch64TargetMachine::AArch64TargetMachine(const Target &T, const Triple &TT,
     this->Options.NoTrapAfterNoreturn = true;
   }
 
+  // Disable jump table compression since it may cause assembler errors after
+  // LFI instrumentation is applied.
+  if (TT.isAArch64LFI())
+    EnableCompressJumpTables = false;
+
   if (getMCAsmInfo()->usesWindowsCFI()) {
     // Unwinding can get confused if the last instruction in an
     // exception-handling region (function, funclet, try block, etc.)
diff --git a/llvm/lib/TargetParser/Triple.cpp b/llvm/lib/TargetParser/Triple.cpp
index e9e6f130f757c..6a18790ca6e83 100644
--- a/llvm/lib/TargetParser/Triple.cpp
+++ b/llvm/lib/TargetParser/Triple.cpp
@@ -112,6 +112,12 @@ StringRef Triple::getArchName(ArchType Kind, SubArchType SubArch) {
       return "arm64ec";
     if (SubArch == AArch64SubArch_arm64e)
       return "arm64e";
+    if (SubArch == AArch64SubArch_lfi)
+      return "aarch64_lfi";
+    if (SubArch == AArch64SubArch_lfi_stores)
+      return "aarch64_lfi_stores";
+    if (SubArch == AArch64SubArch_lfi_jumps)
+      return "aarch64_lfi_jumps";
     break;
   case Triple::spirv:
     switch (SubArch) {
@@ -575,6 +581,9 @@ static Triple::ArchType parseArch(StringRef ArchName) {
           .Case("arm64_32", Triple::aarch64_32)
           .Case("arm64e", Triple::aarch64)
           .Case("arm64ec", Triple::aarch64)
+          .Case("aarch64_lfi", Triple::aarch64)
+          .Case("aarch64_lfi_stores", Triple::aarch64)
+          .Case("aarch64_lfi_jumps", Triple::aarch64)
           .Case("arm", Triple::arm)
           .Case("armeb", Triple::armeb)
           .Case("thumb", Triple::thumb)
@@ -797,6 +806,15 @@ static Triple::SubArchType parseSubArch(StringRef SubArchName) {
   if (SubArchName == "arm64ec")
     return Triple::AArch64SubArch_arm64ec;
 
+  if (SubArchName == "aarch64_lfi")
+    return Triple::AArch64SubArch_lfi;
+
+  if (SubArchName == "aarch64_lfi_stores")
+    return Triple::AArch64SubArch_lfi_stores;
+
+  if (SubArchName == "aarch64_lfi_jumps")
+    return Triple::AArch64SubArch_lfi_jumps;
+
   if (SubArchName.starts_with("spirv"))
     return StringSwitch<Triple::SubArchType>(SubArchName)
         .EndsWith("v1.0", Triple::SPIRVSubArch_v10)

From 08019fe93d816556041fbf153cc2a50e6848644a Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Tue, 19 Aug 2025 12:25:46 -0700
Subject: [PATCH 02/23] [LFI] Add LFILinux Clang toolchain

---
 clang/lib/Basic/Targets/AArch64.cpp      |  4 +++
 clang/lib/Driver/CMakeLists.txt          |  1 +
 clang/lib/Driver/Driver.cpp              |  3 ++
 clang/lib/Driver/ToolChains/LFILinux.cpp | 29 +++++++++++++++++++
 clang/lib/Driver/ToolChains/LFILinux.h   | 36 ++++++++++++++++++++++++
 llvm/include/llvm/TargetParser/Triple.h  |  4 +++
 6 files changed, 77 insertions(+)
 create mode 100644 clang/lib/Driver/ToolChains/LFILinux.cpp
 create mode 100644 clang/lib/Driver/ToolChains/LFILinux.h

diff --git a/clang/lib/Basic/Targets/AArch64.cpp b/clang/lib/Basic/Targets/AArch64.cpp
index cabda0a1323a3..a3464a9b71d9d 100644
--- a/clang/lib/Basic/Targets/AArch64.cpp
+++ b/clang/lib/Basic/Targets/AArch64.cpp
@@ -413,6 +413,10 @@ void AArch64TargetInfo::getTargetDefines(const LangOptions &Opts,
     Builder.defineMacro("__aarch64__");
   }
 
+  if (getTriple().isAArch64LFI()) {
+    Builder.defineMacro("__LFI__");
+  }
+
   // Inline assembly supports AArch64 flag outputs.
   Builder.defineMacro("__GCC_ASM_FLAG_OUTPUTS__");
 
diff --git a/clang/lib/Driver/CMakeLists.txt b/clang/lib/Driver/CMakeLists.txt
index 5bdb6614389cf..9fd7beb4b29a7 100644
--- a/clang/lib/Driver/CMakeLists.txt
+++ b/clang/lib/Driver/CMakeLists.txt
@@ -64,6 +64,7 @@ add_clang_library(clangDriver
   ToolChains/Hexagon.cpp
   ToolChains/HLSL.cpp
   ToolChains/Hurd.cpp
+  ToolChains/LFILinux.cpp
   ToolChains/Linux.cpp
   ToolChains/MipsLinux.cpp
   ToolChains/MinGW.cpp
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index 87855fdb79971..6183798031d96 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -28,6 +28,7 @@
 #include "ToolChains/Haiku.h"
 #include "ToolChains/Hexagon.h"
 #include "ToolChains/Hurd.h"
+#include "ToolChains/LFILinux.h"
 #include "ToolChains/Lanai.h"
 #include "ToolChains/Linux.h"
 #include "ToolChains/MSP430.h"
@@ -6654,6 +6655,8 @@ const ToolChain &Driver::getToolChain(const ArgList &Args,
         TC = std::make_unique<toolchains::VEToolChain>(*this, Target, Args);
       else if (Target.isOHOSFamily())
         TC = std::make_unique<toolchains::OHOS>(*this, Target, Args);
+      else if (Target.isLFI())
+        TC = std::make_unique<toolchains::LFILinuxToolChain>(*this, Target, Args);
       else
         TC = std::make_unique<toolchains::Linux>(*this, Target, Args);
       break;
diff --git a/clang/lib/Driver/ToolChains/LFILinux.cpp b/clang/lib/Driver/ToolChains/LFILinux.cpp
new file mode 100644
index 0000000000000..37510c596b303
--- /dev/null
+++ b/clang/lib/Driver/ToolChains/LFILinux.cpp
@@ -0,0 +1,29 @@
+//===-- LFILinux.cpp - LFI ToolChain Implementations ------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "LFILinux.h"
+#include "clang/Driver/Driver.h"
+#include "clang/Driver/Options.h"
+
+using namespace clang::driver::toolchains;
+using namespace llvm::opt;
+
+void LFILinuxToolChain::AddCXXStdlibLibArgs(
+    const ArgList &Args, ArgStringList &CmdArgs) const {
+  switch (GetCXXStdlibType(Args)) {
+  case ToolChain::CST_Libcxx:
+    CmdArgs.push_back("-lc++");
+    if (Args.hasArg(options::OPT_fexperimental_library))
+      CmdArgs.push_back("-lc++experimental");
+    CmdArgs.push_back("-lc++abi");
+    break;
+  case ToolChain::CST_Libstdcxx:
+    CmdArgs.push_back("-lstdc++");
+    break;
+  }
+}
diff --git a/clang/lib/Driver/ToolChains/LFILinux.h b/clang/lib/Driver/ToolChains/LFILinux.h
new file mode 100644
index 0000000000000..4ef7b7334e8b0
--- /dev/null
+++ b/clang/lib/Driver/ToolChains/LFILinux.h
@@ -0,0 +1,36 @@
+//===--- LFILinux.h - LFI ToolChain Implementations -------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_LFI_LINUX_H
+#define LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_LFI_LINUX_H
+
+#include "Linux.h"
+
+namespace clang {
+namespace driver {
+namespace toolchains {
+
+class LLVM_LIBRARY_VISIBILITY LFILinuxToolChain : public Linux {
+public:
+  LFILinuxToolChain(const Driver &D, const llvm::Triple &Triple,
+                    const llvm::opt::ArgList &Args)
+      : Linux(D, Triple, Args) {
+        ExtraOpts.push_back("-z");
+        ExtraOpts.push_back("separate-code");
+      }
+
+  void
+  AddCXXStdlibLibArgs(const llvm::opt::ArgList &Args,
+                            llvm::opt::ArgStringList &CmdArgs) const override;
+};
+
+} // end namespace toolchains
+} // end namespace driver
+} // end namespace clang
+
+#endif // LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_LFI_LINUX_H
diff --git a/llvm/include/llvm/TargetParser/Triple.h b/llvm/include/llvm/TargetParser/Triple.h
index afecf2c39bf74..642c9e7a04436 100644
--- a/llvm/include/llvm/TargetParser/Triple.h
+++ b/llvm/include/llvm/TargetParser/Triple.h
@@ -680,6 +680,10 @@ class Triple {
            getSubArch() == Triple::AArch64SubArch_arm64ec;
   }
 
+  bool isLFI() const {
+    return isAArch64LFI();
+  }
+
   /// Checks if we're targeting any subarch of AArch64 LFI.
   bool isAArch64LFI() const {
     return getArch() == Triple::aarch64 &&

From cb92ceb0996152ff173b4b98376cfd02a13f42dd Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Tue, 19 Aug 2025 14:28:12 -0700
Subject: [PATCH 03/23] [LFI] [AArch64] Machinery for LFI MC rewriter

This patch establishes the scaffolding for the LFI MC rewriter, based on
the NaCl auto-sandboxing assembler. There is a generic MCLFIExpander,
plus an AArch64-specific subclass where the rewriter will be implemented
in a future patch.
---
 llvm/include/llvm/MC/MCLFI.h                  |  22 ++++
 llvm/include/llvm/MC/MCLFIExpander.h          |  74 ++++++++++++
 llvm/include/llvm/MC/MCStreamer.h             |   7 ++
 llvm/include/llvm/MC/TargetRegistry.h         |  21 ++++
 llvm/lib/MC/CMakeLists.txt                    |   2 +
 llvm/lib/MC/MCAsmStreamer.cpp                 |   4 +
 llvm/lib/MC/MCLFI.cpp                         |  79 +++++++++++++
 llvm/lib/MC/MCLFIExpander.cpp                 | 107 ++++++++++++++++++
 llvm/lib/MC/MCObjectStreamer.cpp              |   6 +-
 llvm/lib/MC/MCParser/AsmParser.cpp            |   8 +-
 llvm/lib/MC/MCParser/CMakeLists.txt           |   1 +
 llvm/lib/MC/MCParser/LFIAsmParser.cpp         |  84 ++++++++++++++
 llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp |   5 +
 .../MCTargetDesc/AArch64MCLFIExpander.cpp     |  79 +++++++++++++
 .../MCTargetDesc/AArch64MCLFIExpander.h       |  72 ++++++++++++
 .../MCTargetDesc/AArch64MCTargetDesc.cpp      |  13 +++
 .../AArch64/MCTargetDesc/CMakeLists.txt       |   1 +
 17 files changed, 583 insertions(+), 2 deletions(-)
 create mode 100644 llvm/include/llvm/MC/MCLFI.h
 create mode 100644 llvm/include/llvm/MC/MCLFIExpander.h
 create mode 100644 llvm/lib/MC/MCLFI.cpp
 create mode 100644 llvm/lib/MC/MCLFIExpander.cpp
 create mode 100644 llvm/lib/MC/MCParser/LFIAsmParser.cpp
 create mode 100644 llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
 create mode 100644 llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.h

diff --git a/llvm/include/llvm/MC/MCLFI.h b/llvm/include/llvm/MC/MCLFI.h
new file mode 100644
index 0000000000000..a50a228d13643
--- /dev/null
+++ b/llvm/include/llvm/MC/MCLFI.h
@@ -0,0 +1,22 @@
+//===- MCLFI.h - LFI-specific code for MC -----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// This file was written by the Native Client authors, modified for LFI.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Support/CommandLine.h"
+
+namespace llvm {
+
+class MCContext;
+class MCStreamer;
+class Triple;
+
+void initializeLFIMCStreamer(MCStreamer &Streamer, MCContext &Ctx,
+                             const Triple &TheTriple);
+
+}
diff --git a/llvm/include/llvm/MC/MCLFIExpander.h b/llvm/include/llvm/MC/MCLFIExpander.h
new file mode 100644
index 0000000000000..195e298c42e84
--- /dev/null
+++ b/llvm/include/llvm/MC/MCLFIExpander.h
@@ -0,0 +1,74 @@
+//===- llvm/MC/MCLFIExpander.h ----------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// This file was written by the Native Client authors, modified for LFI.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declares the MCLFIExpander class. This is an abstract
+// class that encapsulates the expansion logic for MCInsts, and holds
+// state such as available scratch registers.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_MC_MCLFIEXPANDER_H
+#define LLVM_MC_MCLFIEXPANDER_H
+
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCInstrInfo.h"
+#include "llvm/MC/MCRegisterInfo.h"
+
+namespace llvm {
+class MCInst;
+class MCSubtargetInfo;
+class MCStreamer;
+
+class MCLFIExpander {
+private:
+  SmallVector<MCRegister, 2> ScratchRegs;
+  MCContext &Ctx;
+
+protected:
+  std::unique_ptr<MCInstrInfo> InstInfo;
+  std::unique_ptr<MCRegisterInfo> RegInfo;
+  void invalidateScratchRegs(const MCInst &Inst);
+  MCRegister getScratchReg(int index);
+  unsigned numScratchRegs() const;
+  virtual bool isValidScratchRegister(MCRegister Reg) const = 0;
+
+public:
+  MCLFIExpander(MCContext &Ctx, std::unique_ptr<MCRegisterInfo> &&RI,
+                 std::unique_ptr<MCInstrInfo> &&II)
+      : Ctx(Ctx), InstInfo(std::move(II)), RegInfo(std::move(RI)) {}
+
+  void Error(const MCInst &Inst, const char msg[]);
+
+  bool addScratchReg(MCRegister Reg);
+  void clearScratchRegs();
+
+  bool isPseudo(const MCInst &Inst) const;
+
+  bool mayAffectControlFlow(const MCInst &Inst) const;
+  bool isCall(const MCInst &Inst) const;
+  bool isBranch(const MCInst &Inst) const;
+  bool isIndirectBranch(const MCInst &Inst) const;
+  bool isReturn(const MCInst &Inst) const;
+  bool isVariadic(const MCInst &Inst) const;
+
+  bool mayLoad(const MCInst &Inst) const;
+  bool mayStore(const MCInst &Inst) const;
+
+  bool mayModifyRegister(const MCInst &Inst, MCRegister Reg) const;
+  bool explicitlyModifiesRegister(const MCInst &Inst, MCRegister Reg) const;
+
+  virtual ~MCLFIExpander() = default;
+  virtual bool expandInst(const MCInst &Inst, MCStreamer &Out,
+                          const MCSubtargetInfo &STI) = 0;
+};
+
+}
+#endif
diff --git a/llvm/include/llvm/MC/MCStreamer.h b/llvm/include/llvm/MC/MCStreamer.h
index 558b14cebfd3d..96b6aff836932 100644
--- a/llvm/include/llvm/MC/MCStreamer.h
+++ b/llvm/include/llvm/MC/MCStreamer.h
@@ -21,6 +21,7 @@
 #include "llvm/MC/MCDwarf.h"
 #include "llvm/MC/MCFragment.h"
 #include "llvm/MC/MCLinkerOptimizationHint.h"
+#include "llvm/MC/MCLFIExpander.h"
 #include "llvm/MC/MCPseudoProbe.h"
 #include "llvm/MC/MCWinEH.h"
 #include "llvm/Support/Error.h"
@@ -276,6 +277,8 @@ class MCStreamer {
   /// Returns true if the .cv_loc directive is in the right section.
   bool checkCVLocSection(unsigned FuncId, unsigned FileNo, SMLoc Loc);
 
+  std::unique_ptr<MCLFIExpander> LFIExpander;
+
 public:
   MCStreamer(const MCStreamer &) = delete;
   MCStreamer &operator=(const MCStreamer &) = delete;
@@ -293,6 +296,10 @@ class MCStreamer {
     return StartTokLocPtr ? *StartTokLocPtr : SMLoc();
   }
 
+  void setLFIExpander(MCLFIExpander *Exp) { LFIExpander.reset(Exp); }
+
+  MCLFIExpander *getLFIExpander() { return LFIExpander.get(); }
+
   /// State management
   ///
   virtual void reset();
diff --git a/llvm/include/llvm/MC/TargetRegistry.h b/llvm/include/llvm/MC/TargetRegistry.h
index 42d510c17bce3..abeb1d4405be7 100644
--- a/llvm/include/llvm/MC/TargetRegistry.h
+++ b/llvm/include/llvm/MC/TargetRegistry.h
@@ -46,6 +46,7 @@ class MCDisassembler;
 class MCInstPrinter;
 class MCInstrAnalysis;
 class MCInstrInfo;
+class MCLFIExpander;
 class MCObjectWriter;
 class MCRegisterInfo;
 class MCRelocationInfo;
@@ -230,6 +231,10 @@ class Target {
       mca::InstrumentManager *(*)(const MCSubtargetInfo &STI,
                                   const MCInstrInfo &MCII);
 
+  using MCLFIExpanderCtorTy = MCLFIExpander *(*)(
+      MCStreamer &S, std::unique_ptr<MCRegisterInfo> &&RegInfo,
+      std::unique_ptr<MCInstrInfo> &&InstInfo);
+
 private:
   /// Next - The next registered target in the linked list, maintained by the
   /// TargetRegistry.
@@ -340,6 +345,10 @@ class Target {
   /// InstrumentManager, if registered (default = nullptr).
   InstrumentManagerCtorTy InstrumentManagerCtorFn = nullptr;
 
+  // MCLFIExpanderCtorFn - Construction function for this target's
+  // MCLFIExpander, if registered.
+  MCLFIExpanderCtorTy MCLFIExpanderCtorFn;
+
 public:
   Target() = default;
 
@@ -538,6 +547,13 @@ class Target {
                                      const MCSubtargetInfo &STI, bool, bool,
                                      bool) const;
 
+  void createMCLFIExpander(MCStreamer &S,
+                           std::unique_ptr<MCRegisterInfo> &&RegInfo,
+                           std::unique_ptr<MCInstrInfo> &&InstInfo) const {
+    if (MCLFIExpanderCtorFn)
+      MCLFIExpanderCtorFn(S, std::move(RegInfo), std::move(InstInfo));
+  }
+
   MCStreamer *createAsmStreamer(MCContext &Ctx,
                                 std::unique_ptr<formatted_raw_ostream> OS,
                                 MCInstPrinter *IP,
@@ -1008,6 +1024,11 @@ struct TargetRegistry {
     T.InstrumentManagerCtorFn = Fn;
   }
 
+  static void RegisterMCLFIExpander(Target &T,
+                                    Target::MCLFIExpanderCtorTy Fn) {
+    T.MCLFIExpanderCtorFn = Fn;
+  }
+
   /// @}
 };
 
diff --git a/llvm/lib/MC/CMakeLists.txt b/llvm/lib/MC/CMakeLists.txt
index e1d19196c8766..0456768b9cfaf 100644
--- a/llvm/lib/MC/CMakeLists.txt
+++ b/llvm/lib/MC/CMakeLists.txt
@@ -30,6 +30,8 @@ add_llvm_component_library(LLVMMC
   MCInstrAnalysis.cpp
   MCInstrDesc.cpp
   MCInstrInfo.cpp
+  MCLFI.cpp
+  MCLFIExpander.cpp
   MCLabel.cpp
   MCLinkerOptimizationHint.cpp
   MCMachOStreamer.cpp
diff --git a/llvm/lib/MC/MCAsmStreamer.cpp b/llvm/lib/MC/MCAsmStreamer.cpp
index dd8058c6d5cd8..38d8976965c81 100644
--- a/llvm/lib/MC/MCAsmStreamer.cpp
+++ b/llvm/lib/MC/MCAsmStreamer.cpp
@@ -20,6 +20,7 @@
 #include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCInstPrinter.h"
+#include "llvm/MC/MCLFIExpander.h"
 #include "llvm/MC/MCObjectFileInfo.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCPseudoProbe.h"
@@ -2453,6 +2454,9 @@ void MCAsmStreamer::AddEncodingComment(const MCInst &Inst,
 
 void MCAsmStreamer::emitInstruction(const MCInst &Inst,
                                     const MCSubtargetInfo &STI) {
+  if (LFIExpander && LFIExpander->expandInst(Inst, *this, STI))
+    return;
+
   if (MAI->isAIX() && CurFrag)
     // Now that a machine instruction has been assembled into this section, make
     // a line entry for any .loc directive that has been seen.
diff --git a/llvm/lib/MC/MCLFI.cpp b/llvm/lib/MC/MCLFI.cpp
new file mode 100644
index 0000000000000..3cad2e4eff452
--- /dev/null
+++ b/llvm/lib/MC/MCLFI.cpp
@@ -0,0 +1,79 @@
+//===- lib/MC/MCLFI.cpp - LFI-specific MC implementation ------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// This file was written by the Native Client authors, modified for LFI.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/MC/MCLFI.h"
+#include "llvm/BinaryFormat/ELF.h"
+#include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCLFIExpander.h"
+#include "llvm/MC/MCSectionELF.h"
+#include "llvm/MC/MCStreamer.h"
+#include "llvm/MC/TargetRegistry.h"
+#include "llvm/Support/Alignment.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/TargetParser/Triple.h"
+
+static const char NoteNamespace[] = "LFI";
+
+namespace llvm {
+
+cl::opt<bool> FlagEnableAutoSandboxing("nacl-enable-autosandboxing",
+                                cl::desc("Don't use the autosandboxing"
+                                         " assembler for the NaCl SFI."),
+                                cl::init(true));
+
+void initializeLFIMCStreamer(MCStreamer &Streamer, MCContext &Ctx,
+                              const Triple &TheTriple) {
+  assert(TheTriple.isLFI());
+  const char *NoteName;
+  const char *NoteArch;
+  switch (TheTriple.getArch()) {
+    case Triple::aarch64:
+      NoteName = ".note.LFI.ABI.aarch64";
+      NoteArch = "aarch64";
+      break;
+    default:
+      report_fatal_error("Unsupported architecture for LFI");
+  }
+
+  std::string Error; //empty
+  const Target *TheTarget =
+    TargetRegistry::lookupTarget(TheTriple.getTriple(), Error);
+
+  // Create the Target specific MCLFIExpander
+  assert(TheTarget != nullptr);
+  if (FlagEnableAutoSandboxing) {
+    TheTarget->createMCLFIExpander(
+      Streamer,
+      std::unique_ptr<MCRegisterInfo>(
+          TheTarget->createMCRegInfo(TheTriple.getTriple())),
+      std::unique_ptr<MCInstrInfo>(TheTarget->createMCInstrInfo()));
+  }
+
+  // Emit an ELF Note section in its own COMDAT group which identifies LFI
+  // object files.
+  MCSectionELF* Note = Ctx.getELFSection(NoteName, ELF::SHT_NOTE,
+                                         ELF::SHF_ALLOC | ELF::SHF_GROUP,
+                                         0, NoteName, /*IsComdat=*/true);
+
+  Streamer.pushSection();
+  Streamer.switchSection(Note);
+  Streamer.emitIntValue(strlen(NoteNamespace) + 1, 4);
+  Streamer.emitIntValue(strlen(NoteArch) + 1, 4);
+  Streamer.emitIntValue(ELF::NT_VERSION, 4);
+  Streamer.emitBytes(NoteNamespace);
+  Streamer.emitIntValue(0, 1); // NUL terminator
+  Streamer.emitValueToAlignment(Align(4));
+  Streamer.emitBytes(NoteArch);
+  Streamer.emitIntValue(0, 1); // NUL terminator
+  Streamer.emitValueToAlignment(Align(4));
+  Streamer.popSection();
+}
+
+} // namespace llvm
diff --git a/llvm/lib/MC/MCLFIExpander.cpp b/llvm/lib/MC/MCLFIExpander.cpp
new file mode 100644
index 0000000000000..900e1ec17418d
--- /dev/null
+++ b/llvm/lib/MC/MCLFIExpander.cpp
@@ -0,0 +1,107 @@
+//===- MCLFIExpander.cpp ----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// This file was written by the Native Client authors, modified for LFI.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the MCLFIExpander class. This is a base
+// class that encapsulates the expansion logic for MCInsts, and holds
+// state such as available scratch registers.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/MC/MCLFIExpander.h"
+#include "llvm/MC/MCInst.h"
+#include "llvm/MC/MCInstrInfo.h"
+#include "llvm/MC/MCRegisterInfo.h"
+
+namespace llvm {
+
+void MCLFIExpander::Error(const MCInst &Inst, const char msg[]) {
+  Ctx.reportError(Inst.getLoc(), msg);
+}
+
+bool MCLFIExpander::addScratchReg(MCRegister Reg) {
+  if (!isValidScratchRegister(Reg))
+    return true;
+  ScratchRegs.push_back(Reg);
+  return false;
+}
+
+void MCLFIExpander::invalidateScratchRegs(const MCInst &Inst) {
+  // TODO: There are arch-specific special cases where this fails, e.g.
+  // xchg/cmpxchg
+  // TODO(zyedidia): investigate whether there are cases for AArch64.
+  const MCInstrDesc &Desc = InstInfo->get(Inst.getOpcode());
+  for (auto I = ScratchRegs.begin(), E = ScratchRegs.end(); I != E; ++I) {
+    if (Desc.hasDefOfPhysReg(Inst, *I, *RegInfo))
+      I = ScratchRegs.erase(I);
+  }
+}
+
+void MCLFIExpander::clearScratchRegs() {
+  ScratchRegs.clear();
+}
+
+MCRegister MCLFIExpander::getScratchReg(int index) {
+  assert(index >= 0 && static_cast<unsigned>(index) < numScratchRegs());
+  return ScratchRegs[numScratchRegs()  - index - 1];
+}
+
+unsigned MCLFIExpander::numScratchRegs() const { return ScratchRegs.size(); }
+
+bool MCLFIExpander::isPseudo(const MCInst &Inst) const {
+  return InstInfo->get(Inst.getOpcode()).isPseudo();
+}
+
+bool MCLFIExpander::mayAffectControlFlow(const MCInst &Inst) const {
+  return InstInfo->get(Inst.getOpcode()).mayAffectControlFlow(Inst, *RegInfo);
+}
+
+bool MCLFIExpander::isCall(const MCInst &Inst) const {
+  return InstInfo->get(Inst.getOpcode()).isCall();
+}
+
+bool MCLFIExpander::isBranch(const MCInst &Inst) const {
+  return InstInfo->get(Inst.getOpcode()).isBranch();
+}
+
+bool MCLFIExpander::isIndirectBranch(const MCInst &Inst) const {
+  return InstInfo->get(Inst.getOpcode()).isIndirectBranch();
+}
+
+bool MCLFIExpander::isReturn(const MCInst &Inst) const {
+  return InstInfo->get(Inst.getOpcode()).isReturn();
+}
+
+bool MCLFIExpander::isVariadic(const MCInst &Inst) const {
+  return InstInfo->get(Inst.getOpcode()).isVariadic();
+}
+
+bool MCLFIExpander::mayLoad(const MCInst &Inst) const {
+  return InstInfo->get(Inst.getOpcode()).mayLoad();
+}
+
+bool MCLFIExpander::mayStore(const MCInst &Inst) const {
+  return InstInfo->get(Inst.getOpcode()).mayStore();
+}
+
+bool MCLFIExpander::mayModifyRegister(const MCInst &Inst, MCRegister Reg) const {
+  return InstInfo->get(Inst.getOpcode()).hasDefOfPhysReg(Inst, Reg, *RegInfo);
+}
+
+bool MCLFIExpander::explicitlyModifiesRegister(const MCInst &Inst,
+                                                MCRegister Reg) const {
+  const MCInstrDesc &Desc = InstInfo->get(Inst.getOpcode());
+  for (int i = 0; i < Desc.NumDefs; ++i) {
+    if (Desc.operands()[i].OperandType == MCOI::OPERAND_REGISTER &&
+        RegInfo->isSubRegisterEq(Reg, Inst.getOperand(i).getReg()))
+      return true;
+  }
+  return false;
+}
+}
diff --git a/llvm/lib/MC/MCObjectStreamer.cpp b/llvm/lib/MC/MCObjectStreamer.cpp
index fff30955b2576..75c3f00d64ed2 100644
--- a/llvm/lib/MC/MCObjectStreamer.cpp
+++ b/llvm/lib/MC/MCObjectStreamer.cpp
@@ -15,6 +15,7 @@
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCDwarf.h"
 #include "llvm/MC/MCExpr.h"
+#include "llvm/MC/MCLFIExpander.h"
 #include "llvm/MC/MCObjectFileInfo.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCSection.h"
@@ -349,9 +350,12 @@ void MCObjectStreamer::emitInstruction(const MCInst &Inst,
 
 void MCObjectStreamer::emitInstructionImpl(const MCInst &Inst,
                                            const MCSubtargetInfo &STI) {
+  MCSection *Sec = getCurrentSectionOnly();
+  if (LFIExpander && LFIExpander->expandInst(Inst, *this, STI))
+    return;
+
   MCStreamer::emitInstruction(Inst, STI);
 
-  MCSection *Sec = getCurrentSectionOnly();
   Sec->setHasInstructions(true);
 
   // Now that a machine instruction has been assembled into this section, make
diff --git a/llvm/lib/MC/MCParser/AsmParser.cpp b/llvm/lib/MC/MCParser/AsmParser.cpp
index bf952df1b2418..4f4d6fb9a7065 100644
--- a/llvm/lib/MC/MCParser/AsmParser.cpp
+++ b/llvm/lib/MC/MCParser/AsmParser.cpp
@@ -125,6 +125,7 @@ class AsmParser : public MCAsmParser {
   SourceMgr::DiagHandlerTy SavedDiagHandler;
   void *SavedDiagContext;
   std::unique_ptr<MCAsmParserExtension> PlatformParser;
+  std::unique_ptr<MCAsmParserExtension> LFIParser;
   SMLoc StartTokLoc;
   std::optional<SMLoc> CFIStartProcLoc;
 
@@ -769,6 +770,7 @@ extern MCAsmParserExtension *createCOFFAsmParser();
 extern MCAsmParserExtension *createGOFFAsmParser();
 extern MCAsmParserExtension *createXCOFFAsmParser();
 extern MCAsmParserExtension *createWasmAsmParser();
+extern MCAsmParserExtension *createLFIAsmParser(MCLFIExpander *Exp);
 
 } // end namespace llvm
 
@@ -776,7 +778,7 @@ enum { DEFAULT_ADDRSPACE = 0 };
 
 AsmParser::AsmParser(SourceMgr &SM, MCContext &Ctx, MCStreamer &Out,
                      const MCAsmInfo &MAI, unsigned CB = 0)
-    : Lexer(MAI), Ctx(Ctx), Out(Out), MAI(MAI), SrcMgr(SM),
+    : Lexer(MAI), Ctx(Ctx), Out(Out), MAI(MAI), SrcMgr(SM), LFIParser(nullptr),
       CurBuffer(CB ? CB : SM.getMainFileID()), MacrosEnabledFlag(true) {
   HadError = false;
   // Save the old handler.
@@ -819,6 +821,10 @@ AsmParser::AsmParser(SourceMgr &SM, MCContext &Ctx, MCStreamer &Out,
   }
 
   PlatformParser->Initialize(*this);
+  if (Out.getLFIExpander()) {
+    LFIParser.reset(createLFIAsmParser(Out.getLFIExpander()));
+    LFIParser->Initialize(*this);
+  }
   initializeDirectiveKindMap();
   initializeCVDefRangeTypeMap();
 
diff --git a/llvm/lib/MC/MCParser/CMakeLists.txt b/llvm/lib/MC/MCParser/CMakeLists.txt
index d3fa2675a255e..93f0ff4ca0140 100644
--- a/llvm/lib/MC/MCParser/CMakeLists.txt
+++ b/llvm/lib/MC/MCParser/CMakeLists.txt
@@ -6,6 +6,7 @@ add_llvm_component_library(LLVMMCParser
   GOFFAsmParser.cpp
   DarwinAsmParser.cpp
   ELFAsmParser.cpp
+  LFIAsmParser.cpp
   MCAsmLexer.cpp
   MCAsmParser.cpp
   MCAsmParserExtension.cpp
diff --git a/llvm/lib/MC/MCParser/LFIAsmParser.cpp b/llvm/lib/MC/MCParser/LFIAsmParser.cpp
new file mode 100644
index 0000000000000..cf9c172bd423b
--- /dev/null
+++ b/llvm/lib/MC/MCParser/LFIAsmParser.cpp
@@ -0,0 +1,84 @@
+//===- LFIAsmParser.cpp - LFI Assembly Parser -----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file was written by the Native Client authors, modified for LFI.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/MC/MCParser/MCAsmParserExtension.h"
+#include "llvm/MC/MCLFIExpander.h"
+#include "llvm/MC/MCParser/MCAsmLexer.h"
+#include "llvm/MC/MCParser/MCTargetAsmParser.h"
+#include "llvm/MC/MCRegister.h"
+#include "llvm/MC/MCStreamer.h"
+#include "llvm/Support/SourceMgr.h"
+#include "llvm/Support/raw_ostream.h"
+
+using namespace llvm;
+
+class LFIAsmParser : public MCAsmParserExtension {
+  MCLFIExpander *Expander;
+  template<bool (LFIAsmParser::*HandlerMethod)(StringRef, SMLoc)>
+  void addDirectiveHandler(StringRef Directive) {
+    MCAsmParser::ExtensionDirectiveHandler Handler = std::make_pair(
+        this, HandleDirective<LFIAsmParser, HandlerMethod>);
+
+    getParser().addDirectiveHandler(Directive, Handler);
+  }
+
+ public:
+  LFIAsmParser(MCLFIExpander *Exp) : Expander(Exp) {}
+  void Initialize(MCAsmParser &Parser) override {
+    // Call the base implementation.
+    MCAsmParserExtension::Initialize(Parser);
+    addDirectiveHandler<&LFIAsmParser::ParseScratch>(".scratch");
+    addDirectiveHandler<&LFIAsmParser::ParseUnscratch>(".scratch_clear");
+  }
+
+  /// ::= {.scratch} reg
+  bool ParseScratch(StringRef Directive, SMLoc Loc) {
+    getParser().checkForValidSection();
+    MCRegister RegNo;
+    const char *kInvalidOptionError =
+        "expected register name after '.scratch' directive";
+
+    if (getLexer().isNot(AsmToken::EndOfStatement)) {
+      if (getParser().getTargetParser().parseRegister(RegNo, Loc, Loc))
+        return Error(Loc, kInvalidOptionError);
+
+      else if (getLexer().isNot(AsmToken::EndOfStatement))
+        return Error(Loc, kInvalidOptionError);
+    }
+    else {
+      return Error(Loc, kInvalidOptionError);
+    }
+    Lex();
+
+    if(Expander->addScratchReg(RegNo))
+      return Error(Loc, "Register can't be used as a scratch register");
+    return false;
+  }
+
+  /// ::= {.unscratch}
+  bool ParseUnscratch(StringRef Directive, SMLoc Loc) {
+    getParser().checkForValidSection();
+    if (getLexer().isNot(AsmToken::EndOfStatement))
+      return TokError("unexpected token in '.scratch_clear' directive");
+    Lex();
+
+    Expander->clearScratchRegs();
+
+    return false;
+  }
+};
+
+namespace llvm {
+MCAsmParserExtension *createLFIAsmParser(MCLFIExpander *Exp) {
+  return new LFIAsmParser(Exp);
+}
+}
diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index c6b4a219d201f..b899c73d529ad 100644
--- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -50,6 +50,7 @@
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCInstBuilder.h"
+#include "llvm/MC/MCLFI.h"
 #include "llvm/MC/MCSectionELF.h"
 #include "llvm/MC/MCSectionMachO.h"
 #include "llvm/MC/MCStreamer.h"
@@ -394,6 +395,10 @@ void AArch64AsmPrinter::emitStartOfAsmFile(Module &M) {
 
   // Emit AArch64 Build Attributes
   emitAttributes(BAFlags, PAuthABIPlatform, PAuthABIVersion, TS);
+
+  if (TT.isLFI())
+    initializeLFIMCStreamer(*OutStreamer.get(), OutContext, TT);
+
   // Emit a .note.gnu.property section with the flags.
   TS->emitNoteSection(GNUFlags, PAuthABIPlatform, PAuthABIVersion);
 }
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
new file mode 100644
index 0000000000000..a2decbe6db0e1
--- /dev/null
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
@@ -0,0 +1,79 @@
+//===- AArch64MCLFIExpander.cpp ---------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the AArch64MCLFIExpander class, the AArch64 specific
+// subclass of MCLFIExpander.
+//
+//===----------------------------------------------------------------------===//
+#include "AArch64MCLFIExpander.h"
+#include "AArch64AddressingModes.h"
+#include "Utils/AArch64BaseInfo.h"
+
+#include "llvm/MC/MCInst.h"
+#include "llvm/MC/MCInstrDesc.h"
+#include "llvm/MC/MCInstrInfo.h"
+#include "llvm/MC/MCLFIExpander.h"
+#include "llvm/MC/MCObjectFileInfo.h"
+#include "llvm/MC/MCRegisterInfo.h"
+#include "llvm/MC/MCStreamer.h"
+#include "llvm/Support/Debug.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "lfi"
+
+bool AArch64::AArch64MCLFIExpander::isValidScratchRegister(MCRegister Reg) const {
+  return false;
+}
+
+void AArch64::AArch64MCLFIExpander::expandIndirectBranch(const MCInst &Inst,
+                                                  MCStreamer &Out,
+                                                  const MCSubtargetInfo &STI,
+                                                  bool isCall) {
+}
+
+void AArch64::AArch64MCLFIExpander::expandCall(const MCInst &Inst, MCStreamer &Out,
+                                        const MCSubtargetInfo &STI) {
+}
+
+void AArch64::AArch64MCLFIExpander::expandReturn(const MCInst &Inst, MCStreamer &Out,
+                                          const MCSubtargetInfo &STI) {
+}
+
+void AArch64::AArch64MCLFIExpander::expandControlFlow(const MCInst &Inst,
+                                               MCStreamer &Out,
+                                               const MCSubtargetInfo &STI) {
+
+}
+
+bool AArch64::AArch64MCLFIExpander::mayModifyStack(const MCInst &Inst) {
+  return false;
+}
+
+void AArch64::AArch64MCLFIExpander::expandStackManipulation(
+    const MCInst &Inst, MCStreamer &Out, const MCSubtargetInfo &STI) {
+}
+
+void AArch64::AArch64MCLFIExpander::expandPrefetch(const MCInst &Inst, MCStreamer &Out,
+                                            const MCSubtargetInfo &STI) {
+}
+
+void AArch64::AArch64MCLFIExpander::expandLoadStore(const MCInst &Inst,
+                                             MCStreamer &Out,
+                                             const MCSubtargetInfo &STI) {
+}
+
+void AArch64::AArch64MCLFIExpander::doExpandInst(const MCInst &Inst, MCStreamer &Out,
+                                          const MCSubtargetInfo &STI) {
+}
+
+bool AArch64::AArch64MCLFIExpander::expandInst(const MCInst &Inst, MCStreamer &Out,
+                                        const MCSubtargetInfo &STI) {
+  return false;
+}
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.h b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.h
new file mode 100644
index 0000000000000..194fa36eb9c2b
--- /dev/null
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.h
@@ -0,0 +1,72 @@
+//===- AArch64MCLFIExpander.h -----------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// This file was written by the Native Client authors, modified for LFI.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declares the AArch64MCLFIExpander class, the AArch64 specific
+// subclass of MCLFIExpander.
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_MC_AARCH64MCLFIEXPANDER_H
+#define LLVM_MC_AARCH64MCLFIEXPANDER_H
+
+#include "llvm/MC/MCInstrInfo.h"
+#include "llvm/MC/MCLFIExpander.h"
+#include "llvm/MC/MCRegisterInfo.h"
+
+namespace llvm {
+class MCContext;
+class MCInst;
+class MCStreamer;
+class MCSubtargetInfo;
+
+namespace AArch64 {
+class AArch64MCLFIExpander : public MCLFIExpander {
+public:
+  AArch64MCLFIExpander(MCContext &Ctx, std::unique_ptr<MCRegisterInfo> &&RI,
+                    std::unique_ptr<MCInstrInfo> &&II)
+      : MCLFIExpander(Ctx, std::move(RI), std::move(II)) {}
+
+  bool expandInst(const MCInst &Inst, MCStreamer &Out,
+                  const MCSubtargetInfo &STI) override;
+
+protected:
+  bool isValidScratchRegister(MCRegister Reg) const override;
+
+private:
+  bool Guard = false; // recursion guard
+
+  void expandIndirectBranch(const MCInst &Inst, MCStreamer &Out,
+                            const MCSubtargetInfo &STI, bool isCall);
+
+  void expandCall(const MCInst &Inst, MCStreamer &Out,
+                  const MCSubtargetInfo &STI);
+
+  void expandReturn(const MCInst &Inst, MCStreamer &Out,
+                    const MCSubtargetInfo &STI);
+
+  bool mayModifyStack(const MCInst &Inst);
+
+  void expandControlFlow(const MCInst &Inst, MCStreamer &Out,
+                         const MCSubtargetInfo &STI);
+
+  void expandStackManipulation(const MCInst &Inst, MCStreamer &Out,
+                               const MCSubtargetInfo &STI);
+
+  void expandPrefetch(const MCInst &Inst, MCStreamer &Out,
+                      const MCSubtargetInfo &STI);
+
+  void expandLoadStore(const MCInst &Inst, MCStreamer &Out,
+                       const MCSubtargetInfo &STI);
+
+  void doExpandInst(const MCInst &Inst, MCStreamer &Out,
+                    const MCSubtargetInfo &STI);
+};
+} // namespace AArch64
+} // namespace llvm
+#endif
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp
index c7f44ec018f5a..c7a044346baab 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp
@@ -13,6 +13,7 @@
 #include "AArch64MCTargetDesc.h"
 #include "AArch64ELFStreamer.h"
 #include "AArch64MCAsmInfo.h"
+#include "AArch64MCLFIExpander.h"
 #include "AArch64WinCOFFStreamer.h"
 #include "MCTargetDesc/AArch64AddressingModes.h"
 #include "MCTargetDesc/AArch64InstPrinter.h"
@@ -518,6 +519,16 @@ static MCInstrAnalysis *createAArch64InstrAnalysis(const MCInstrInfo *Info) {
   return new AArch64MCInstrAnalysis(Info);
 }
 
+static MCLFIExpander *createAArch64MCLFIExpander(MCStreamer &S,
+    std::unique_ptr<MCRegisterInfo> &&RegInfo,
+    std::unique_ptr<MCInstrInfo> &&InstInfo) {
+  auto *Exp = new AArch64::AArch64MCLFIExpander(S.getContext(),
+                                                std::move(RegInfo),
+                                                std::move(InstInfo));
+  S.setLFIExpander(Exp);
+  return Exp;
+}
+
 // Force static initialization.
 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAArch64TargetMC() {
   for (Target *T : {&getTheAArch64leTarget(), &getTheAArch64beTarget(),
@@ -546,6 +557,8 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAArch64TargetMC() {
     TargetRegistry::RegisterMachOStreamer(*T, createMachOStreamer);
     TargetRegistry::RegisterCOFFStreamer(*T, createWinCOFFStreamer);
 
+    TargetRegistry::RegisterMCLFIExpander(*T, createAArch64MCLFIExpander);
+
     // Register the obj target streamer.
     TargetRegistry::RegisterObjectTargetStreamer(
         *T, createAArch64ObjectTargetStreamer);
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/CMakeLists.txt b/llvm/lib/Target/AArch64/MCTargetDesc/CMakeLists.txt
index 7f220657e45f8..bd144eaddc0f4 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/CMakeLists.txt
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/CMakeLists.txt
@@ -6,6 +6,7 @@ add_llvm_component_library(LLVMAArch64Desc
   AArch64MCAsmInfo.cpp
   AArch64MCCodeEmitter.cpp
   AArch64MCExpr.cpp
+  AArch64MCLFIExpander.cpp
   AArch64MCTargetDesc.cpp
   AArch64MachObjectWriter.cpp
   AArch64TargetStreamer.cpp

From 8c37241040c79c73b311d6b11d1305b96e952564 Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Tue, 19 Aug 2025 15:03:40 -0700
Subject: [PATCH 04/23] [LFI] Apply LFI rewriter to .s files in clang
 cc1as/llvm-mc

When assembling a .s file using clang or llvm-mc, automatically run the
LFI rewriter if targeting LFI.
---
 clang/tools/driver/cc1as_main.cpp |  3 +++
 llvm/tools/llvm-mc/llvm-mc.cpp    | 14 ++++++++++++++
 2 files changed, 17 insertions(+)

diff --git a/clang/tools/driver/cc1as_main.cpp b/clang/tools/driver/cc1as_main.cpp
index 7fe97cc6e6ace..cb2f6bdb9c096 100644
--- a/clang/tools/driver/cc1as_main.cpp
+++ b/clang/tools/driver/cc1as_main.cpp
@@ -27,6 +27,7 @@
 #include "llvm/MC/MCCodeEmitter.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCInstrInfo.h"
+#include "llvm/MC/MCLFI.h"
 #include "llvm/MC/MCObjectFileInfo.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCParser/MCAsmParser.h"
@@ -577,6 +578,8 @@ static bool ExecuteAssemblerImpl(AssemblerInvocation &Opts,
     Str.reset(TheTarget->createMCObjectStreamer(
         T, Ctx, std::move(MAB), std::move(OW), std::move(CE), *STI));
     Str.get()->initSections(Opts.NoExecStack, *STI);
+    if (T.isLFI())
+      initializeLFIMCStreamer(*Str.get(), Ctx, T);
     if (T.isOSBinFormatMachO() && T.isOSDarwin()) {
       Triple *TVT = Opts.DarwinTargetVariantTriple
                         ? &*Opts.DarwinTargetVariantTriple
diff --git a/llvm/tools/llvm-mc/llvm-mc.cpp b/llvm/tools/llvm-mc/llvm-mc.cpp
index 70f92d09aded7..6de991c6d1bba 100644
--- a/llvm/tools/llvm-mc/llvm-mc.cpp
+++ b/llvm/tools/llvm-mc/llvm-mc.cpp
@@ -18,6 +18,7 @@
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCInstPrinter.h"
 #include "llvm/MC/MCInstrInfo.h"
+#include "llvm/MC/MCLFI.h"
 #include "llvm/MC/MCObjectFileInfo.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCParser/AsmLexer.h"
@@ -551,6 +552,12 @@ int main(int argc, char **argv) {
     Str.reset(TheTarget->createAsmStreamer(Ctx, std::move(FOut), IP,
                                            std::move(CE), std::move(MAB)));
 
+    Triple T(TripleName);
+    if (T.isLFI()) {
+      Str->initSections(NoExecStack, *STI);
+      initializeLFIMCStreamer(*Str.get(), Ctx, T);
+    }
+
   } else if (FileType == OFT_Null) {
     Str.reset(TheTarget->createNullStreamer(Ctx));
   } else {
@@ -570,6 +577,13 @@ int main(int argc, char **argv) {
         std::unique_ptr<MCCodeEmitter>(CE), *STI));
     if (NoExecStack)
       Str->switchSection(Ctx.getAsmInfo()->getNonexecutableStackSection(Ctx));
+
+    Triple T(TripleName);
+    if (T.isLFI()) {
+      Str->initSections(NoExecStack, *STI);
+      initializeLFIMCStreamer(*Str.get(), Ctx, T);
+    }
+
     Str->emitVersionForTarget(TheTriple, VersionTuple(), nullptr,
                               VersionTuple());
   }

From 6f62734b8689b82f79ceaf35a637d486038aab86 Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Tue, 19 Aug 2025 15:24:08 -0700
Subject: [PATCH 05/23] [LFI] [AArch64] LFI rewrites for indirect branches

---
 .../MCTargetDesc/AArch64MCLFIExpander.cpp     | 68 ++++++++++++++++++-
 llvm/test/MC/AArch64/LFI/branch.s             | 13 ++++
 2 files changed, 78 insertions(+), 3 deletions(-)
 create mode 100644 llvm/test/MC/AArch64/LFI/branch.s

diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
index a2decbe6db0e1..91588a4b46df3 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
@@ -28,18 +28,51 @@ using namespace llvm;
 
 #define DEBUG_TYPE "lfi"
 
+static MCRegister LFIAddrReg = AArch64::X28;
+static MCRegister LFIBaseReg = AArch64::X27;
+
 bool AArch64::AArch64MCLFIExpander::isValidScratchRegister(MCRegister Reg) const {
-  return false;
+  return Reg != AArch64::SP;
+}
+
+static void emitAddMask(MCRegister Dest, MCRegister Src, MCStreamer &Out,
+                        const MCSubtargetInfo &STI) {
+  MCInst Add;
+  Add.setOpcode(AArch64::ADDXrx);
+  Add.addOperand(MCOperand::createReg(Dest));
+  Add.addOperand(MCOperand::createReg(LFIBaseReg));
+  Add.addOperand(MCOperand::createReg(getWRegFromXReg(Src)));
+  Add.addOperand(MCOperand::createImm(AArch64_AM::getArithExtendImm(AArch64_AM::UXTW, 0)));
+  Out.emitInstruction(Add, STI);;
+}
+
+static void emitBranch(unsigned int Opcode, MCRegister Target, MCStreamer &Out,
+                       const MCSubtargetInfo &STI) {
+  MCInst Branch;
+  Branch.setOpcode(Opcode);
+  Branch.addOperand(MCOperand::createReg(Target));
+  Out.emitInstruction(Branch, STI);
 }
 
 void AArch64::AArch64MCLFIExpander::expandIndirectBranch(const MCInst &Inst,
                                                   MCStreamer &Out,
                                                   const MCSubtargetInfo &STI,
-                                                  bool isCall) {
+                                                  bool IsCall) {
+  (void) IsCall;
+
+  assert(Inst.getOperand(0).isReg());
+  MCRegister BranchReg = Inst.getOperand(0).getReg();
+
+  emitAddMask(LFIAddrReg, BranchReg, Out, STI);
+  emitBranch(Inst.getOpcode(), LFIAddrReg, Out, STI);
 }
 
 void AArch64::AArch64MCLFIExpander::expandCall(const MCInst &Inst, MCStreamer &Out,
                                         const MCSubtargetInfo &STI) {
+  if (Inst.getOperand(0).isReg())
+    expandIndirectBranch(Inst, Out, STI, true);
+  else
+    Out.emitInstruction(Inst, STI);
 }
 
 void AArch64::AArch64MCLFIExpander::expandReturn(const MCInst &Inst, MCStreamer &Out,
@@ -71,9 +104,38 @@ void AArch64::AArch64MCLFIExpander::expandLoadStore(const MCInst &Inst,
 
 void AArch64::AArch64MCLFIExpander::doExpandInst(const MCInst &Inst, MCStreamer &Out,
                                           const MCSubtargetInfo &STI) {
+  if (isReturn(Inst))
+    return expandReturn(Inst, Out, STI);
+
+  if (isIndirectBranch(Inst))
+    return expandIndirectBranch(Inst, Out, STI, false);
+
+  if (isCall(Inst))
+    return expandCall(Inst, Out, STI);
+
+  if (isBranch(Inst))
+    return Out.emitInstruction(Inst, STI);
+
+  if (mayAffectControlFlow(Inst))
+    return expandControlFlow(Inst, Out, STI);
+
+  if (mayModifyStack(Inst))
+    return expandStackManipulation(Inst, Out, STI);
+
+  if (mayLoad(Inst) || mayStore(Inst))
+    return expandLoadStore(Inst, Out, STI);
+
+  return Out.emitInstruction(Inst, STI);
 }
 
 bool AArch64::AArch64MCLFIExpander::expandInst(const MCInst &Inst, MCStreamer &Out,
                                         const MCSubtargetInfo &STI) {
-  return false;
+  if (Guard)
+    return false;
+  Guard = true;
+
+  doExpandInst(Inst, Out, STI);
+
+  Guard = false;
+  return true;
 }
diff --git a/llvm/test/MC/AArch64/LFI/branch.s b/llvm/test/MC/AArch64/LFI/branch.s
new file mode 100644
index 0000000000000..33e469e51aa20
--- /dev/null
+++ b/llvm/test/MC/AArch64/LFI/branch.s
@@ -0,0 +1,13 @@
+// RUN: llvm-mc -filetype asm -triple aarch64_lfi %s | FileCheck %s
+
+foo:
+  b foo
+// CHECK: b foo
+
+  br x0
+// CHECK:      add x28, x27, w0, uxtw
+// CHECK-NEXT: br x28
+
+  blr x0
+// CHECK:      add x28, x27, w0, uxtw
+// CHECK-NEXT: blr x28

From 5151a9a8ece16dc7c9a6e1c9492eda8075b6bc56 Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Tue, 19 Aug 2025 16:48:30 -0700
Subject: [PATCH 06/23] [LFI] [AArch64] Rewrites for stack manipulation and
 returns

---
 .../MCTargetDesc/AArch64MCLFIExpander.cpp     | 55 +++++++++++--------
 .../MCTargetDesc/AArch64MCLFIExpander.h       |  2 +
 llvm/test/MC/AArch64/LFI/branch.s             |  7 +++
 llvm/test/MC/AArch64/LFI/stack.s              | 37 +++++++++++++
 4 files changed, 79 insertions(+), 22 deletions(-)
 create mode 100644 llvm/test/MC/AArch64/LFI/stack.s

diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
index 91588a4b46df3..a79611dbe0bc8 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
@@ -30,6 +30,7 @@ using namespace llvm;
 
 static MCRegister LFIAddrReg = AArch64::X28;
 static MCRegister LFIBaseReg = AArch64::X27;
+static MCRegister LFIScratchReg = AArch64::X26;
 
 bool AArch64::AArch64MCLFIExpander::isValidScratchRegister(MCRegister Reg) const {
   return Reg != AArch64::SP;
@@ -55,9 +56,9 @@ static void emitBranch(unsigned int Opcode, MCRegister Target, MCStreamer &Out,
 }
 
 void AArch64::AArch64MCLFIExpander::expandIndirectBranch(const MCInst &Inst,
-                                                  MCStreamer &Out,
-                                                  const MCSubtargetInfo &STI,
-                                                  bool IsCall) {
+                                                         MCStreamer &Out,
+                                                         const MCSubtargetInfo &STI,
+                                                         bool IsCall) {
   (void) IsCall;
 
   assert(Inst.getOperand(0).isReg());
@@ -68,7 +69,7 @@ void AArch64::AArch64MCLFIExpander::expandIndirectBranch(const MCInst &Inst,
 }
 
 void AArch64::AArch64MCLFIExpander::expandCall(const MCInst &Inst, MCStreamer &Out,
-                                        const MCSubtargetInfo &STI) {
+                                               const MCSubtargetInfo &STI) {
   if (Inst.getOperand(0).isReg())
     expandIndirectBranch(Inst, Out, STI, true);
   else
@@ -76,34 +77,47 @@ void AArch64::AArch64MCLFIExpander::expandCall(const MCInst &Inst, MCStreamer &O
 }
 
 void AArch64::AArch64MCLFIExpander::expandReturn(const MCInst &Inst, MCStreamer &Out,
-                                          const MCSubtargetInfo &STI) {
+                                                 const MCSubtargetInfo &STI) {
+  assert(Inst.getOperand(0).isReg());
+  if (Inst.getOperand(0).getReg() != AArch64::LR)
+    expandIndirectBranch(Inst, Out, STI, false);
+  else
+    Out.emitInstruction(Inst, STI);
 }
 
-void AArch64::AArch64MCLFIExpander::expandControlFlow(const MCInst &Inst,
-                                               MCStreamer &Out,
-                                               const MCSubtargetInfo &STI) {
-
+bool AArch64::AArch64MCLFIExpander::mayModifyStack(const MCInst &Inst) {
+  return mayModifyRegister(Inst, AArch64::SP);
 }
 
-bool AArch64::AArch64MCLFIExpander::mayModifyStack(const MCInst &Inst) {
-  return false;
+bool AArch64::AArch64MCLFIExpander::mayModifyReserved(const MCInst &Inst) {
+  return mayModifyRegister(Inst, LFIAddrReg) || mayModifyRegister(Inst, LFIBaseReg);
 }
 
 void AArch64::AArch64MCLFIExpander::expandStackManipulation(
     const MCInst &Inst, MCStreamer &Out, const MCSubtargetInfo &STI) {
-}
-
-void AArch64::AArch64MCLFIExpander::expandPrefetch(const MCInst &Inst, MCStreamer &Out,
-                                            const MCSubtargetInfo &STI) {
+  if (mayLoad(Inst) || mayStore(Inst)) {
+    return Out.emitInstruction(Inst, STI);
+  }
+
+  MCInst ModInst;
+  MCRegister Scratch = LFIScratchReg;
+  assert(Inst.getOperand(0).isReg() && Inst.getOperand(0).getReg() == AArch64::SP);
+  ModInst.setOpcode(Inst.getOpcode());
+  ModInst.addOperand(MCOperand::createReg(Scratch));
+  for (unsigned I = 1, E = Inst.getNumOperands(); I != E; ++I) {
+    ModInst.addOperand(Inst.getOperand(I));
+  }
+  Out.emitInstruction(ModInst, STI);
+  emitAddMask(AArch64::SP, Scratch, Out, STI);
 }
 
 void AArch64::AArch64MCLFIExpander::expandLoadStore(const MCInst &Inst,
-                                             MCStreamer &Out,
-                                             const MCSubtargetInfo &STI) {
+                                                    MCStreamer &Out,
+                                                    const MCSubtargetInfo &STI) {
 }
 
 void AArch64::AArch64MCLFIExpander::doExpandInst(const MCInst &Inst, MCStreamer &Out,
-                                          const MCSubtargetInfo &STI) {
+                                                 const MCSubtargetInfo &STI) {
   if (isReturn(Inst))
     return expandReturn(Inst, Out, STI);
 
@@ -116,9 +130,6 @@ void AArch64::AArch64MCLFIExpander::doExpandInst(const MCInst &Inst, MCStreamer
   if (isBranch(Inst))
     return Out.emitInstruction(Inst, STI);
 
-  if (mayAffectControlFlow(Inst))
-    return expandControlFlow(Inst, Out, STI);
-
   if (mayModifyStack(Inst))
     return expandStackManipulation(Inst, Out, STI);
 
@@ -129,7 +140,7 @@ void AArch64::AArch64MCLFIExpander::doExpandInst(const MCInst &Inst, MCStreamer
 }
 
 bool AArch64::AArch64MCLFIExpander::expandInst(const MCInst &Inst, MCStreamer &Out,
-                                        const MCSubtargetInfo &STI) {
+                                               const MCSubtargetInfo &STI) {
   if (Guard)
     return false;
   Guard = true;
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.h b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.h
index 194fa36eb9c2b..00e547c9b3925 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.h
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.h
@@ -52,6 +52,8 @@ class AArch64MCLFIExpander : public MCLFIExpander {
 
   bool mayModifyStack(const MCInst &Inst);
 
+  bool mayModifyReserved(const MCInst &Inst);
+
   void expandControlFlow(const MCInst &Inst, MCStreamer &Out,
                          const MCSubtargetInfo &STI);
 
diff --git a/llvm/test/MC/AArch64/LFI/branch.s b/llvm/test/MC/AArch64/LFI/branch.s
index 33e469e51aa20..e2c126cf24db7 100644
--- a/llvm/test/MC/AArch64/LFI/branch.s
+++ b/llvm/test/MC/AArch64/LFI/branch.s
@@ -11,3 +11,10 @@ foo:
   blr x0
 // CHECK:      add x28, x27, w0, uxtw
 // CHECK-NEXT: blr x28
+
+  ret
+// CHECK: ret
+
+  ret x0
+// CHECK:      add x28, x27, w0, uxtw
+// CHECK-NEXT: ret x28
diff --git a/llvm/test/MC/AArch64/LFI/stack.s b/llvm/test/MC/AArch64/LFI/stack.s
new file mode 100644
index 0000000000000..349a85677cee5
--- /dev/null
+++ b/llvm/test/MC/AArch64/LFI/stack.s
@@ -0,0 +1,37 @@
+// RUN: llvm-mc -filetype asm -triple aarch64_lfi %s | FileCheck %s
+
+ldr x0, [sp, #16]!
+// CHECK: ldr x0, [sp, #16]!
+
+ldr x0, [sp], #16
+// CHECK: ldr x0, [sp], #16
+
+str x0, [sp, #16]!
+// CHECK: str x0, [sp, #16]!
+
+str x0, [sp], #16
+// CHECK: str x0, [sp], #16
+
+mov sp, x0
+// CHECK:      add x26, x0, #0
+// CHECK-NEXT: add sp, x27, w26, uxtw
+
+add sp, sp, #8
+// CHECK:      add x26, sp, #8
+// CHECK-NEXT: add sp, x27, w26, uxtw
+
+sub sp, sp, #8
+// CHECK:      sub x26, sp, #8
+// CHECK-NEXT: add sp, x27, w26, uxtw
+
+add sp, sp, x0
+// CHECK:      add x26, sp, x0
+// CHECK-NEXT: add sp, x27, w26, uxtw
+
+sub sp, sp, x0
+// CHECK:      sub x26, sp, x0
+// CHECK-NEXT: add sp, x27, w26, uxtw
+
+sub sp, sp, #1, lsl #12
+// CHECK:      sub x26, sp, #1, lsl #12
+// CHECK-NEXT: add sp, x27, w26, uxtw

From 003d56da02e2e311a844bb755da41b257465f402 Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Tue, 19 Aug 2025 23:04:10 -0700
Subject: [PATCH 07/23] [LFI] [AArch64] Rewrites for syscalls and TLS

---
 .../MCTargetDesc/AArch64MCLFIExpander.cpp     | 128 +++++++++++++++++-
 .../MCTargetDesc/AArch64MCLFIExpander.h       |  16 ++-
 llvm/test/MC/AArch64/LFI/sys.s                |  41 ++++++
 3 files changed, 182 insertions(+), 3 deletions(-)
 create mode 100644 llvm/test/MC/AArch64/LFI/sys.s

diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
index a79611dbe0bc8..95c787ecae606 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
@@ -93,9 +93,20 @@ bool AArch64::AArch64MCLFIExpander::mayModifyReserved(const MCInst &Inst) {
   return mayModifyRegister(Inst, LFIAddrReg) || mayModifyRegister(Inst, LFIBaseReg);
 }
 
-void AArch64::AArch64MCLFIExpander::expandStackManipulation(
+bool AArch64::AArch64MCLFIExpander::mayModifyLR(const MCInst &Inst) {
+  return mayModifyRegister(Inst, AArch64::LR);
+}
+
+// Rewrites for modifications of "special" registers: x28, x27, lr.
+void AArch64::AArch64MCLFIExpander::expandSpecialModification(
+    const MCInst &Inst, MCStreamer &Out, const MCSubtargetInfo &STI) {
+}
+
+void AArch64::AArch64MCLFIExpander::expandStackModification(
     const MCInst &Inst, MCStreamer &Out, const MCSubtargetInfo &STI) {
   if (mayLoad(Inst) || mayStore(Inst)) {
+    if (mayModifyReserved(Inst) || mayModifyLR(Inst))
+      return expandSpecialModification(Inst, Out, STI);
     return Out.emitInstruction(Inst, STI);
   }
 
@@ -116,8 +127,118 @@ void AArch64::AArch64MCLFIExpander::expandLoadStore(const MCInst &Inst,
                                                     const MCSubtargetInfo &STI) {
 }
 
+static void emit(unsigned int Op, MCRegister Rd, MCRegister Rs,
+    int64_t Imm, MCStreamer &Out, const MCSubtargetInfo &STI) {
+  MCInst Inst;
+  Inst.setOpcode(Op);
+  Inst.addOperand(MCOperand::createReg(Rd));
+  Inst.addOperand(MCOperand::createReg(Rs));
+  Inst.addOperand(MCOperand::createImm(Imm));
+  Out.emitInstruction(Inst, STI);
+}
+
+static void emit(unsigned int Op, MCRegister Rd, MCRegister Rt1,
+    MCRegister Rt2, int64_t Imm, MCStreamer &Out, const MCSubtargetInfo &STI) {
+  MCInst Inst;
+  Inst.setOpcode(Op);
+  Inst.addOperand(MCOperand::createReg(Rd));
+  Inst.addOperand(MCOperand::createReg(Rt1));
+  Inst.addOperand(MCOperand::createReg(Rt2));
+  Inst.addOperand(MCOperand::createImm(Imm));
+  Out.emitInstruction(Inst, STI);
+}
+
+static void emit(unsigned int Op, MCRegister Reg, MCStreamer &Out,
+    const MCSubtargetInfo &STI) {
+  MCInst Inst;
+  Inst.setOpcode(Op);
+  Inst.addOperand(MCOperand::createReg(Reg));
+  Out.emitInstruction(Inst, STI);
+}
+
+static void emitMov(MCRegister Dest, MCRegister Src, MCStreamer &Out, const MCSubtargetInfo &STI) {
+  emit(AArch64::ORRXrs, Dest, AArch64::XZR, Src, 0, Out, STI);
+}
+
+enum LFICallType {
+  LFISyscall,
+  LFITLSRead,
+  LFITLSWrite,
+};
+
+static void emitLFICall(LFICallType CallType, MCStreamer &Out, const MCSubtargetInfo &STI) {
+  emitMov(LFIScratchReg, AArch64::LR, Out, STI);
+  unsigned Offset;
+  switch (CallType) {
+    case LFISyscall: Offset = 0; break;
+    case LFITLSRead: Offset = 1; break;
+    case LFITLSWrite: Offset = 2; break;
+  }
+  emit(AArch64::LDRXui, AArch64::LR, LFIBaseReg, Offset, Out, STI);
+  emit(AArch64::BLR, AArch64::LR, Out, STI);
+  emitAddMask(AArch64::LR, LFIScratchReg, Out, STI);
+}
+
+void AArch64::AArch64MCLFIExpander::expandSyscall(const MCInst &Inst, MCStreamer &Out,
+                   const MCSubtargetInfo &STI) {
+  emitLFICall(LFISyscall, Out, STI);
+}
+
+static void emitSwap(MCRegister Reg1, MCRegister Reg2, MCStreamer &Out, const MCSubtargetInfo &STI) {
+  emit(AArch64::EORXrs, Reg1, Reg1, Reg2, 0, Out, STI);
+  emit(AArch64::EORXrs, Reg2, Reg1, Reg2, 0, Out, STI);
+  emit(AArch64::EORXrs, Reg1, Reg1, Reg2, 0, Out, STI);
+}
+
+void AArch64::AArch64MCLFIExpander::expandTLSRead(const MCInst &Inst, MCStreamer &Out,
+                   const MCSubtargetInfo &STI) {
+  MCRegister Reg = Inst.getOperand(0).getReg();
+  if (Reg == AArch64::X0) {
+    emitLFICall(LFITLSRead, Out, STI);
+  } else {
+    emitMov(Reg, AArch64::X0, Out, STI);
+    emitLFICall(LFITLSRead, Out, STI);
+    emitSwap(AArch64::X0, Reg, Out, STI);
+  }
+}
+
+void AArch64::AArch64MCLFIExpander::expandTLSWrite(const MCInst &Inst, MCStreamer &Out,
+                    const MCSubtargetInfo &STI) {
+  MCRegister Reg = Inst.getOperand(1).getReg();
+  if (Reg == AArch64::X0) {
+    emitLFICall(LFITLSWrite, Out, STI);
+  } else {
+    emitSwap(Reg, AArch64::X0, Out, STI);
+    emitLFICall(LFITLSWrite, Out, STI);
+    emitSwap(AArch64::X0, Reg, Out, STI);
+  }
+}
+
+static bool isSyscall(const MCInst &Inst) {
+  return Inst.getOpcode() == AArch64::SVC;
+}
+
+static bool isTLSRead(const MCInst &Inst) {
+  return Inst.getOpcode() == AArch64::MRS &&
+    Inst.getOperand(1).getReg() == AArch64SysReg::TPIDR_EL0;
+}
+
+static bool isTLSWrite(const MCInst &Inst) {
+  return Inst.getOpcode() == AArch64::MSR &&
+    Inst.getOperand(0).getReg() == AArch64SysReg::TPIDR_EL0;
+}
+
 void AArch64::AArch64MCLFIExpander::doExpandInst(const MCInst &Inst, MCStreamer &Out,
                                                  const MCSubtargetInfo &STI) {
+  if (isSyscall(Inst))
+    return expandSyscall(Inst, Out, STI);
+
+  if (isTLSRead(Inst))
+    return expandTLSRead(Inst, Out, STI);
+
+  if (isTLSWrite(Inst))
+    return expandTLSWrite(Inst, Out, STI);
+
   if (isReturn(Inst))
     return expandReturn(Inst, Out, STI);
 
@@ -131,7 +252,10 @@ void AArch64::AArch64MCLFIExpander::doExpandInst(const MCInst &Inst, MCStreamer
     return Out.emitInstruction(Inst, STI);
 
   if (mayModifyStack(Inst))
-    return expandStackManipulation(Inst, Out, STI);
+    return expandStackModification(Inst, Out, STI);
+
+  if (mayModifyReserved(Inst) || mayModifyLR(Inst))
+    return expandSpecialModification(Inst, Out, STI);
 
   if (mayLoad(Inst) || mayStore(Inst))
     return expandLoadStore(Inst, Out, STI);
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.h b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.h
index 00e547c9b3925..1d2869c6095ac 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.h
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.h
@@ -54,18 +54,32 @@ class AArch64MCLFIExpander : public MCLFIExpander {
 
   bool mayModifyReserved(const MCInst &Inst);
 
+  bool mayModifyLR(const MCInst &Inst);
+
   void expandControlFlow(const MCInst &Inst, MCStreamer &Out,
                          const MCSubtargetInfo &STI);
 
-  void expandStackManipulation(const MCInst &Inst, MCStreamer &Out,
+  void expandStackModification(const MCInst &Inst, MCStreamer &Out,
                                const MCSubtargetInfo &STI);
 
+  void expandSpecialModification(const MCInst &Inst, MCStreamer &Out,
+                                 const MCSubtargetInfo &STI);
+
   void expandPrefetch(const MCInst &Inst, MCStreamer &Out,
                       const MCSubtargetInfo &STI);
 
   void expandLoadStore(const MCInst &Inst, MCStreamer &Out,
                        const MCSubtargetInfo &STI);
 
+  void expandSyscall(const MCInst &Inst, MCStreamer &Out,
+                     const MCSubtargetInfo &STI);
+
+  void expandTLSRead(const MCInst &Inst, MCStreamer &Out,
+                     const MCSubtargetInfo &STI);
+
+  void expandTLSWrite(const MCInst &Inst, MCStreamer &Out,
+                      const MCSubtargetInfo &STI);
+
   void doExpandInst(const MCInst &Inst, MCStreamer &Out,
                     const MCSubtargetInfo &STI);
 };
diff --git a/llvm/test/MC/AArch64/LFI/sys.s b/llvm/test/MC/AArch64/LFI/sys.s
new file mode 100644
index 0000000000000..679f1512191e0
--- /dev/null
+++ b/llvm/test/MC/AArch64/LFI/sys.s
@@ -0,0 +1,41 @@
+// RUN: llvm-mc -filetype asm -triple aarch64_lfi %s | FileCheck %s
+
+svc #0
+// CHECK:      mov x26, x30
+// CHECK-NEXT: ldr x30, [x27]
+// CHECK-NEXT: blr x30
+// CHECK-NEXT: add x30, x27, w26, uxtw
+
+msr tpidr_el0, x0
+// CHECK:      mov x26, x30
+// CHECK-NEXT: ldr x30, [x27, #16]
+// CHECK-NEXT: blr x30
+// CHECK-NEXT: add x30, x27, w26, uxtw
+
+msr tpidr_el0, x1
+// CHECK:      eor x1, x1, x0
+// CHECK-NEXT: eor x0, x1, x0
+// CHECK-NEXT: eor x1, x1, x0
+// CHECK-NEXT: mov x26, x30
+// CHECK-NEXT: ldr x30, [x27, #16]
+// CHECK-NEXT: blr x30
+// CHECK-NEXT: add x30, x27, w26, uxtw
+// CHECK-NEXT: eor x0, x0, x1
+// CHECK-NEXT: eor x1, x0, x1
+// CHECK-NEXT: eor x0, x0, x1
+
+mrs x0, tpidr_el0
+// CHECK:      mov x26, x30
+// CHECK-NEXT: ldr x30, [x27, #8]
+// CHECK-NEXT: blr x30
+// CHECK-NEXT: add x30, x27, w26, uxtw
+
+mrs x1, tpidr_el0
+// CHECK:      mov x1, x0
+// CHECK-NEXT: mov x26, x30
+// CHECK-NEXT: ldr x30, [x27, #8]
+// CHECK-NEXT: blr x30
+// CHECK-NEXT: add x30, x27, w26, uxtw
+// CHECK-NEXT: eor x0, x0, x1
+// CHECK-NEXT: eor x1, x0, x1
+// CHECK-NEXT: eor x0, x0, x1

From a3c01617a1c04ad4129e6de71030a680cd2de760 Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Wed, 20 Aug 2025 11:56:01 -0700
Subject: [PATCH 08/23] [LFI] [AArch64] Use .scratch register if available

---
 .../MCTargetDesc/AArch64MCLFIExpander.cpp     | 101 +++++++++---------
 .../MCTargetDesc/AArch64MCLFIExpander.h       |  11 ++
 llvm/test/MC/AArch64/LFI/stack.s              |   6 ++
 llvm/test/MC/AArch64/LFI/sys.s                |   8 ++
 4 files changed, 76 insertions(+), 50 deletions(-)

diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
index 95c787ecae606..9ab22122de3fb 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
@@ -36,15 +36,54 @@ bool AArch64::AArch64MCLFIExpander::isValidScratchRegister(MCRegister Reg) const
   return Reg != AArch64::SP;
 }
 
+MCRegister AArch64::AArch64MCLFIExpander::getScratch() {
+  if (numScratchRegs() == 0) {
+    return LFIScratchReg;
+  }
+  return getScratchReg(0);
+}
+
+static void emit(unsigned int Op, MCRegister Rd, MCRegister Rs,
+    int64_t Imm, MCStreamer &Out, const MCSubtargetInfo &STI) {
+  MCInst Inst;
+  Inst.setOpcode(Op);
+  Inst.addOperand(MCOperand::createReg(Rd));
+  Inst.addOperand(MCOperand::createReg(Rs));
+  Inst.addOperand(MCOperand::createImm(Imm));
+  Out.emitInstruction(Inst, STI);
+}
+
+static void emit(unsigned int Op, MCRegister Rd, MCRegister Rt1,
+    MCRegister Rt2, int64_t Imm, MCStreamer &Out, const MCSubtargetInfo &STI) {
+  MCInst Inst;
+  Inst.setOpcode(Op);
+  Inst.addOperand(MCOperand::createReg(Rd));
+  Inst.addOperand(MCOperand::createReg(Rt1));
+  Inst.addOperand(MCOperand::createReg(Rt2));
+  Inst.addOperand(MCOperand::createImm(Imm));
+  Out.emitInstruction(Inst, STI);
+}
+
+static void emit(unsigned int Op, MCRegister Reg, MCStreamer &Out,
+    const MCSubtargetInfo &STI) {
+  MCInst Inst;
+  Inst.setOpcode(Op);
+  Inst.addOperand(MCOperand::createReg(Reg));
+  Out.emitInstruction(Inst, STI);
+}
+
+static void emitMov(MCRegister Dest, MCRegister Src, MCStreamer &Out, const MCSubtargetInfo &STI) {
+  emit(AArch64::ORRXrs, Dest, AArch64::XZR, Src, 0, Out, STI);
+}
+
 static void emitAddMask(MCRegister Dest, MCRegister Src, MCStreamer &Out,
                         const MCSubtargetInfo &STI) {
-  MCInst Add;
-  Add.setOpcode(AArch64::ADDXrx);
-  Add.addOperand(MCOperand::createReg(Dest));
-  Add.addOperand(MCOperand::createReg(LFIBaseReg));
-  Add.addOperand(MCOperand::createReg(getWRegFromXReg(Src)));
-  Add.addOperand(MCOperand::createImm(AArch64_AM::getArithExtendImm(AArch64_AM::UXTW, 0)));
-  Out.emitInstruction(Add, STI);;
+  emit(AArch64::ADDXrx,
+      Dest,
+      LFIBaseReg,
+      getWRegFromXReg(Src),
+      AArch64_AM::getArithExtendImm(AArch64_AM::UXTW, 0),
+      Out, STI);
 }
 
 static void emitBranch(unsigned int Opcode, MCRegister Target, MCStreamer &Out,
@@ -111,7 +150,7 @@ void AArch64::AArch64MCLFIExpander::expandStackModification(
   }
 
   MCInst ModInst;
-  MCRegister Scratch = LFIScratchReg;
+  MCRegister Scratch = getScratch();
   assert(Inst.getOperand(0).isReg() && Inst.getOperand(0).getReg() == AArch64::SP);
   ModInst.setOpcode(Inst.getOpcode());
   ModInst.addOperand(MCOperand::createReg(Scratch));
@@ -127,47 +166,9 @@ void AArch64::AArch64MCLFIExpander::expandLoadStore(const MCInst &Inst,
                                                     const MCSubtargetInfo &STI) {
 }
 
-static void emit(unsigned int Op, MCRegister Rd, MCRegister Rs,
-    int64_t Imm, MCStreamer &Out, const MCSubtargetInfo &STI) {
-  MCInst Inst;
-  Inst.setOpcode(Op);
-  Inst.addOperand(MCOperand::createReg(Rd));
-  Inst.addOperand(MCOperand::createReg(Rs));
-  Inst.addOperand(MCOperand::createImm(Imm));
-  Out.emitInstruction(Inst, STI);
-}
-
-static void emit(unsigned int Op, MCRegister Rd, MCRegister Rt1,
-    MCRegister Rt2, int64_t Imm, MCStreamer &Out, const MCSubtargetInfo &STI) {
-  MCInst Inst;
-  Inst.setOpcode(Op);
-  Inst.addOperand(MCOperand::createReg(Rd));
-  Inst.addOperand(MCOperand::createReg(Rt1));
-  Inst.addOperand(MCOperand::createReg(Rt2));
-  Inst.addOperand(MCOperand::createImm(Imm));
-  Out.emitInstruction(Inst, STI);
-}
-
-static void emit(unsigned int Op, MCRegister Reg, MCStreamer &Out,
-    const MCSubtargetInfo &STI) {
-  MCInst Inst;
-  Inst.setOpcode(Op);
-  Inst.addOperand(MCOperand::createReg(Reg));
-  Out.emitInstruction(Inst, STI);
-}
-
-static void emitMov(MCRegister Dest, MCRegister Src, MCStreamer &Out, const MCSubtargetInfo &STI) {
-  emit(AArch64::ORRXrs, Dest, AArch64::XZR, Src, 0, Out, STI);
-}
-
-enum LFICallType {
-  LFISyscall,
-  LFITLSRead,
-  LFITLSWrite,
-};
-
-static void emitLFICall(LFICallType CallType, MCStreamer &Out, const MCSubtargetInfo &STI) {
-  emitMov(LFIScratchReg, AArch64::LR, Out, STI);
+void AArch64::AArch64MCLFIExpander::emitLFICall(LFICallType CallType, MCStreamer &Out, const MCSubtargetInfo &STI) {
+  MCRegister Scratch = getScratch();
+  emitMov(Scratch, AArch64::LR, Out, STI);
   unsigned Offset;
   switch (CallType) {
     case LFISyscall: Offset = 0; break;
@@ -176,7 +177,7 @@ static void emitLFICall(LFICallType CallType, MCStreamer &Out, const MCSubtarget
   }
   emit(AArch64::LDRXui, AArch64::LR, LFIBaseReg, Offset, Out, STI);
   emit(AArch64::BLR, AArch64::LR, Out, STI);
-  emitAddMask(AArch64::LR, LFIScratchReg, Out, STI);
+  emitAddMask(AArch64::LR, Scratch, Out, STI);
 }
 
 void AArch64::AArch64MCLFIExpander::expandSyscall(const MCInst &Inst, MCStreamer &Out,
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.h b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.h
index 1d2869c6095ac..e5529f3a58ee8 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.h
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.h
@@ -41,6 +41,8 @@ class AArch64MCLFIExpander : public MCLFIExpander {
 private:
   bool Guard = false; // recursion guard
 
+  MCRegister getScratch();
+
   void expandIndirectBranch(const MCInst &Inst, MCStreamer &Out,
                             const MCSubtargetInfo &STI, bool isCall);
 
@@ -82,6 +84,15 @@ class AArch64MCLFIExpander : public MCLFIExpander {
 
   void doExpandInst(const MCInst &Inst, MCStreamer &Out,
                     const MCSubtargetInfo &STI);
+
+  enum LFICallType {
+    LFISyscall,
+    LFITLSRead,
+    LFITLSWrite,
+  };
+
+  void emitLFICall(LFICallType CallType, MCStreamer &Out,
+                   const MCSubtargetInfo &STI);
 };
 } // namespace AArch64
 } // namespace llvm
diff --git a/llvm/test/MC/AArch64/LFI/stack.s b/llvm/test/MC/AArch64/LFI/stack.s
index 349a85677cee5..bae875071dc72 100644
--- a/llvm/test/MC/AArch64/LFI/stack.s
+++ b/llvm/test/MC/AArch64/LFI/stack.s
@@ -35,3 +35,9 @@ sub sp, sp, x0
 sub sp, sp, #1, lsl #12
 // CHECK:      sub x26, sp, #1, lsl #12
 // CHECK-NEXT: add sp, x27, w26, uxtw
+
+.scratch x10
+add sp, sp, x0
+// CHECK:      add x10, sp, x0
+// CHECK-NEXT: add sp, x27, w10, uxtw
+.scratch_clear
diff --git a/llvm/test/MC/AArch64/LFI/sys.s b/llvm/test/MC/AArch64/LFI/sys.s
index 679f1512191e0..6f54fcad754d6 100644
--- a/llvm/test/MC/AArch64/LFI/sys.s
+++ b/llvm/test/MC/AArch64/LFI/sys.s
@@ -6,6 +6,14 @@ svc #0
 // CHECK-NEXT: blr x30
 // CHECK-NEXT: add x30, x27, w26, uxtw
 
+.scratch x10
+svc #0
+// CHECK:      mov x10, x30
+// CHECK-NEXT: ldr x30, [x27]
+// CHECK-NEXT: blr x30
+// CHECK-NEXT: add x30, x27, w10, uxtw
+.scratch_clear
+
 msr tpidr_el0, x0
 // CHECK:      mov x26, x30
 // CHECK-NEXT: ldr x30, [x27, #16]

From 006dfb698da71ffcb9007a1483168799ca71bbd1 Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Wed, 20 Aug 2025 13:24:36 -0700
Subject: [PATCH 09/23] [LFI] [AArch64] Rewrites for LR modifications

---
 .../MCTargetDesc/AArch64MCLFIExpander.cpp     | 38 ++++++++++++++++---
 .../MCTargetDesc/AArch64MCLFIExpander.h       |  2 +-
 llvm/test/MC/AArch64/LFI/reserved.s           | 12 ++++++
 llvm/test/MC/AArch64/LFI/return.s             | 19 ++++++++++
 4 files changed, 64 insertions(+), 7 deletions(-)
 create mode 100644 llvm/test/MC/AArch64/LFI/reserved.s
 create mode 100644 llvm/test/MC/AArch64/LFI/return.s

diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
index 9ab22122de3fb..c81d1fa340bc9 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
@@ -136,16 +136,32 @@ bool AArch64::AArch64MCLFIExpander::mayModifyLR(const MCInst &Inst) {
   return mayModifyRegister(Inst, AArch64::LR);
 }
 
-// Rewrites for modifications of "special" registers: x28, x27, lr.
-void AArch64::AArch64MCLFIExpander::expandSpecialModification(
+void AArch64::AArch64MCLFIExpander::expandLRModification(
     const MCInst &Inst, MCStreamer &Out, const MCSubtargetInfo &STI) {
+  MCRegister Scratch = getScratch();
+  MCInst New;
+  New.setLoc(Inst.getLoc());
+  New.setOpcode(Inst.getOpcode());
+  for (unsigned I = 0; I < Inst.getNumOperands(); ++I) {
+    const MCOperand &Op = Inst.getOperand(I);
+    if (Op.isReg() && Op.getReg() == AArch64::LR) {
+      New.addOperand(MCOperand::createReg(Scratch));
+    } else {
+      New.addOperand(Op);
+    }
+  }
+  if (mayLoad(New) || mayStore(New))
+    expandLoadStore(New, Out, STI);
+  else
+    Out.emitInstruction(New, STI);
+  emitAddMask(AArch64::LR, Scratch, Out, STI);
 }
 
 void AArch64::AArch64MCLFIExpander::expandStackModification(
     const MCInst &Inst, MCStreamer &Out, const MCSubtargetInfo &STI) {
   if (mayLoad(Inst) || mayStore(Inst)) {
-    if (mayModifyReserved(Inst) || mayModifyLR(Inst))
-      return expandSpecialModification(Inst, Out, STI);
+    if (mayModifyLR(Inst))
+      return expandLRModification(Inst, Out, STI);
     return Out.emitInstruction(Inst, STI);
   }
 
@@ -153,6 +169,7 @@ void AArch64::AArch64MCLFIExpander::expandStackModification(
   MCRegister Scratch = getScratch();
   assert(Inst.getOperand(0).isReg() && Inst.getOperand(0).getReg() == AArch64::SP);
   ModInst.setOpcode(Inst.getOpcode());
+  ModInst.setLoc(Inst.getLoc());
   ModInst.addOperand(MCOperand::createReg(Scratch));
   for (unsigned I = 1, E = Inst.getNumOperands(); I != E; ++I) {
     ModInst.addOperand(Inst.getOperand(I));
@@ -164,6 +181,9 @@ void AArch64::AArch64MCLFIExpander::expandStackModification(
 void AArch64::AArch64MCLFIExpander::expandLoadStore(const MCInst &Inst,
                                                     MCStreamer &Out,
                                                     const MCSubtargetInfo &STI) {
+  Out.getContext().reportWarning(
+      Inst.getLoc(), "TODO: expandLoadStore");
+  Out.emitInstruction(Inst, STI);
 }
 
 void AArch64::AArch64MCLFIExpander::emitLFICall(LFICallType CallType, MCStreamer &Out, const MCSubtargetInfo &STI) {
@@ -252,11 +272,17 @@ void AArch64::AArch64MCLFIExpander::doExpandInst(const MCInst &Inst, MCStreamer
   if (isBranch(Inst))
     return Out.emitInstruction(Inst, STI);
 
+  // Bail out with an error. In the future, we could consider automatically
+  // rewriting uses of reserved LFI registers.
+  if (mayModifyReserved(Inst))
+    return Out.getContext().reportError(
+        Inst.getLoc(), "illegal modification of reserved LFI register");
+
   if (mayModifyStack(Inst))
     return expandStackModification(Inst, Out, STI);
 
-  if (mayModifyReserved(Inst) || mayModifyLR(Inst))
-    return expandSpecialModification(Inst, Out, STI);
+  if (mayModifyLR(Inst))
+    return expandLRModification(Inst, Out, STI);
 
   if (mayLoad(Inst) || mayStore(Inst))
     return expandLoadStore(Inst, Out, STI);
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.h b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.h
index e5529f3a58ee8..0ca366566df57 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.h
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.h
@@ -64,7 +64,7 @@ class AArch64MCLFIExpander : public MCLFIExpander {
   void expandStackModification(const MCInst &Inst, MCStreamer &Out,
                                const MCSubtargetInfo &STI);
 
-  void expandSpecialModification(const MCInst &Inst, MCStreamer &Out,
+  void expandLRModification(const MCInst &Inst, MCStreamer &Out,
                                  const MCSubtargetInfo &STI);
 
   void expandPrefetch(const MCInst &Inst, MCStreamer &Out,
diff --git a/llvm/test/MC/AArch64/LFI/reserved.s b/llvm/test/MC/AArch64/LFI/reserved.s
new file mode 100644
index 0000000000000..7b9ffb83474c0
--- /dev/null
+++ b/llvm/test/MC/AArch64/LFI/reserved.s
@@ -0,0 +1,12 @@
+// RUN: not llvm-mc -filetype asm -triple aarch64_lfi %s 2> %t
+// RUN: FileCheck --check-prefix=CHECK-ERROR %s <%t
+
+mov x28, x0
+// CHECK-ERROR: error: illegal modification of reserved LFI register
+// CHECK-ERROR:        mov x28, x0
+// CHECK-ERROR:        ^
+
+ldr x27, [sp]
+// CHECK-ERROR: error: illegal modification of reserved LFI register
+// CHECK-ERROR:        ldr x27, [sp]
+// CHECK-ERROR:        ^
diff --git a/llvm/test/MC/AArch64/LFI/return.s b/llvm/test/MC/AArch64/LFI/return.s
new file mode 100644
index 0000000000000..1295626134969
--- /dev/null
+++ b/llvm/test/MC/AArch64/LFI/return.s
@@ -0,0 +1,19 @@
+// RUN: llvm-mc -filetype asm -triple aarch64_lfi %s | FileCheck %s
+
+mov x30, x0
+// CHECK:      mov x26, x0
+// CHECK-NEXT: add x30, x27, w26, uxtw
+
+ldr x30, [sp]
+// CHECK:      ldr x26, [sp]
+// CHECK-NEXT: add x30, x27, w26, uxtw
+
+ldp x29, x30, [sp]
+// CHECK:      ldp x29, x26, [sp]
+// CHECK-NEXT: add x30, x27, w26, uxtw
+
+.scratch x10
+ldr x30, [sp]
+// CHECK:      ldr x10, [sp]
+// CHECK-NEXT: add x30, x27, w10, uxtw
+.scratch_clear

From 5e4cc704cf0b8fc873bd8e1ad8972e9e5c6969d8 Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Wed, 20 Aug 2025 15:52:52 -0700
Subject: [PATCH 10/23] [LFI] [AArch64] Rewrites for loads/stores

Doesn't handle basic pre/post-indexed loads/stores yet.
---
 .../MCTargetDesc/AArch64MCLFIExpander.cpp     | 458 +++++++++-
 .../MCTargetDesc/AArch64MCLFIExpander.h       |   7 +
 .../Target/AArch64/Utils/AArch64BaseInfo.h    | 811 ++++++++++++++++++
 llvm/test/MC/AArch64/LFI/mem.s                | 106 +++
 4 files changed, 1379 insertions(+), 3 deletions(-)
 create mode 100644 llvm/test/MC/AArch64/LFI/mem.s

diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
index c81d1fa340bc9..dc8ad0039ebad 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
@@ -64,6 +64,31 @@ static void emit(unsigned int Op, MCRegister Rd, MCRegister Rt1,
   Out.emitInstruction(Inst, STI);
 }
 
+static void emit(unsigned int Op, MCRegister Rd, MCRegister Rt1,
+    MCRegister Rt2, int64_t Imm1, int64_t Imm2,
+    MCStreamer &Out, const MCSubtargetInfo &STI) {
+  MCInst Inst;
+  Inst.setOpcode(Op);
+  Inst.addOperand(MCOperand::createReg(Rd));
+  Inst.addOperand(MCOperand::createReg(Rt1));
+  Inst.addOperand(MCOperand::createReg(Rt2));
+  Inst.addOperand(MCOperand::createImm(Imm1));
+  Inst.addOperand(MCOperand::createImm(Imm2));
+  Out.emitInstruction(Inst, STI);
+}
+
+static void emit(unsigned int Op, MCRegister Rd, MCRegister Rt,
+    int64_t Imm1, int64_t Imm2,
+    MCStreamer &Out, const MCSubtargetInfo &STI) {
+  MCInst Inst;
+  Inst.setOpcode(Op);
+  Inst.addOperand(MCOperand::createReg(Rd));
+  Inst.addOperand(MCOperand::createReg(Rt));
+  Inst.addOperand(MCOperand::createImm(Imm1));
+  Inst.addOperand(MCOperand::createImm(Imm2));
+  Out.emitInstruction(Inst, STI);
+}
+
 static void emit(unsigned int Op, MCRegister Reg, MCStreamer &Out,
     const MCSubtargetInfo &STI) {
   MCInst Inst;
@@ -76,6 +101,7 @@ static void emitMov(MCRegister Dest, MCRegister Src, MCStreamer &Out, const MCSu
   emit(AArch64::ORRXrs, Dest, AArch64::XZR, Src, 0, Out, STI);
 }
 
+// Emit 'add Dest, LFIBaseReg, W(Src), uxtw'
 static void emitAddMask(MCRegister Dest, MCRegister Src, MCStreamer &Out,
                         const MCSubtargetInfo &STI) {
   emit(AArch64::ADDXrx,
@@ -86,6 +112,11 @@ static void emitAddMask(MCRegister Dest, MCRegister Src, MCStreamer &Out,
       Out, STI);
 }
 
+// Emit 'Op(ld/st) Dest, [LFIBaseReg, W(Target), uxtw]'
+static void emitMemMask(unsigned Op, MCRegister Dest, MCRegister Target, MCStreamer &Out, const MCSubtargetInfo &STI) {
+  emit(Op, Dest, LFIBaseReg, getWRegFromXReg(Target), 0, 0, Out, STI);
+}
+
 static void emitBranch(unsigned int Opcode, MCRegister Target, MCStreamer &Out,
                        const MCSubtargetInfo &STI) {
   MCInst Branch;
@@ -178,12 +209,124 @@ void AArch64::AArch64MCLFIExpander::expandStackModification(
   emitAddMask(AArch64::SP, Scratch, Out, STI);
 }
 
+static bool canConvertToRoW(unsigned Op);
+static unsigned convertRoXToRoW(unsigned Op, unsigned &Shift);
+static unsigned convertRoWToRoW(unsigned Op, unsigned &Shift);
+static unsigned convertUiToRoW(unsigned Op);
+static unsigned convertPreToRoW(unsigned Op);
+static unsigned convertPostToRoW(unsigned Op);
+
+static void emitSafeLoadStore(const MCInst &Inst, unsigned N, MCStreamer &Out, const MCSubtargetInfo &STI) {
+  MCInst LoadStore;
+  LoadStore.setOpcode(Inst.getOpcode());
+  for (unsigned I = 0; I < N; ++I)
+    LoadStore.addOperand(Inst.getOperand(I));
+  LoadStore.addOperand(MCOperand::createReg(LFIAddrReg));
+  for (unsigned I = N + 1; I < Inst.getNumOperands(); ++I)
+    LoadStore.addOperand(Inst.getOperand(I));
+  Out.emitInstruction(LoadStore, STI);
+}
+
+void AArch64::AArch64MCLFIExpander::expandLoadStoreBasic(const MCInst &Inst, MemInstInfo &MII,
+    MCStreamer &Out, const MCSubtargetInfo &STI) {
+  emitAddMask(LFIAddrReg, Inst.getOperand(MII.BaseRegIdx).getReg(), Out, STI);
+
+  if (MII.IsPrePost) {
+    assert(OffsetIdx != -1 && "Pre/Post must have valid OffsetIdx");
+    return Out.getContext().reportWarning(
+        Inst.getLoc(), "TODO: pre/post index");
+  }
+
+  return emitSafeLoadStore(Inst, MII.BaseRegIdx, Out, STI);
+}
+
+void AArch64::AArch64MCLFIExpander::expandLoadStoreRoW(const MCInst &Inst, MemInstInfo &MII,
+    MCStreamer &Out, const MCSubtargetInfo &STI) {
+  unsigned MemOp;
+  unsigned Op = Inst.getOpcode();
+  if ((MemOp = convertUiToRoW(Op)) != AArch64::INSTRUCTION_LIST_END) {
+    auto OffsetMCO = Inst.getOperand(2);
+    if (OffsetMCO.isImm() && OffsetMCO.getImm() == 0)
+      return emitMemMask(MemOp, Inst.getOperand(0).getReg(), Inst.getOperand(1).getReg(), Out, STI);
+    return expandLoadStoreBasic(Inst, MII, Out, STI);
+  }
+
+  if ((MemOp = convertPreToRoW(Op)) != AArch64::INSTRUCTION_LIST_END) {
+    MCRegister Reg = Inst.getOperand(2).getReg();
+    int64_t Imm = Inst.getOperand(3).getImm();
+    if (Imm >= 0)
+      emit(AArch64::ADDXri, Reg, Reg, Imm, 0, Out, STI);
+    else
+      emit(AArch64::SUBXri, Reg, Reg, -Imm, 0, Out, STI);
+    return emitMemMask(MemOp, Inst.getOperand(1).getReg(), Reg, Out, STI);
+  }
+
+  if ((MemOp = convertPostToRoW(Op)) != AArch64::INSTRUCTION_LIST_END) {
+    MCRegister Reg = Inst.getOperand(2).getReg();
+    emitMemMask(MemOp, Inst.getOperand(1).getReg(), Reg, Out, STI);
+    int64_t Imm = Inst.getOperand(3).getImm();
+    if (Imm >= 0)
+      emit(AArch64::ADDXri, Reg, Reg, Imm, 0, Out, STI);
+    else
+      emit(AArch64::SUBXri, Reg, Reg, -Imm, 0, Out, STI);
+    return;
+  }
+
+  unsigned Shift;
+  if ((MemOp = convertRoXToRoW(Op, Shift)) != AArch64::INSTRUCTION_LIST_END) {
+    MCRegister Reg1 = Inst.getOperand(1).getReg();
+    MCRegister Reg2 = Inst.getOperand(2).getReg();
+    int64_t Extend = Inst.getOperand(3).getImm();
+    int64_t IsShift = Inst.getOperand(4).getImm();
+    MCRegister Scratch = getScratch();
+    if (!IsShift)
+      Shift = 0;
+    if (Extend)
+      emit(AArch64::ADDXrx, Scratch, Reg1, Reg2, AArch64_AM::getArithExtendImm(AArch64_AM::SXTX, Shift), Out, STI);
+    else
+      emit(AArch64::ADDXrs, Scratch, Reg1, Reg2, AArch64_AM::getShifterImm(AArch64_AM::LSL, Shift), Out, STI);
+    return emitMemMask(MemOp, Inst.getOperand(0).getReg(), Scratch, Out, STI);
+  }
+
+  if ((MemOp = convertRoWToRoW(Op, Shift)) != AArch64::INSTRUCTION_LIST_END) {
+    MCRegister Reg1 = Inst.getOperand(1).getReg();
+    MCRegister Reg2 = Inst.getOperand(2).getReg();
+    int64_t S = Inst.getOperand(3).getImm();
+    int64_t IsShift = Inst.getOperand(4).getImm();
+    MCRegister Scratch = getScratch();
+    if (!IsShift)
+      Shift = 0;
+    if (S)
+      emit(AArch64::ADDXrx, Scratch, Reg1, Reg2, AArch64_AM::getArithExtendImm(AArch64_AM::SXTW, Shift), Out, STI);
+    else
+      emit(AArch64::ADDXrx, Scratch, Reg1, Reg2, AArch64_AM::getArithExtendImm(AArch64_AM::UXTW, Shift), Out, STI);
+    return emitMemMask(MemOp, Inst.getOperand(0).getReg(), Scratch, Out, STI);
+  }
+}
+
 void AArch64::AArch64MCLFIExpander::expandLoadStore(const MCInst &Inst,
                                                     MCStreamer &Out,
                                                     const MCSubtargetInfo &STI) {
-  Out.getContext().reportWarning(
-      Inst.getLoc(), "TODO: expandLoadStore");
-  Out.emitInstruction(Inst, STI);
+  auto MII = getLoadInfo(Inst);
+  if (!MII.has_value()) {
+    MII = getStoreInfo(Inst);
+    if (!MII.has_value())
+      return Out.getContext().reportError(
+          Inst.getLoc(), "this load/store is not supported by LFI");
+  }
+
+  // Stack accesses without a register offset don't need rewriting.
+  if (Inst.getOperand(MII->BaseRegIdx).getReg() == AArch64::SP) {
+    if (MII->BaseRegIdx == (int) Inst.getNumOperands() - 1 ||
+        !Inst.getOperand(MII->BaseRegIdx + 1).isReg())
+      return Out.emitInstruction(Inst, STI);
+  }
+
+  // Try to convert to RoW if we can, otherwise use fallback.
+  if (canConvertToRoW(Inst.getOpcode()))
+    expandLoadStoreRoW(Inst, MII.value(), Out, STI);
+  else
+    expandLoadStoreBasic(Inst, MII.value(), Out, STI);
 }
 
 void AArch64::AArch64MCLFIExpander::emitLFICall(LFICallType CallType, MCStreamer &Out, const MCSubtargetInfo &STI) {
@@ -301,3 +444,312 @@ bool AArch64::AArch64MCLFIExpander::expandInst(const MCInst &Inst, MCStreamer &O
   Guard = false;
   return true;
 }
+
+static unsigned convertRoXToRoW(unsigned Op, unsigned &Shift) {
+  Shift = 0;
+  switch (Op) {
+  case AArch64::LDRBBroX:
+    return AArch64::LDRBBroW;
+  case AArch64::LDRBroX:
+    return AArch64::LDRBroW;
+  case AArch64::LDRDroX:
+    Shift = 3;
+    return AArch64::LDRDroW;
+  case AArch64::LDRHHroX:
+    Shift = 1;
+    return AArch64::LDRHHroW;
+  case AArch64::LDRHroX:
+    Shift = 1;
+    return AArch64::LDRHroW;
+  case AArch64::LDRQroX:
+    Shift = 4;
+    return AArch64::LDRQroW;
+  case AArch64::LDRSBWroX:
+    Shift = 1;
+    return AArch64::LDRSBWroW;
+  case AArch64::LDRSBXroX:
+    Shift = 1;
+    return AArch64::LDRSBXroW;
+  case AArch64::LDRSHWroX:
+    Shift = 1;
+    return AArch64::LDRSHWroW;
+  case AArch64::LDRSHXroX:
+    Shift = 1;
+    return AArch64::LDRSHXroW;
+  case AArch64::LDRSWroX:
+    Shift = 2;
+    return AArch64::LDRSWroW;
+  case AArch64::LDRSroX:
+    Shift = 2;
+    return AArch64::LDRSroW;
+  case AArch64::LDRWroX:
+    Shift = 2;
+    return AArch64::LDRWroW;
+  case AArch64::LDRXroX:
+    Shift = 3;
+    return AArch64::LDRXroW;
+  case AArch64::STRBBroX:
+    return AArch64::STRBBroW;
+  case AArch64::STRBroX:
+    return AArch64::STRBroW;
+  case AArch64::STRDroX:
+    Shift = 3;
+    return AArch64::STRDroW;
+  case AArch64::STRHHroX:
+    Shift = 1;
+    return AArch64::STRHHroW;
+  case AArch64::STRHroX:
+    Shift = 1;
+    return AArch64::STRHroW;
+  case AArch64::STRQroX:
+    Shift = 4;
+    return AArch64::STRQroW;
+  case AArch64::STRSroX:
+    Shift = 2;
+    return AArch64::STRSroW;
+  case AArch64::STRWroX:
+    Shift = 2;
+    return AArch64::STRWroW;
+  case AArch64::STRXroX:
+    Shift = 3;
+    return AArch64::STRXroW;
+  }
+  return AArch64::INSTRUCTION_LIST_END;
+}
+
+static unsigned convertRoWToRoW(unsigned Op, unsigned &Shift) {
+  Shift = 0;
+  switch (Op) {
+  case AArch64::LDRBBroW:
+    return AArch64::LDRBBroW;
+  case AArch64::LDRBroW:
+    return AArch64::LDRBroW;
+  case AArch64::LDRDroW:
+    Shift = 3;
+    return AArch64::LDRDroW;
+  case AArch64::LDRHHroW:
+    Shift = 1;
+    return AArch64::LDRHHroW;
+  case AArch64::LDRHroW:
+    Shift = 1;
+    return AArch64::LDRHroW;
+  case AArch64::LDRQroW:
+    Shift = 4;
+    return AArch64::LDRQroW;
+  case AArch64::LDRSBWroW:
+    Shift = 1;
+    return AArch64::LDRSBWroW;
+  case AArch64::LDRSBXroW:
+    Shift = 1;
+    return AArch64::LDRSBXroW;
+  case AArch64::LDRSHWroW:
+    Shift = 1;
+    return AArch64::LDRSHWroW;
+  case AArch64::LDRSHXroW:
+    Shift = 1;
+    return AArch64::LDRSHXroW;
+  case AArch64::LDRSWroW:
+    Shift = 2;
+    return AArch64::LDRSWroW;
+  case AArch64::LDRSroW:
+    Shift = 2;
+    return AArch64::LDRSroW;
+  case AArch64::LDRWroW:
+    Shift = 2;
+    return AArch64::LDRWroW;
+  case AArch64::LDRXroW:
+    Shift = 3;
+    return AArch64::LDRXroW;
+  case AArch64::STRBBroW:
+    return AArch64::STRBBroW;
+  case AArch64::STRBroW:
+    return AArch64::STRBroW;
+  case AArch64::STRDroW:
+    Shift = 3;
+    return AArch64::STRDroW;
+  case AArch64::STRHHroW:
+    Shift = 1;
+    return AArch64::STRHHroW;
+  case AArch64::STRHroW:
+    Shift = 1;
+    return AArch64::STRHroW;
+  case AArch64::STRQroW:
+    Shift = 4;
+    return AArch64::STRQroW;
+  case AArch64::STRSroW:
+    Shift = 2;
+    return AArch64::STRSroW;
+  case AArch64::STRWroW:
+    Shift = 2;
+    return AArch64::STRWroW;
+  case AArch64::STRXroW:
+    Shift = 3;
+    return AArch64::STRXroW;
+  }
+  return AArch64::INSTRUCTION_LIST_END;
+}
+
+static unsigned convertUiToRoW(unsigned Op) {
+  switch (Op) {
+  case AArch64::LDRBBui:
+    return AArch64::LDRBBroW;
+  case AArch64::LDRBui:
+    return AArch64::LDRBroW;
+  case AArch64::LDRDui:
+    return AArch64::LDRDroW;
+  case AArch64::LDRHHui:
+    return AArch64::LDRHHroW;
+  case AArch64::LDRHui:
+    return AArch64::LDRHroW;
+  case AArch64::LDRQui:
+    return AArch64::LDRQroW;
+  case AArch64::LDRSBWui:
+    return AArch64::LDRSBWroW;
+  case AArch64::LDRSBXui:
+    return AArch64::LDRSBXroW;
+  case AArch64::LDRSHWui:
+    return AArch64::LDRSHWroW;
+  case AArch64::LDRSHXui:
+    return AArch64::LDRSHXroW;
+  case AArch64::LDRSWui:
+    return AArch64::LDRSWroW;
+  case AArch64::LDRSui:
+    return AArch64::LDRSroW;
+  case AArch64::LDRWui:
+    return AArch64::LDRWroW;
+  case AArch64::LDRXui:
+    return AArch64::LDRXroW;
+  case AArch64::STRBBui:
+    return AArch64::STRBBroW;
+  case AArch64::STRBui:
+    return AArch64::STRBroW;
+  case AArch64::STRDui:
+    return AArch64::STRDroW;
+  case AArch64::STRHHui:
+    return AArch64::STRHHroW;
+  case AArch64::STRHui:
+    return AArch64::STRHroW;
+  case AArch64::STRQui:
+    return AArch64::STRQroW;
+  case AArch64::STRSui:
+    return AArch64::STRSroW;
+  case AArch64::STRWui:
+    return AArch64::STRWroW;
+  case AArch64::STRXui:
+    return AArch64::STRXroW;
+  }
+  return AArch64::INSTRUCTION_LIST_END;
+}
+
+static unsigned convertPreToRoW(unsigned Op) {
+  switch (Op) {
+  case AArch64::LDRBBpre:
+    return AArch64::LDRBBroW;
+  case AArch64::LDRBpre:
+    return AArch64::LDRBroW;
+  case AArch64::LDRDpre:
+    return AArch64::LDRDroW;
+  case AArch64::LDRHHpre:
+    return AArch64::LDRHHroW;
+  case AArch64::LDRHpre:
+    return AArch64::LDRHroW;
+  case AArch64::LDRQpre:
+    return AArch64::LDRQroW;
+  case AArch64::LDRSBWpre:
+    return AArch64::LDRSBWroW;
+  case AArch64::LDRSBXpre:
+    return AArch64::LDRSBXroW;
+  case AArch64::LDRSHWpre:
+    return AArch64::LDRSHWroW;
+  case AArch64::LDRSHXpre:
+    return AArch64::LDRSHXroW;
+  case AArch64::LDRSWpre:
+    return AArch64::LDRSWroW;
+  case AArch64::LDRSpre:
+    return AArch64::LDRSroW;
+  case AArch64::LDRWpre:
+    return AArch64::LDRWroW;
+  case AArch64::LDRXpre:
+    return AArch64::LDRXroW;
+  case AArch64::STRBBpre:
+    return AArch64::STRBBroW;
+  case AArch64::STRBpre:
+    return AArch64::STRBroW;
+  case AArch64::STRDpre:
+    return AArch64::STRDroW;
+  case AArch64::STRHHpre:
+    return AArch64::STRHHroW;
+  case AArch64::STRHpre:
+    return AArch64::STRHroW;
+  case AArch64::STRQpre:
+    return AArch64::STRQroW;
+  case AArch64::STRSpre:
+    return AArch64::STRSroW;
+  case AArch64::STRWpre:
+    return AArch64::STRWroW;
+  case AArch64::STRXpre:
+    return AArch64::STRXroW;
+  }
+  return AArch64::INSTRUCTION_LIST_END;
+}
+
+static unsigned convertPostToRoW(unsigned Op) {
+  switch (Op) {
+  case AArch64::LDRBBpost:
+    return AArch64::LDRBBroW;
+  case AArch64::LDRBpost:
+    return AArch64::LDRBroW;
+  case AArch64::LDRDpost:
+    return AArch64::LDRDroW;
+  case AArch64::LDRHHpost:
+    return AArch64::LDRHHroW;
+  case AArch64::LDRHpost:
+    return AArch64::LDRHroW;
+  case AArch64::LDRQpost:
+    return AArch64::LDRQroW;
+  case AArch64::LDRSBWpost:
+    return AArch64::LDRSBWroW;
+  case AArch64::LDRSBXpost:
+    return AArch64::LDRSBXroW;
+  case AArch64::LDRSHWpost:
+    return AArch64::LDRSHWroW;
+  case AArch64::LDRSHXpost:
+    return AArch64::LDRSHXroW;
+  case AArch64::LDRSWpost:
+    return AArch64::LDRSWroW;
+  case AArch64::LDRSpost:
+    return AArch64::LDRSroW;
+  case AArch64::LDRWpost:
+    return AArch64::LDRWroW;
+  case AArch64::LDRXpost:
+    return AArch64::LDRXroW;
+  case AArch64::STRBBpost:
+    return AArch64::STRBBroW;
+  case AArch64::STRBpost:
+    return AArch64::STRBroW;
+  case AArch64::STRDpost:
+    return AArch64::STRDroW;
+  case AArch64::STRHHpost:
+    return AArch64::STRHHroW;
+  case AArch64::STRHpost:
+    return AArch64::STRHroW;
+  case AArch64::STRQpost:
+    return AArch64::STRQroW;
+  case AArch64::STRSpost:
+    return AArch64::STRSroW;
+  case AArch64::STRWpost:
+    return AArch64::STRWroW;
+  case AArch64::STRXpost:
+    return AArch64::STRXroW;
+  }
+  return AArch64::INSTRUCTION_LIST_END;
+}
+
+static bool canConvertToRoW(unsigned Op) {
+  unsigned Shift;
+  return convertUiToRoW(Op) != AArch64::INSTRUCTION_LIST_END ||
+    convertPreToRoW(Op) != AArch64::INSTRUCTION_LIST_END ||
+    convertPostToRoW(Op) != AArch64::INSTRUCTION_LIST_END ||
+    convertRoXToRoW(Op, Shift) != AArch64::INSTRUCTION_LIST_END ||
+    convertRoWToRoW(Op, Shift) != AArch64::INSTRUCTION_LIST_END;
+}
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.h b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.h
index 0ca366566df57..2bfb66955f47c 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.h
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.h
@@ -15,6 +15,7 @@
 #ifndef LLVM_MC_AARCH64MCLFIEXPANDER_H
 #define LLVM_MC_AARCH64MCLFIEXPANDER_H
 
+#include "Utils/AArch64BaseInfo.h"
 #include "llvm/MC/MCInstrInfo.h"
 #include "llvm/MC/MCLFIExpander.h"
 #include "llvm/MC/MCRegisterInfo.h"
@@ -73,6 +74,12 @@ class AArch64MCLFIExpander : public MCLFIExpander {
   void expandLoadStore(const MCInst &Inst, MCStreamer &Out,
                        const MCSubtargetInfo &STI);
 
+  void expandLoadStoreBasic(const MCInst &Inst, MemInstInfo &InstInfo,
+      MCStreamer &Out, const MCSubtargetInfo &STI);
+
+  void expandLoadStoreRoW(const MCInst &Inst, MemInstInfo &InstInfo,
+      MCStreamer &Out, const MCSubtargetInfo &STI);
+
   void expandSyscall(const MCInst &Inst, MCStreamer &Out,
                      const MCSubtargetInfo &STI);
 
diff --git a/llvm/lib/Target/AArch64/Utils/AArch64BaseInfo.h b/llvm/lib/Target/AArch64/Utils/AArch64BaseInfo.h
index 9671fa3b3d92f..a3dea22335d21 100644
--- a/llvm/lib/Target/AArch64/Utils/AArch64BaseInfo.h
+++ b/llvm/lib/Target/AArch64/Utils/AArch64BaseInfo.h
@@ -22,6 +22,7 @@
 #include "llvm/ADT/BitmaskEnum.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringSwitch.h"
+#include "llvm/MC/MCInst.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/TargetParser/SubtargetFeature.h"
 
@@ -247,6 +248,816 @@ static inline bool atomicBarrierDroppedOnZero(unsigned Opcode) {
   return false;
 }
 
+struct MemInstInfo {
+  MemInstInfo() = default;
+
+  int DestRegIdx;
+  int BaseRegIdx;
+  int OffsetIdx;
+  bool IsPrePost = false;
+  bool IsPair = false;
+};
+
+static inline std::optional<MemInstInfo> getLoadInfo(const MCInst &Inst) {
+  int DestRegIdx;
+  int BaseRegIdx;
+  int OffsetIdx;
+  bool IsPrePost = false;
+  bool IsPair = false;
+  switch (Inst.getOpcode()) {
+  default:
+    return std::nullopt;
+
+  case AArch64::LD1i64:
+  case AArch64::LD2i64:
+    DestRegIdx = 0;
+    BaseRegIdx = 3;
+    OffsetIdx = -1;
+    IsPrePost = false;
+    break;
+
+  case AArch64::LD1i8:
+  case AArch64::LD1i16:
+  case AArch64::LD1i32:
+  case AArch64::LD2i8:
+  case AArch64::LD2i16:
+  case AArch64::LD2i32:
+  case AArch64::LD3i8:
+  case AArch64::LD3i16:
+  case AArch64::LD3i32:
+  case AArch64::LD3i64:
+  case AArch64::LD4i8:
+  case AArch64::LD4i16:
+  case AArch64::LD4i32:
+  case AArch64::LD4i64:
+    DestRegIdx = -1;
+    BaseRegIdx = 3;
+    OffsetIdx = -1;
+    IsPrePost = false;
+    break;
+
+  case AArch64::LD1Onev1d:
+  case AArch64::LD1Onev2s:
+  case AArch64::LD1Onev4h:
+  case AArch64::LD1Onev8b:
+  case AArch64::LD1Onev2d:
+  case AArch64::LD1Onev4s:
+  case AArch64::LD1Onev8h:
+  case AArch64::LD1Onev16b:
+  case AArch64::LD1Rv1d:
+  case AArch64::LD1Rv2s:
+  case AArch64::LD1Rv4h:
+  case AArch64::LD1Rv8b:
+  case AArch64::LD1Rv2d:
+  case AArch64::LD1Rv4s:
+  case AArch64::LD1Rv8h:
+  case AArch64::LD1Rv16b:
+    DestRegIdx = 0;
+    BaseRegIdx = 1;
+    OffsetIdx = -1;
+    IsPrePost = false;
+    break;
+
+  case AArch64::LD1Twov1d:
+  case AArch64::LD1Twov2s:
+  case AArch64::LD1Twov4h:
+  case AArch64::LD1Twov8b:
+  case AArch64::LD1Twov2d:
+  case AArch64::LD1Twov4s:
+  case AArch64::LD1Twov8h:
+  case AArch64::LD1Twov16b:
+  case AArch64::LD1Threev1d:
+  case AArch64::LD1Threev2s:
+  case AArch64::LD1Threev4h:
+  case AArch64::LD1Threev8b:
+  case AArch64::LD1Threev2d:
+  case AArch64::LD1Threev4s:
+  case AArch64::LD1Threev8h:
+  case AArch64::LD1Threev16b:
+  case AArch64::LD1Fourv1d:
+  case AArch64::LD1Fourv2s:
+  case AArch64::LD1Fourv4h:
+  case AArch64::LD1Fourv8b:
+  case AArch64::LD1Fourv2d:
+  case AArch64::LD1Fourv4s:
+  case AArch64::LD1Fourv8h:
+  case AArch64::LD1Fourv16b:
+  case AArch64::LD2Twov2s:
+  case AArch64::LD2Twov4s:
+  case AArch64::LD2Twov8b:
+  case AArch64::LD2Twov2d:
+  case AArch64::LD2Twov4h:
+  case AArch64::LD2Twov8h:
+  case AArch64::LD2Twov16b:
+  case AArch64::LD2Rv1d:
+  case AArch64::LD2Rv2s:
+  case AArch64::LD2Rv4s:
+  case AArch64::LD2Rv8b:
+  case AArch64::LD2Rv2d:
+  case AArch64::LD2Rv4h:
+  case AArch64::LD2Rv8h:
+  case AArch64::LD2Rv16b:
+  case AArch64::LD3Threev2s:
+  case AArch64::LD3Threev4h:
+  case AArch64::LD3Threev8b:
+  case AArch64::LD3Threev2d:
+  case AArch64::LD3Threev4s:
+  case AArch64::LD3Threev8h:
+  case AArch64::LD3Threev16b:
+  case AArch64::LD3Rv1d:
+  case AArch64::LD3Rv2s:
+  case AArch64::LD3Rv4h:
+  case AArch64::LD3Rv8b:
+  case AArch64::LD3Rv2d:
+  case AArch64::LD3Rv4s:
+  case AArch64::LD3Rv8h:
+  case AArch64::LD3Rv16b:
+  case AArch64::LD4Fourv2s:
+  case AArch64::LD4Fourv4h:
+  case AArch64::LD4Fourv8b:
+  case AArch64::LD4Fourv2d:
+  case AArch64::LD4Fourv4s:
+  case AArch64::LD4Fourv8h:
+  case AArch64::LD4Fourv16b:
+  case AArch64::LD4Rv1d:
+  case AArch64::LD4Rv2s:
+  case AArch64::LD4Rv4h:
+  case AArch64::LD4Rv8b:
+  case AArch64::LD4Rv2d:
+  case AArch64::LD4Rv4s:
+  case AArch64::LD4Rv8h:
+  case AArch64::LD4Rv16b:
+    DestRegIdx = -1;
+    BaseRegIdx = 1;
+    OffsetIdx = -1;
+    IsPrePost = false;
+    break;
+
+  case AArch64::LD1i64_POST:
+  case AArch64::LD2i64_POST:
+    DestRegIdx = 1;
+    BaseRegIdx = 4;
+    OffsetIdx = 5;
+    IsPrePost = true;
+    break;
+
+  case AArch64::LD1i8_POST:
+  case AArch64::LD1i16_POST:
+  case AArch64::LD1i32_POST:
+  case AArch64::LD2i8_POST:
+  case AArch64::LD2i16_POST:
+  case AArch64::LD2i32_POST:
+  case AArch64::LD3i8_POST:
+  case AArch64::LD3i16_POST:
+  case AArch64::LD3i32_POST:
+  case AArch64::LD3i64_POST:
+  case AArch64::LD4i8_POST:
+  case AArch64::LD4i16_POST:
+  case AArch64::LD4i32_POST:
+  case AArch64::LD4i64_POST:
+    DestRegIdx = -1;
+    BaseRegIdx = 4;
+    OffsetIdx = 5;
+    IsPrePost = true;
+    break;
+
+  case AArch64::LD1Onev1d_POST:
+  case AArch64::LD1Onev2s_POST:
+  case AArch64::LD1Onev4h_POST:
+  case AArch64::LD1Onev8b_POST:
+  case AArch64::LD1Onev2d_POST:
+  case AArch64::LD1Onev4s_POST:
+  case AArch64::LD1Onev8h_POST:
+  case AArch64::LD1Onev16b_POST:
+  case AArch64::LD1Rv1d_POST:
+  case AArch64::LD1Rv2s_POST:
+  case AArch64::LD1Rv4h_POST:
+  case AArch64::LD1Rv8b_POST:
+  case AArch64::LD1Rv2d_POST:
+  case AArch64::LD1Rv4s_POST:
+  case AArch64::LD1Rv8h_POST:
+  case AArch64::LD1Rv16b_POST:
+    DestRegIdx = 1;
+    BaseRegIdx = 2;
+    OffsetIdx = 3;
+    IsPrePost = true;
+    break;
+
+  case AArch64::LD1Twov1d_POST:
+  case AArch64::LD1Twov2s_POST:
+  case AArch64::LD1Twov4h_POST:
+  case AArch64::LD1Twov8b_POST:
+  case AArch64::LD1Twov2d_POST:
+  case AArch64::LD1Twov4s_POST:
+  case AArch64::LD1Twov8h_POST:
+  case AArch64::LD1Twov16b_POST:
+  case AArch64::LD1Threev1d_POST:
+  case AArch64::LD1Threev2s_POST:
+  case AArch64::LD1Threev4h_POST:
+  case AArch64::LD1Threev8b_POST:
+  case AArch64::LD1Threev2d_POST:
+  case AArch64::LD1Threev4s_POST:
+  case AArch64::LD1Threev8h_POST:
+  case AArch64::LD1Threev16b_POST:
+  case AArch64::LD1Fourv1d_POST:
+  case AArch64::LD1Fourv2s_POST:
+  case AArch64::LD1Fourv4h_POST:
+  case AArch64::LD1Fourv8b_POST:
+  case AArch64::LD1Fourv2d_POST:
+  case AArch64::LD1Fourv4s_POST:
+  case AArch64::LD1Fourv8h_POST:
+  case AArch64::LD1Fourv16b_POST:
+  case AArch64::LD2Twov2s_POST:
+  case AArch64::LD2Twov4s_POST:
+  case AArch64::LD2Twov8b_POST:
+  case AArch64::LD2Twov2d_POST:
+  case AArch64::LD2Twov4h_POST:
+  case AArch64::LD2Twov8h_POST:
+  case AArch64::LD2Twov16b_POST:
+  case AArch64::LD2Rv1d_POST:
+  case AArch64::LD2Rv2s_POST:
+  case AArch64::LD2Rv4s_POST:
+  case AArch64::LD2Rv8b_POST:
+  case AArch64::LD2Rv2d_POST:
+  case AArch64::LD2Rv4h_POST:
+  case AArch64::LD2Rv8h_POST:
+  case AArch64::LD2Rv16b_POST:
+  case AArch64::LD3Threev2s_POST:
+  case AArch64::LD3Threev4h_POST:
+  case AArch64::LD3Threev8b_POST:
+  case AArch64::LD3Threev2d_POST:
+  case AArch64::LD3Threev4s_POST:
+  case AArch64::LD3Threev8h_POST:
+  case AArch64::LD3Threev16b_POST:
+  case AArch64::LD3Rv1d_POST:
+  case AArch64::LD3Rv2s_POST:
+  case AArch64::LD3Rv4h_POST:
+  case AArch64::LD3Rv8b_POST:
+  case AArch64::LD3Rv2d_POST:
+  case AArch64::LD3Rv4s_POST:
+  case AArch64::LD3Rv8h_POST:
+  case AArch64::LD3Rv16b_POST:
+  case AArch64::LD4Fourv2s_POST:
+  case AArch64::LD4Fourv4h_POST:
+  case AArch64::LD4Fourv8b_POST:
+  case AArch64::LD4Fourv2d_POST:
+  case AArch64::LD4Fourv4s_POST:
+  case AArch64::LD4Fourv8h_POST:
+  case AArch64::LD4Fourv16b_POST:
+  case AArch64::LD4Rv1d_POST:
+  case AArch64::LD4Rv2s_POST:
+  case AArch64::LD4Rv4h_POST:
+  case AArch64::LD4Rv8b_POST:
+  case AArch64::LD4Rv2d_POST:
+  case AArch64::LD4Rv4s_POST:
+  case AArch64::LD4Rv8h_POST:
+  case AArch64::LD4Rv16b_POST:
+    DestRegIdx = -1;
+    BaseRegIdx = 2;
+    OffsetIdx = 3;
+    IsPrePost = true;
+    break;
+
+  case AArch64::LDRBBroW:
+  case AArch64::LDRBBroX:
+  case AArch64::LDRBBui:
+  case AArch64::LDRBroW:
+  case AArch64::LDRBroX:
+  case AArch64::LDRBui:
+  case AArch64::LDRDl:
+  case AArch64::LDRDroW:
+  case AArch64::LDRDroX:
+  case AArch64::LDRDui:
+  case AArch64::LDRHHroW:
+  case AArch64::LDRHHroX:
+  case AArch64::LDRHHui:
+  case AArch64::LDRHroW:
+  case AArch64::LDRHroX:
+  case AArch64::LDRHui:
+  case AArch64::LDRQl:
+  case AArch64::LDRQroW:
+  case AArch64::LDRQroX:
+  case AArch64::LDRQui:
+  case AArch64::LDRSBWroW:
+  case AArch64::LDRSBWroX:
+  case AArch64::LDRSBWui:
+  case AArch64::LDRSBXroW:
+  case AArch64::LDRSBXroX:
+  case AArch64::LDRSBXui:
+  case AArch64::LDRSHWroW:
+  case AArch64::LDRSHWroX:
+  case AArch64::LDRSHWui:
+  case AArch64::LDRSHXroW:
+  case AArch64::LDRSHXroX:
+  case AArch64::LDRSHXui:
+  case AArch64::LDRSWl:
+  case AArch64::LDRSWroW:
+  case AArch64::LDRSWroX:
+  case AArch64::LDRSWui:
+  case AArch64::LDRSl:
+  case AArch64::LDRSroW:
+  case AArch64::LDRSroX:
+  case AArch64::LDRSui:
+  case AArch64::LDRWl:
+  case AArch64::LDRWroW:
+  case AArch64::LDRWroX:
+  case AArch64::LDRWui:
+  case AArch64::LDRXl:
+  case AArch64::LDRXroW:
+  case AArch64::LDRXroX:
+  case AArch64::LDRXui:
+  case AArch64::LDURBBi:
+  case AArch64::LDURBi:
+  case AArch64::LDURDi:
+  case AArch64::LDURHHi:
+  case AArch64::LDURHi:
+  case AArch64::LDURQi:
+  case AArch64::LDURSBWi:
+  case AArch64::LDURSBXi:
+  case AArch64::LDURSHWi:
+  case AArch64::LDURSHXi:
+  case AArch64::LDURSWi:
+  case AArch64::LDURSi:
+  case AArch64::LDURWi:
+  case AArch64::LDURXi:
+    DestRegIdx = 0;
+    BaseRegIdx = 1;
+    OffsetIdx = 2;
+    IsPrePost = false;
+    break;
+
+  case AArch64::LDRBBpost:
+  case AArch64::LDRBBpre:
+  case AArch64::LDRBpost:
+  case AArch64::LDRBpre:
+  case AArch64::LDRDpost:
+  case AArch64::LDRDpre:
+  case AArch64::LDRHHpost:
+  case AArch64::LDRHHpre:
+  case AArch64::LDRHpost:
+  case AArch64::LDRHpre:
+  case AArch64::LDRQpost:
+  case AArch64::LDRQpre:
+  case AArch64::LDRSBWpost:
+  case AArch64::LDRSBWpre:
+  case AArch64::LDRSBXpost:
+  case AArch64::LDRSBXpre:
+  case AArch64::LDRSHWpost:
+  case AArch64::LDRSHWpre:
+  case AArch64::LDRSHXpost:
+  case AArch64::LDRSHXpre:
+  case AArch64::LDRSWpost:
+  case AArch64::LDRSWpre:
+  case AArch64::LDRSpost:
+  case AArch64::LDRSpre:
+  case AArch64::LDRWpost:
+  case AArch64::LDRWpre:
+  case AArch64::LDRXpost:
+  case AArch64::LDRXpre:
+    DestRegIdx = 1;
+    BaseRegIdx = 2;
+    OffsetIdx = 3;
+    IsPrePost = true;
+    break;
+
+  case AArch64::LDNPDi:
+  case AArch64::LDNPQi:
+  case AArch64::LDNPSi:
+  case AArch64::LDPQi:
+  case AArch64::LDPDi:
+  case AArch64::LDPSi:
+    DestRegIdx = 0;
+    BaseRegIdx = 2;
+    OffsetIdx = 3;
+    IsPrePost = false;
+    IsPair = true;
+    break;
+
+  case AArch64::LDPSWi:
+  case AArch64::LDPWi:
+  case AArch64::LDPXi:
+    DestRegIdx = 0;
+    BaseRegIdx = 2;
+    OffsetIdx = 3;
+    IsPrePost = false;
+    IsPair = true;
+    break;
+
+  case AArch64::LDPQpost:
+  case AArch64::LDPQpre:
+  case AArch64::LDPDpost:
+  case AArch64::LDPDpre:
+  case AArch64::LDPSpost:
+  case AArch64::LDPSpre:
+    DestRegIdx = 1;
+    BaseRegIdx = 3;
+    OffsetIdx = 4;
+    IsPrePost = true;
+    IsPair = true;
+    break;
+
+  case AArch64::LDPSWpost:
+  case AArch64::LDPSWpre:
+  case AArch64::LDPWpost:
+  case AArch64::LDPWpre:
+  case AArch64::LDPXpost:
+  case AArch64::LDPXpre:
+    DestRegIdx = 1;
+    BaseRegIdx = 3;
+    OffsetIdx = 4;
+    IsPrePost = true;
+    IsPair = true;
+    break;
+
+  // LD(A)?(X)?R(B|H|W|X)
+  case AArch64::LDXRB:
+  case AArch64::LDXRH:
+  case AArch64::LDXRW:
+  case AArch64::LDXRX:
+  case AArch64::LDAXRB:
+  case AArch64::LDAXRH:
+  case AArch64::LDAXRW:
+  case AArch64::LDAXRX:
+  case AArch64::LDARB:
+  case AArch64::LDARH:
+  case AArch64::LDARW:
+  case AArch64::LDARX:
+    DestRegIdx = 0;
+    BaseRegIdx = 1;
+    OffsetIdx = -1;
+    IsPrePost = false;
+    break;
+
+  // LD(A|X)P(W|X)
+  case AArch64::LDXPW:
+  case AArch64::LDXPX:
+  case AArch64::LDAXPW:
+  case AArch64::LDAXPX:
+    DestRegIdx = 0;
+    BaseRegIdx = 2;
+    OffsetIdx = -1;
+    IsPrePost = false;
+    IsPair = true;
+    break;
+
+  }
+
+  return MemInstInfo { DestRegIdx, BaseRegIdx, OffsetIdx, IsPrePost, IsPair };
+}
+
+static inline std::optional<MemInstInfo> getStoreInfo(const MCInst &Inst) {
+  int DestRegIdx;
+  int BaseRegIdx;
+  int OffsetIdx;
+  bool IsPrePost = false;
+  bool IsPair = false;
+  switch (Inst.getOpcode()) {
+  default:
+    return std::nullopt;
+
+  case AArch64::ST1i8:
+  case AArch64::ST1i16:
+  case AArch64::ST1i32:
+  case AArch64::ST1i64:
+  case AArch64::ST2i8:
+  case AArch64::ST2i16:
+  case AArch64::ST2i32:
+  case AArch64::ST2i64:
+  case AArch64::ST3i8:
+  case AArch64::ST3i16:
+  case AArch64::ST3i32:
+  case AArch64::ST3i64:
+  case AArch64::ST4i8:
+  case AArch64::ST4i16:
+  case AArch64::ST4i32:
+  case AArch64::ST4i64:
+    DestRegIdx = -1;
+    BaseRegIdx = 2;
+    OffsetIdx = -1;
+    IsPrePost = false;
+    break;
+
+  case AArch64::ST1Onev1d:
+  case AArch64::ST1Onev2s:
+  case AArch64::ST1Onev4h:
+  case AArch64::ST1Onev8b:
+  case AArch64::ST1Onev2d:
+  case AArch64::ST1Onev4s:
+  case AArch64::ST1Onev8h:
+  case AArch64::ST1Onev16b:
+    DestRegIdx = 0;
+    BaseRegIdx = 1;
+    OffsetIdx = -1;
+    IsPrePost = false;
+    break;
+
+  case AArch64::ST1Twov1d:
+  case AArch64::ST1Twov2s:
+  case AArch64::ST1Twov4h:
+  case AArch64::ST1Twov8b:
+  case AArch64::ST1Twov2d:
+  case AArch64::ST1Twov4s:
+  case AArch64::ST1Twov8h:
+  case AArch64::ST1Twov16b:
+  case AArch64::ST1Threev1d:
+  case AArch64::ST1Threev2s:
+  case AArch64::ST1Threev4h:
+  case AArch64::ST1Threev8b:
+  case AArch64::ST1Threev2d:
+  case AArch64::ST1Threev4s:
+  case AArch64::ST1Threev8h:
+  case AArch64::ST1Threev16b:
+  case AArch64::ST1Fourv1d:
+  case AArch64::ST1Fourv2s:
+  case AArch64::ST1Fourv4h:
+  case AArch64::ST1Fourv8b:
+  case AArch64::ST1Fourv2d:
+  case AArch64::ST1Fourv4s:
+  case AArch64::ST1Fourv8h:
+  case AArch64::ST1Fourv16b:
+  case AArch64::ST2Twov2s:
+  case AArch64::ST2Twov4s:
+  case AArch64::ST2Twov8b:
+  case AArch64::ST2Twov2d:
+  case AArch64::ST2Twov4h:
+  case AArch64::ST2Twov8h:
+  case AArch64::ST2Twov16b:
+  case AArch64::ST3Threev2s:
+  case AArch64::ST3Threev4h:
+  case AArch64::ST3Threev8b:
+  case AArch64::ST3Threev2d:
+  case AArch64::ST3Threev4s:
+  case AArch64::ST3Threev8h:
+  case AArch64::ST3Threev16b:
+  case AArch64::ST4Fourv2s:
+  case AArch64::ST4Fourv4h:
+  case AArch64::ST4Fourv8b:
+  case AArch64::ST4Fourv2d:
+  case AArch64::ST4Fourv4s:
+  case AArch64::ST4Fourv8h:
+  case AArch64::ST4Fourv16b:
+    DestRegIdx = -1;
+    BaseRegIdx = 1;
+    OffsetIdx = -1;
+    IsPrePost = false;
+    break;
+
+  case AArch64::ST1i64_POST:
+  case AArch64::ST2i64_POST:
+    DestRegIdx = 1;
+    BaseRegIdx = 4;
+    OffsetIdx = 5;
+    IsPrePost = true;
+    break;
+
+  // SIMDStSingleS
+  case AArch64::ST1i8_POST:
+  case AArch64::ST1i16_POST:
+  case AArch64::ST1i32_POST:
+  case AArch64::ST2i8_POST:
+  case AArch64::ST2i16_POST:
+  case AArch64::ST2i32_POST:
+  case AArch64::ST3i8_POST:
+  case AArch64::ST3i16_POST:
+  case AArch64::ST3i32_POST:
+  case AArch64::ST3i64_POST:
+  case AArch64::ST4i8_POST:
+  case AArch64::ST4i16_POST:
+  case AArch64::ST4i32_POST:
+  case AArch64::ST4i64_POST:
+    DestRegIdx = -1;
+    BaseRegIdx = 3;
+    OffsetIdx = 4;
+    IsPrePost = true;
+    break;
+
+  case AArch64::ST1Onev1d_POST:
+  case AArch64::ST1Onev2s_POST:
+  case AArch64::ST1Onev4h_POST:
+  case AArch64::ST1Onev8b_POST:
+  case AArch64::ST1Onev2d_POST:
+  case AArch64::ST1Onev4s_POST:
+  case AArch64::ST1Onev8h_POST:
+  case AArch64::ST1Onev16b_POST:
+    DestRegIdx = 1;
+    BaseRegIdx = 2;
+    OffsetIdx = 3;
+    IsPrePost = true;
+    break;
+
+  case AArch64::ST1Twov1d_POST:
+  case AArch64::ST1Twov2s_POST:
+  case AArch64::ST1Twov4h_POST:
+  case AArch64::ST1Twov8b_POST:
+  case AArch64::ST1Twov2d_POST:
+  case AArch64::ST1Twov4s_POST:
+  case AArch64::ST1Twov8h_POST:
+  case AArch64::ST1Twov16b_POST:
+  case AArch64::ST1Threev1d_POST:
+  case AArch64::ST1Threev2s_POST:
+  case AArch64::ST1Threev4h_POST:
+  case AArch64::ST1Threev8b_POST:
+  case AArch64::ST1Threev2d_POST:
+  case AArch64::ST1Threev4s_POST:
+  case AArch64::ST1Threev8h_POST:
+  case AArch64::ST1Threev16b_POST:
+  case AArch64::ST1Fourv1d_POST:
+  case AArch64::ST1Fourv2s_POST:
+  case AArch64::ST1Fourv4h_POST:
+  case AArch64::ST1Fourv8b_POST:
+  case AArch64::ST1Fourv2d_POST:
+  case AArch64::ST1Fourv4s_POST:
+  case AArch64::ST1Fourv8h_POST:
+  case AArch64::ST1Fourv16b_POST:
+  case AArch64::ST2Twov2s_POST:
+  case AArch64::ST2Twov4s_POST:
+  case AArch64::ST2Twov8b_POST:
+  case AArch64::ST2Twov2d_POST:
+  case AArch64::ST2Twov4h_POST:
+  case AArch64::ST2Twov8h_POST:
+  case AArch64::ST2Twov16b_POST:
+  case AArch64::ST3Threev2s_POST:
+  case AArch64::ST3Threev4h_POST:
+  case AArch64::ST3Threev8b_POST:
+  case AArch64::ST3Threev2d_POST:
+  case AArch64::ST3Threev4s_POST:
+  case AArch64::ST3Threev8h_POST:
+  case AArch64::ST3Threev16b_POST:
+  case AArch64::ST4Fourv2s_POST:
+  case AArch64::ST4Fourv4h_POST:
+  case AArch64::ST4Fourv8b_POST:
+  case AArch64::ST4Fourv2d_POST:
+  case AArch64::ST4Fourv4s_POST:
+  case AArch64::ST4Fourv8h_POST:
+  case AArch64::ST4Fourv16b_POST:
+    DestRegIdx = -1;
+    BaseRegIdx = 2;
+    OffsetIdx = 3;
+    IsPrePost = true;
+    break;
+
+  case AArch64::STRBBroW:
+  case AArch64::STRBBroX:
+  case AArch64::STRBBui:
+  case AArch64::STRBroW:
+  case AArch64::STRBroX:
+  case AArch64::STRBui:
+  case AArch64::STRDroW:
+  case AArch64::STRDroX:
+  case AArch64::STRDui:
+  case AArch64::STRHHroW:
+  case AArch64::STRHHroX:
+  case AArch64::STRHHui:
+  case AArch64::STRHroW:
+  case AArch64::STRHroX:
+  case AArch64::STRHui:
+  case AArch64::STRQroW:
+  case AArch64::STRQroX:
+  case AArch64::STRQui:
+  case AArch64::STRSroW:
+  case AArch64::STRSroX:
+  case AArch64::STRSui:
+  case AArch64::STRWroW:
+  case AArch64::STRWroX:
+  case AArch64::STRWui:
+  case AArch64::STRXroW:
+  case AArch64::STRXroX:
+  case AArch64::STRXui:
+  case AArch64::STURBBi:
+  case AArch64::STURBi:
+  case AArch64::STURDi:
+  case AArch64::STURHHi:
+  case AArch64::STURHi:
+  case AArch64::STURQi:
+  case AArch64::STURSi:
+  case AArch64::STURWi:
+  case AArch64::STURXi:
+    DestRegIdx = 0;
+    BaseRegIdx = 1;
+    OffsetIdx = 2;
+    IsPrePost = false;
+    break;
+
+  case AArch64::STRBBpost:
+  case AArch64::STRBBpre:
+  case AArch64::STRBpost:
+  case AArch64::STRBpre:
+  case AArch64::STRDpost:
+  case AArch64::STRDpre:
+  case AArch64::STRHHpost:
+  case AArch64::STRHHpre:
+  case AArch64::STRHpost:
+  case AArch64::STRHpre:
+  case AArch64::STRQpost:
+  case AArch64::STRQpre:
+  case AArch64::STRSpost:
+  case AArch64::STRSpre:
+  case AArch64::STRWpost:
+  case AArch64::STRWpre:
+  case AArch64::STRXpost:
+  case AArch64::STRXpre:
+    DestRegIdx = 1;
+    BaseRegIdx = 2;
+    OffsetIdx = 3;
+    IsPrePost = true;
+    break;
+
+  case AArch64::STNPDi:
+  case AArch64::STNPQi:
+  case AArch64::STNPSi:
+  case AArch64::STPQi:
+  case AArch64::STPDi:
+  case AArch64::STPSi:
+    DestRegIdx = 0;
+    BaseRegIdx = 2;
+    OffsetIdx = 3;
+    IsPrePost = false;
+    IsPair = true;
+    break;
+
+  case AArch64::STPWi:
+  case AArch64::STPXi:
+    DestRegIdx = 0;
+    BaseRegIdx = 2;
+    OffsetIdx = 3;
+    IsPrePost = false;
+    IsPair = true;
+    break;
+
+  case AArch64::STPQpost:
+  case AArch64::STPQpre:
+  case AArch64::STPDpost:
+  case AArch64::STPDpre:
+  case AArch64::STPSpost:
+  case AArch64::STPSpre:
+    DestRegIdx = 1;
+    BaseRegIdx = 3;
+    OffsetIdx = 4;
+    IsPrePost = true;
+    IsPair = true;
+    break;
+
+  case AArch64::STPWpost:
+  case AArch64::STPWpre:
+  case AArch64::STPXpost:
+  case AArch64::STPXpre:
+    DestRegIdx = 1;
+    BaseRegIdx = 3;
+    OffsetIdx = 4;
+    IsPrePost = true;
+    IsPair = true;
+    break;
+
+  // op Dst, [Base]
+  case AArch64::STLRB:
+  case AArch64::STLRH:
+  case AArch64::STLRW:
+  case AArch64::STLRX:
+    DestRegIdx = 0;
+    BaseRegIdx = 1;
+    OffsetIdx = -1;
+    IsPrePost = false;
+    break;
+
+  // Base = op Dst, [Base, (-4|-8)]!
+  case AArch64::STLRWpre:
+  case AArch64::STLRXpre:
+    DestRegIdx = 1;
+    BaseRegIdx = 2;
+    OffsetIdx = -1; // constant offset
+    IsPrePost = true;
+    break;
+
+  case AArch64::STXRB:
+  case AArch64::STXRH:
+  case AArch64::STXRW:
+  case AArch64::STXRX:
+  case AArch64::STLXRB:
+  case AArch64::STLXRH:
+  case AArch64::STLXRW:
+  case AArch64::STLXRX:
+    DestRegIdx = 0;
+    BaseRegIdx = 2;
+    OffsetIdx = -1;
+    IsPrePost = false;
+    break;
+
+  case AArch64::STXPW:
+  case AArch64::STXPX:
+  case AArch64::STLXPW:
+  case AArch64::STLXPX:
+    DestRegIdx = 1; // return the status result at 0
+    BaseRegIdx = 3;
+    OffsetIdx = -1;
+    IsPrePost = false;
+    IsPair = true;
+    break;
+  }
+
+  return MemInstInfo { DestRegIdx, BaseRegIdx, OffsetIdx, IsPrePost, IsPair };
+}
+
+
 namespace AArch64CC {
 
 // The CondCodes constants map directly to the 4-bit encoding of the condition
diff --git a/llvm/test/MC/AArch64/LFI/mem.s b/llvm/test/MC/AArch64/LFI/mem.s
new file mode 100644
index 0000000000000..65be8cd1197d4
--- /dev/null
+++ b/llvm/test/MC/AArch64/LFI/mem.s
@@ -0,0 +1,106 @@
+// RUN: llvm-mc -filetype asm -triple aarch64_lfi %s | FileCheck %s
+
+ldr x0, [sp]
+// CHECK: ldr x0, [sp]
+
+ldr x0, [sp, #8]
+// CHECK: ldr x0, [sp, #8]
+
+ldp x0, x1, [sp, #8]
+// CHECK: ldp x0, x1, [sp, #8]
+
+str x0, [sp]
+// CHECK: str x0, [sp]
+
+str x0, [sp, #8]
+// CHECK: str x0, [sp, #8]
+
+stp x0, x1, [sp, #8]
+// CHECK: stp x0, x1, [sp, #8]
+
+ldur x0, [x1]
+// CHECK:      add x28, x27, w1, uxtw
+// CHECK-NEXT: ldur x0, [x28]
+
+stur x0, [x1]
+// CHECK:      add x28, x27, w1, uxtw
+// CHECK-NEXT: stur x0, [x28]
+
+ldp x0, x1, [x2]
+// CHECK:      add x28, x27, w2, uxtw
+// CHECK-NEXT: ldp x0, x1, [x28]
+
+stp x0, x1, [x2]
+// CHECK:      add x28, x27, w2, uxtw
+// CHECK-NEXT: stp x0, x1, [x28]
+
+ldr x0, [x1]
+// CHECK: ldr x0, [x27, w1, uxtw]
+
+ldr x0, [x1, #8]
+// CHECK:      add x28, x27, w1, uxtw
+// CHECK-NEXT: ldr x0, [x28, #8]
+
+ldr x0, [x1, #8]!
+// CHECK:      add x1, x1, #8
+// CHECK-NEXT: ldr x0, [x27, w1, uxtw]
+
+str x0, [x1, #8]!
+// CHECK:      add x1, x1, #8
+// CHECK-NEXT: str x0, [x27, w1, uxtw]
+
+ldr x0, [x1, #-8]!
+// CHECK:      sub x1, x1, #8
+// CHECK-NEXT: ldr x0, [x27, w1, uxtw]
+
+str x0, [x1, #-8]!
+// CHECK:      sub x1, x1, #8
+// CHECK-NEXT: str x0, [x27, w1, uxtw]
+
+ldr x0, [x1], #8
+// CHECK:      ldr x0, [x27, w1, uxtw]
+// CHECK-NEXT: add x1, x1, #8
+
+str x0, [x1], #8
+// CHECK:      str x0, [x27, w1, uxtw]
+// CHECK-NEXT: add x1, x1, #8
+
+ldr x0, [x1], #-8
+// CHECK:      ldr x0, [x27, w1, uxtw]
+// CHECK-NEXT: sub x1, x1, #8
+
+str x0, [x1], #-8
+// CHECK:      str x0, [x27, w1, uxtw]
+// CHECK-NEXT: sub x1, x1, #8
+
+ldr x0, [x1, x2]
+// CHECK:      add x26, x1, x2
+// CHECK-NEXT: ldr x0, [x27, w26, uxtw]
+
+ldr x0, [x1, x2, lsl #3]
+// CHECK:      add x26, x1, x2, lsl #3
+// CHECK-NEXT: ldr x0, [x27, w26, uxtw]
+
+ldr x0, [x1, x2, sxtx #0]
+// CHECK:      add x26, x1, x2, sxtx
+// CHECK-NEXT: ldr x0, [x27, w26, uxtw]
+
+ldr x0, [x1, x2, sxtx #3]
+// CHECK:      add x26, x1, x2, sxtx #3
+// CHECK-NEXT: ldr x0, [x27, w26, uxtw]
+
+ldr x0, [x1, w2, uxtw]
+// CHECK:      add x26, x1, w2, uxtw
+// CHECK-NEXT: ldr x0, [x27, w26, uxtw]
+
+ldr x0, [x1, w2, uxtw #3]
+// CHECK:      add x26, x1, w2, uxtw #3
+// CHECK-NEXT: ldr x0, [x27, w26, uxtw]
+
+ldr x0, [x1, w2, sxtw]
+// CHECK:      add x26, x1, w2, sxtw
+// CHECK-NEXT: ldr x0, [x27, w26, uxtw]
+
+ldr x0, [x1, w2, sxtw #3]
+// CHECK:      add x26, x1, w2, sxtw #3
+// CHECK-NEXT: ldr x0, [x27, w26, uxtw]

From 309c571891e44b14991eebb69da63f92615448cc Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Wed, 20 Aug 2025 16:30:38 -0700
Subject: [PATCH 11/23] [LFI] [AArch64] Rewrites for basic pre/post indexed
 load/stores

There is a remaining corner case for certain instructions with
hard-coded immediate offset values.
---
 .../MCTargetDesc/AArch64MCLFIExpander.cpp     | 667 +++++++++++++++++-
 llvm/test/MC/AArch64/LFI/mem.s                |  28 +
 2 files changed, 693 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
index dc8ad0039ebad..f3de76761ecbe 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
@@ -216,6 +216,24 @@ static unsigned convertUiToRoW(unsigned Op);
 static unsigned convertPreToRoW(unsigned Op);
 static unsigned convertPostToRoW(unsigned Op);
 
+static unsigned convertPrePostToBase(unsigned Op, bool &IsPre, bool &IsBaseNoOffset);
+static unsigned getPrePostScale(unsigned Op);
+
+static void emitSafeLoadStoreDemoted(const MCInst &Inst, unsigned N, MCStreamer &Out, const MCSubtargetInfo &STI) {
+  MCInst LoadStore;
+  bool IsPre, IsBaseNoOffset;
+  auto NewOpCode = convertPrePostToBase(Inst.getOpcode(), IsPre, IsBaseNoOffset);
+  LoadStore.setOpcode(NewOpCode);
+  for (unsigned I = 1; I < N; I++)
+    LoadStore.addOperand(Inst.getOperand(I));
+  LoadStore.addOperand(MCOperand::createReg(LFIAddrReg));
+  if (IsPre)
+    LoadStore.addOperand(Inst.getOperand(N + 1));
+  else if (!IsBaseNoOffset)
+    LoadStore.addOperand(MCOperand::createImm(0));
+  Out.emitInstruction(LoadStore, STI);
+}
+
 static void emitSafeLoadStore(const MCInst &Inst, unsigned N, MCStreamer &Out, const MCSubtargetInfo &STI) {
   MCInst LoadStore;
   LoadStore.setOpcode(Inst.getOpcode());
@@ -233,8 +251,33 @@ void AArch64::AArch64MCLFIExpander::expandLoadStoreBasic(const MCInst &Inst, Mem
 
   if (MII.IsPrePost) {
     assert(OffsetIdx != -1 && "Pre/Post must have valid OffsetIdx");
-    return Out.getContext().reportWarning(
-        Inst.getLoc(), "TODO: pre/post index");
+
+    emitSafeLoadStoreDemoted(Inst, MII.BaseRegIdx, Out, STI);
+    MCRegister Base = Inst.getOperand(MII.BaseRegIdx).getReg();
+    MCOperand OffsetMO = Inst.getOperand(MII.OffsetIdx);
+    if (OffsetMO.isReg()) {
+      // The immediate offset of post-indexed addressing NEON Instrs has a fixed value, and
+      // it is encoded as a post-index addressing with XZR register operand.
+      // e.g., LD3Threev3d_POST can only have #48 as its operand and its offset
+      // MachineOperand holds XZR, which is a *Register* kind, not Imm.
+
+      // TODO: handle this case: ld3 { v0.4s, v1.4s, v2.4s }, [x0], #48
+      // MCRegister OffReg = OffsetMO.getReg();
+      // if (OffReg == AArch64::XZR) {
+      //   const LdStNInstrDesc *Info = getLdStNInstrDesc(Inst.getOpcode());
+      //   assert(Info && Info->NaturalOffset >= 0);
+      //   return emit(AArch64::ADDXri, Base, Base, Info->NaturalOffset, 0, Out, STI);
+      // } else {
+      //   assert(OffReg != AArch64::WZR);
+      //   return emit(AArch64::ADDXrs, Base, Base, OffsetMO.getReg(), 0, Out, STI);
+      // }
+      return;
+    } else {
+      auto Offset = Inst.getOperand(MII.OffsetIdx).getImm() * getPrePostScale(Inst.getOpcode());
+      if (Offset >= 0)
+        return emit(AArch64::ADDXri, Base, Base, Offset, 0, Out, STI);
+      return emit(AArch64::SUBXri, Base, Base, -Offset, 0, Out, STI);
+    }
   }
 
   return emitSafeLoadStore(Inst, MII.BaseRegIdx, Out, STI);
@@ -753,3 +796,623 @@ static bool canConvertToRoW(unsigned Op) {
     convertRoXToRoW(Op, Shift) != AArch64::INSTRUCTION_LIST_END ||
     convertRoWToRoW(Op, Shift) != AArch64::INSTRUCTION_LIST_END;
 }
+
+static unsigned convertPrePostToBase(unsigned Op, bool &IsPre, bool &IsBaseNoOffset) {
+  IsPre = false;
+  IsBaseNoOffset = false;
+  switch (Op) {
+  case AArch64::LDPDpost:
+    return AArch64::LDPDi;
+  case AArch64::LDPDpre:
+    IsPre = true;
+    return AArch64::LDPDi;
+  case AArch64::LDPQpost:
+    return AArch64::LDPQi;
+  case AArch64::LDPQpre:
+    IsPre = true;
+    return AArch64::LDPQi;
+  case AArch64::LDPSWpost:
+    return AArch64::LDPSWi;
+  case AArch64::LDPSWpre:
+    IsPre = true;
+    return AArch64::LDPSWi;
+  case AArch64::LDPSpost:
+    return AArch64::LDPSi;
+  case AArch64::LDPSpre:
+    IsPre = true;
+    return AArch64::LDPSi;
+  case AArch64::LDPWpost:
+    return AArch64::LDPWi;
+  case AArch64::LDPWpre:
+    IsPre = true;
+    return AArch64::LDPWi;
+  case AArch64::LDPXpost:
+    return AArch64::LDPXi;
+  case AArch64::LDPXpre:
+    IsPre = true;
+    return AArch64::LDPXi;
+  case AArch64::STPDpost:
+    return AArch64::STPDi;
+  case AArch64::STPDpre:
+    IsPre = true;
+    return AArch64::STPDi;
+  case AArch64::STPQpost:
+    return AArch64::STPQi;
+  case AArch64::STPQpre:
+    IsPre = true;
+    return AArch64::STPQi;
+  case AArch64::STPSpost:
+    return AArch64::STPSi;
+  case AArch64::STPSpre:
+    IsPre = true;
+    return AArch64::STPSi;
+  case AArch64::STPWpost:
+    return AArch64::STPWi;
+  case AArch64::STPWpre:
+    IsPre = true;
+    return AArch64::STPWi;
+  case AArch64::STPXpost:
+    return AArch64::STPXi;
+  case AArch64::STPXpre:
+    IsPre = true;
+    return AArch64::STPXi;
+  // case AArch64::STLRWpre:
+  //   IsPre = true;
+  //   IsBaseNoOffset = true;
+  //   return AArch64::STPWi;
+  // case AArch64::STLRXpre:
+  //   IsPre = true;
+  //   IsBaseNoOffset = true;
+    // return AArch64::STLRX;
+  case AArch64::LD1i64_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1i64;
+  case AArch64::LD2i64_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD2i64;
+  case AArch64::LD1i8_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1i8;
+  case AArch64::LD1i16_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1i16;
+  case AArch64::LD1i32_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1i32;
+  case AArch64::LD2i8_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD2i8;
+  case AArch64::LD2i16_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD2i16;
+  case AArch64::LD2i32_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD2i32;
+  case AArch64::LD3i8_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD3i8;
+  case AArch64::LD3i16_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD3i16;
+  case AArch64::LD3i32_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD3i32;
+  case AArch64::LD3i64_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD3i64;
+  case AArch64::LD4i8_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD4i8;
+  case AArch64::LD4i16_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD4i16;
+  case AArch64::LD4i32_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD4i32;
+  case AArch64::LD4i64_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD4i64;
+  case AArch64::LD1Onev1d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Onev1d;
+  case AArch64::LD1Onev2s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Onev2s;
+  case AArch64::LD1Onev4h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Onev4h;
+  case AArch64::LD1Onev8b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Onev8b;
+  case AArch64::LD1Onev2d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Onev2d;
+  case AArch64::LD1Onev4s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Onev4s;
+  case AArch64::LD1Onev8h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Onev8h;
+  case AArch64::LD1Onev16b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Onev16b;
+  case AArch64::LD1Rv1d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Rv1d;
+  case AArch64::LD1Rv2s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Rv2s;
+  case AArch64::LD1Rv4h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Rv4h;
+  case AArch64::LD1Rv8b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Rv8b;
+  case AArch64::LD1Rv2d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Rv2d;
+  case AArch64::LD1Rv4s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Rv4s;
+  case AArch64::LD1Rv8h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Rv8h;
+  case AArch64::LD1Rv16b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Rv16b;
+  case AArch64::LD1Twov1d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Twov1d;
+  case AArch64::LD1Twov2s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Twov2s;
+  case AArch64::LD1Twov4h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Twov4h;
+  case AArch64::LD1Twov8b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Twov8b;
+  case AArch64::LD1Twov2d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Twov2d;
+  case AArch64::LD1Twov4s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Twov4s;
+  case AArch64::LD1Twov8h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Twov8h;
+  case AArch64::LD1Twov16b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Twov16b;
+  case AArch64::LD1Threev1d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Threev1d;
+  case AArch64::LD1Threev2s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Threev2s;
+  case AArch64::LD1Threev4h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Threev4h;
+  case AArch64::LD1Threev8b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Threev8b;
+  case AArch64::LD1Threev2d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Threev2d;
+  case AArch64::LD1Threev4s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Threev4s;
+  case AArch64::LD1Threev8h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Threev8h;
+  case AArch64::LD1Threev16b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Threev16b;
+  case AArch64::LD1Fourv1d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Fourv1d;
+  case AArch64::LD1Fourv2s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Fourv2s;
+  case AArch64::LD1Fourv4h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Fourv4h;
+  case AArch64::LD1Fourv8b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Fourv8b;
+  case AArch64::LD1Fourv2d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Fourv2d;
+  case AArch64::LD1Fourv4s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Fourv4s;
+  case AArch64::LD1Fourv8h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Fourv8h;
+  case AArch64::LD1Fourv16b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Fourv16b;
+  case AArch64::LD2Twov2s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD2Twov2s;
+  case AArch64::LD2Twov4s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD2Twov4s;
+  case AArch64::LD2Twov8b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD2Twov8b;
+  case AArch64::LD2Twov2d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD2Twov2d;
+  case AArch64::LD2Twov4h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD2Twov4h;
+  case AArch64::LD2Twov8h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD2Twov8h;
+  case AArch64::LD2Twov16b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD2Twov16b;
+  case AArch64::LD2Rv1d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD2Rv1d;
+  case AArch64::LD2Rv2s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD2Rv2s;
+  case AArch64::LD2Rv4s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD2Rv4s;
+  case AArch64::LD2Rv8b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD2Rv8b;
+  case AArch64::LD2Rv2d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD2Rv2d;
+  case AArch64::LD2Rv4h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD2Rv4h;
+  case AArch64::LD2Rv8h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD2Rv8h;
+  case AArch64::LD2Rv16b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD2Rv16b;
+  case AArch64::LD3Threev2s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD3Threev2s;
+  case AArch64::LD3Threev4h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD3Threev4h;
+  case AArch64::LD3Threev8b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD3Threev8b;
+  case AArch64::LD3Threev2d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD3Threev2d;
+  case AArch64::LD3Threev4s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD3Threev4s;
+  case AArch64::LD3Threev8h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD3Threev8h;
+  case AArch64::LD3Threev16b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD3Threev16b;
+  case AArch64::LD3Rv1d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD3Rv1d;
+  case AArch64::LD3Rv2s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD3Rv2s;
+  case AArch64::LD3Rv4h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD3Rv4h;
+  case AArch64::LD3Rv8b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD3Rv8b;
+  case AArch64::LD3Rv2d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD3Rv2d;
+  case AArch64::LD3Rv4s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD3Rv4s;
+  case AArch64::LD3Rv8h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD3Rv8h;
+  case AArch64::LD3Rv16b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD3Rv16b;
+  case AArch64::LD4Fourv2s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD4Fourv2s;
+  case AArch64::LD4Fourv4h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD4Fourv4h;
+  case AArch64::LD4Fourv8b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD4Fourv8b;
+  case AArch64::LD4Fourv2d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD4Fourv2d;
+  case AArch64::LD4Fourv4s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD4Fourv4s;
+  case AArch64::LD4Fourv8h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD4Fourv8h;
+  case AArch64::LD4Fourv16b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD4Fourv16b;
+  case AArch64::LD4Rv1d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD4Rv1d;
+  case AArch64::LD4Rv2s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD4Rv2s;
+  case AArch64::LD4Rv4h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD4Rv4h;
+  case AArch64::LD4Rv8b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD4Rv8b;
+  case AArch64::LD4Rv2d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD4Rv2d;
+  case AArch64::LD4Rv4s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD4Rv4s;
+  case AArch64::LD4Rv8h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD4Rv8h;
+  case AArch64::LD4Rv16b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD4Rv16b;
+  case AArch64::ST1i64_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1i64;
+  case AArch64::ST2i64_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST2i64;
+  case AArch64::ST1i8_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1i8;
+  case AArch64::ST1i16_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1i16;
+  case AArch64::ST1i32_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1i32;
+  case AArch64::ST2i8_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST2i8;
+  case AArch64::ST2i16_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST2i16;
+  case AArch64::ST2i32_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST2i32;
+  case AArch64::ST3i8_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST3i8;
+  case AArch64::ST3i16_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST3i16;
+  case AArch64::ST3i32_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST3i32;
+  case AArch64::ST3i64_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST3i64;
+  case AArch64::ST4i8_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST4i8;
+  case AArch64::ST4i16_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST4i16;
+  case AArch64::ST4i32_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST4i32;
+  case AArch64::ST4i64_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST4i64;
+  case AArch64::ST1Onev1d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Onev1d;
+  case AArch64::ST1Onev2s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Onev2s;
+  case AArch64::ST1Onev4h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Onev4h;
+  case AArch64::ST1Onev8b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Onev8b;
+  case AArch64::ST1Onev2d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Onev2d;
+  case AArch64::ST1Onev4s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Onev4s;
+  case AArch64::ST1Onev8h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Onev8h;
+  case AArch64::ST1Onev16b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Onev16b;
+  case AArch64::ST1Twov1d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Twov1d;
+  case AArch64::ST1Twov2s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Twov2s;
+  case AArch64::ST1Twov4h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Twov4h;
+  case AArch64::ST1Twov8b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Twov8b;
+  case AArch64::ST1Twov2d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Twov2d;
+  case AArch64::ST1Twov4s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Twov4s;
+  case AArch64::ST1Twov8h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Twov8h;
+  case AArch64::ST1Twov16b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Twov16b;
+  case AArch64::ST1Threev1d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Threev1d;
+  case AArch64::ST1Threev2s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Threev2s;
+  case AArch64::ST1Threev4h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Threev4h;
+  case AArch64::ST1Threev8b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Threev8b;
+  case AArch64::ST1Threev2d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Threev2d;
+  case AArch64::ST1Threev4s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Threev4s;
+  case AArch64::ST1Threev8h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Threev8h;
+  case AArch64::ST1Threev16b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Threev16b;
+  case AArch64::ST1Fourv1d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Fourv1d;
+  case AArch64::ST1Fourv2s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Fourv2s;
+  case AArch64::ST1Fourv4h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Fourv4h;
+  case AArch64::ST1Fourv8b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Fourv8b;
+  case AArch64::ST1Fourv2d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Fourv2d;
+  case AArch64::ST1Fourv4s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Fourv4s;
+  case AArch64::ST1Fourv8h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Fourv8h;
+  case AArch64::ST1Fourv16b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Fourv16b;
+  case AArch64::ST2Twov2s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST2Twov2s;
+  case AArch64::ST2Twov4s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST2Twov4s;
+  case AArch64::ST2Twov8b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST2Twov8b;
+  case AArch64::ST2Twov2d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST2Twov2d;
+  case AArch64::ST2Twov4h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST2Twov4h;
+  case AArch64::ST2Twov8h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST2Twov8h;
+  case AArch64::ST2Twov16b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST2Twov16b;
+  case AArch64::ST3Threev2s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST3Threev2s;
+  case AArch64::ST3Threev4h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST3Threev4h;
+  case AArch64::ST3Threev8b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST3Threev8b;
+  case AArch64::ST3Threev2d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST3Threev2d;
+  case AArch64::ST3Threev4s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST3Threev4s;
+  case AArch64::ST3Threev8h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST3Threev8h;
+  case AArch64::ST3Threev16b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST3Threev16b;
+  case AArch64::ST4Fourv2s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST4Fourv2s;
+  case AArch64::ST4Fourv4h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST4Fourv4h;
+  case AArch64::ST4Fourv8b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST4Fourv8b;
+  case AArch64::ST4Fourv2d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST4Fourv2d;
+  case AArch64::ST4Fourv4s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST4Fourv4s;
+  case AArch64::ST4Fourv8h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST4Fourv8h;
+  case AArch64::ST4Fourv16b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST4Fourv16b;
+  }
+  return AArch64::INSTRUCTION_LIST_END;
+}
+
+static unsigned getPrePostScale(unsigned Op) {
+  switch (Op) {
+  case AArch64::LDPDpost:
+  case AArch64::LDPDpre:
+    return 8;
+  case AArch64::LDPQpost:
+  case AArch64::LDPQpre:
+    return 16;
+  case AArch64::LDPSWpost:
+  case AArch64::LDPSWpre:
+    return 4;
+  case AArch64::LDPSpost:
+  case AArch64::LDPSpre:
+    return 4;
+  case AArch64::LDPWpost:
+  case AArch64::LDPWpre:
+    return 4;
+  case AArch64::LDPXpost:
+  case AArch64::LDPXpre:
+    return 8;
+  case AArch64::STPDpost:
+  case AArch64::STPDpre:
+    return 8;
+  case AArch64::STPQpost:
+  case AArch64::STPQpre:
+    return 16;
+  case AArch64::STPSpost:
+  case AArch64::STPSpre:
+    return 4;
+  case AArch64::STPWpost:
+  case AArch64::STPWpre:
+    return 4;
+  case AArch64::STPXpost:
+  case AArch64::STPXpre:
+    return 8;
+  }
+  return 0;
+}
diff --git a/llvm/test/MC/AArch64/LFI/mem.s b/llvm/test/MC/AArch64/LFI/mem.s
index 65be8cd1197d4..03dfd42b30454 100644
--- a/llvm/test/MC/AArch64/LFI/mem.s
+++ b/llvm/test/MC/AArch64/LFI/mem.s
@@ -104,3 +104,31 @@ ldr x0, [x1, w2, sxtw]
 ldr x0, [x1, w2, sxtw #3]
 // CHECK:      add x26, x1, w2, sxtw #3
 // CHECK-NEXT: ldr x0, [x27, w26, uxtw]
+
+ldp x0, x1, [sp], #8
+// CHECK: ldp x0, x1, [sp], #8
+
+ldp x0, x1, [x2], #8
+// CHECK:      add x28, x27, w2, uxtw
+// CHECK-NEXT: ldp x0, x1, [x28]
+// CHECK-NEXT: add x2, x2, #8
+
+ldp x0, x1, [x2, #8]!
+// CHECK:      add x28, x27, w2, uxtw
+// CHECK-NEXT: ldp x0, x1, [x28, #8]
+// CHECK-NEXT: add x2, x2, #8
+
+ldp x0, x1, [x2], #-8
+// CHECK:      add x28, x27, w2, uxtw
+// CHECK-NEXT: ldp x0, x1, [x28]
+// CHECK-NEXT: sub x2, x2, #8
+
+ldp x0, x1, [x2, #-8]!
+// CHECK:      add x28, x27, w2, uxtw
+// CHECK-NEXT: ldp x0, x1, [x28, #-8]
+// CHECK-NEXT: sub x2, x2, #8
+
+stp x0, x1, [x2, #-8]!
+// CHECK:      add x28, x27, w2, uxtw
+// CHECK-NEXT: stp x0, x1, [x28, #-8]
+// CHECK-NEXT: sub x2, x2, #8

From 905fd5a6eb1373125310a9120a40a47e3044ef72 Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Thu, 21 Aug 2025 11:01:01 -0700
Subject: [PATCH 12/23] [LFI] [AArch64] Assertion fixes for MRS/MSR

---
 .../Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
index f3de76761ecbe..b74dcccc8279c 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
@@ -250,7 +250,7 @@ void AArch64::AArch64MCLFIExpander::expandLoadStoreBasic(const MCInst &Inst, Mem
   emitAddMask(LFIAddrReg, Inst.getOperand(MII.BaseRegIdx).getReg(), Out, STI);
 
   if (MII.IsPrePost) {
-    assert(OffsetIdx != -1 && "Pre/Post must have valid OffsetIdx");
+    assert(MII.OffsetIdx != -1 && "Pre/Post must have valid OffsetIdx");
 
     emitSafeLoadStoreDemoted(Inst, MII.BaseRegIdx, Out, STI);
     MCRegister Base = Inst.getOperand(MII.BaseRegIdx).getReg();
@@ -354,8 +354,7 @@ void AArch64::AArch64MCLFIExpander::expandLoadStore(const MCInst &Inst,
   if (!MII.has_value()) {
     MII = getStoreInfo(Inst);
     if (!MII.has_value())
-      return Out.getContext().reportError(
-          Inst.getLoc(), "this load/store is not supported by LFI");
+      return Out.emitInstruction(Inst, STI);
   }
 
   // Stack accesses without a register offset don't need rewriting.
@@ -427,12 +426,12 @@ static bool isSyscall(const MCInst &Inst) {
 
 static bool isTLSRead(const MCInst &Inst) {
   return Inst.getOpcode() == AArch64::MRS &&
-    Inst.getOperand(1).getReg() == AArch64SysReg::TPIDR_EL0;
+    Inst.getOperand(1).getImm() == AArch64SysReg::TPIDR_EL0;
 }
 
 static bool isTLSWrite(const MCInst &Inst) {
   return Inst.getOpcode() == AArch64::MSR &&
-    Inst.getOperand(0).getReg() == AArch64SysReg::TPIDR_EL0;
+    Inst.getOperand(0).getImm() == AArch64SysReg::TPIDR_EL0;
 }
 
 void AArch64::AArch64MCLFIExpander::doExpandInst(const MCInst &Inst, MCStreamer &Out,

From 2b4b00bf2104e4e61e9f12b8620c04af118009d7 Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Thu, 21 Aug 2025 11:32:59 -0700
Subject: [PATCH 13/23] [LFI] [Compiler-RT] Update CMake for aarch64_lfi
 support

---
 compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake | 2 +-
 compiler-rt/cmake/builtin-config-ix.cmake            | 2 +-
 compiler-rt/lib/builtins/CMakeLists.txt              | 1 +
 libunwind/src/UnwindRegistersRestore.S               | 5 +++++
 4 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake b/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake
index ee5be276f3df7..f19b0b7961a9e 100644
--- a/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake
+++ b/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake
@@ -1,4 +1,4 @@
-set(ARM64 aarch64)
+set(ARM64 aarch64 aarch64_lfi)
 set(ARM32 arm armhf)
 set(HEXAGON hexagon)
 set(X86 i386)
diff --git a/compiler-rt/cmake/builtin-config-ix.cmake b/compiler-rt/cmake/builtin-config-ix.cmake
index b1bde47ec8555..8e043ba24c985 100644
--- a/compiler-rt/cmake/builtin-config-ix.cmake
+++ b/compiler-rt/cmake/builtin-config-ix.cmake
@@ -58,7 +58,7 @@ else()
 endif()
 
 set(AMDGPU amdgcn)
-set(ARM64 aarch64)
+set(ARM64 aarch64 aarch64_lfi)
 set(ARM32 arm armhf armv4t armv5te armv6 armv6m armv7m armv7em armv7 armv7s armv7k armv8m.base armv8m.main armv8.1m.main)
 set(AVR avr)
 set(HEXAGON hexagon)
diff --git a/compiler-rt/lib/builtins/CMakeLists.txt b/compiler-rt/lib/builtins/CMakeLists.txt
index 19316c52d12ce..d059277e6f006 100644
--- a/compiler-rt/lib/builtins/CMakeLists.txt
+++ b/compiler-rt/lib/builtins/CMakeLists.txt
@@ -649,6 +649,7 @@ set(armv7k_SOURCES ${arm_SOURCES})
 set(arm64_SOURCES ${aarch64_SOURCES})
 set(arm64e_SOURCES ${aarch64_SOURCES})
 set(arm64_32_SOURCES ${aarch64_SOURCES})
+set(aarch64_lfi_SOURCES ${aarch64_SOURCES})
 
 # macho_embedded archs
 set(armv6m_SOURCES ${thumb1_SOURCES})
diff --git a/libunwind/src/UnwindRegistersRestore.S b/libunwind/src/UnwindRegistersRestore.S
index 1702d016c368b..bc3fff38a13dc 100644
--- a/libunwind/src/UnwindRegistersRestore.S
+++ b/libunwind/src/UnwindRegistersRestore.S
@@ -655,8 +655,13 @@ DEFINE_LIBUNWIND_FUNCTION(__libunwind_Registers_arm64_jumpto)
   ldp    x20,x21, [x0, #0x0A0]
   ldp    x22,x23, [x0, #0x0B0]
   ldp    x24,x25, [x0, #0x0C0]
+#ifdef __LFI__
+  ldp    x26,xzr, [x0, #0x0D0]
+  ldp    xzr,x29, [x0, #0x0E0]
+#else
   ldp    x26,x27, [x0, #0x0D0]
   ldp    x28,x29, [x0, #0x0E0]
+#endif
   ldr    x30,     [x0, #0x100]  // restore pc into lr
 #if defined(__ARM_FP) && __ARM_FP != 0
   ldp    d0, d1,  [x0, #0x110]

From c76e3b390206b621e62b04e08353d1e7f484062c Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Thu, 21 Aug 2025 13:01:54 -0700
Subject: [PATCH 14/23] [LFI] Add .no_expand/.expand directives

These directives make it possible to disable the LFI rewriter in GNU
assembly.
---
 llvm/include/llvm/MC/MCLFIExpander.h  |  5 +++++
 llvm/lib/MC/MCAsmStreamer.cpp         |  2 +-
 llvm/lib/MC/MCLFIExpander.cpp         | 12 ++++++++++++
 llvm/lib/MC/MCObjectStreamer.cpp      |  2 +-
 llvm/lib/MC/MCParser/LFIAsmParser.cpp | 28 ++++++++++++++++++++++++++-
 llvm/test/MC/AArch64/LFI/other.s      |  6 ++++++
 6 files changed, 52 insertions(+), 3 deletions(-)
 create mode 100644 llvm/test/MC/AArch64/LFI/other.s

diff --git a/llvm/include/llvm/MC/MCLFIExpander.h b/llvm/include/llvm/MC/MCLFIExpander.h
index 195e298c42e84..d77383c2576f5 100644
--- a/llvm/include/llvm/MC/MCLFIExpander.h
+++ b/llvm/include/llvm/MC/MCLFIExpander.h
@@ -31,6 +31,7 @@ class MCLFIExpander {
 private:
   SmallVector<MCRegister, 2> ScratchRegs;
   MCContext &Ctx;
+  bool Enabled;
 
 protected:
   std::unique_ptr<MCInstrInfo> InstInfo;
@@ -50,6 +51,10 @@ class MCLFIExpander {
   bool addScratchReg(MCRegister Reg);
   void clearScratchRegs();
 
+  void disable();
+  void enable();
+  bool isEnabled();
+
   bool isPseudo(const MCInst &Inst) const;
 
   bool mayAffectControlFlow(const MCInst &Inst) const;
diff --git a/llvm/lib/MC/MCAsmStreamer.cpp b/llvm/lib/MC/MCAsmStreamer.cpp
index 38d8976965c81..00f49fd90ca3e 100644
--- a/llvm/lib/MC/MCAsmStreamer.cpp
+++ b/llvm/lib/MC/MCAsmStreamer.cpp
@@ -2454,7 +2454,7 @@ void MCAsmStreamer::AddEncodingComment(const MCInst &Inst,
 
 void MCAsmStreamer::emitInstruction(const MCInst &Inst,
                                     const MCSubtargetInfo &STI) {
-  if (LFIExpander && LFIExpander->expandInst(Inst, *this, STI))
+  if (LFIExpander && LFIExpander->isEnabled() && LFIExpander->expandInst(Inst, *this, STI))
     return;
 
   if (MAI->isAIX() && CurFrag)
diff --git a/llvm/lib/MC/MCLFIExpander.cpp b/llvm/lib/MC/MCLFIExpander.cpp
index 900e1ec17418d..9ed2286d61ee9 100644
--- a/llvm/lib/MC/MCLFIExpander.cpp
+++ b/llvm/lib/MC/MCLFIExpander.cpp
@@ -47,6 +47,18 @@ void MCLFIExpander::clearScratchRegs() {
   ScratchRegs.clear();
 }
 
+void MCLFIExpander::disable() {
+  Enabled = false;
+}
+
+void MCLFIExpander::enable() {
+  Enabled = true;
+}
+
+bool MCLFIExpander::isEnabled() {
+  return Enabled;
+}
+
 MCRegister MCLFIExpander::getScratchReg(int index) {
   assert(index >= 0 && static_cast<unsigned>(index) < numScratchRegs());
   return ScratchRegs[numScratchRegs()  - index - 1];
diff --git a/llvm/lib/MC/MCObjectStreamer.cpp b/llvm/lib/MC/MCObjectStreamer.cpp
index 75c3f00d64ed2..cc18cccf779fd 100644
--- a/llvm/lib/MC/MCObjectStreamer.cpp
+++ b/llvm/lib/MC/MCObjectStreamer.cpp
@@ -351,7 +351,7 @@ void MCObjectStreamer::emitInstruction(const MCInst &Inst,
 void MCObjectStreamer::emitInstructionImpl(const MCInst &Inst,
                                            const MCSubtargetInfo &STI) {
   MCSection *Sec = getCurrentSectionOnly();
-  if (LFIExpander && LFIExpander->expandInst(Inst, *this, STI))
+  if (LFIExpander && LFIExpander->isEnabled() && LFIExpander->expandInst(Inst, *this, STI))
     return;
 
   MCStreamer::emitInstruction(Inst, STI);
diff --git a/llvm/lib/MC/MCParser/LFIAsmParser.cpp b/llvm/lib/MC/MCParser/LFIAsmParser.cpp
index cf9c172bd423b..f0932a0df6b6b 100644
--- a/llvm/lib/MC/MCParser/LFIAsmParser.cpp
+++ b/llvm/lib/MC/MCParser/LFIAsmParser.cpp
@@ -38,6 +38,8 @@ class LFIAsmParser : public MCAsmParserExtension {
     MCAsmParserExtension::Initialize(Parser);
     addDirectiveHandler<&LFIAsmParser::ParseScratch>(".scratch");
     addDirectiveHandler<&LFIAsmParser::ParseUnscratch>(".scratch_clear");
+    addDirectiveHandler<&LFIAsmParser::ParseExpandDisable>(".no_expand");
+    addDirectiveHandler<&LFIAsmParser::ParseExpandEnable>(".expand");
   }
 
   /// ::= {.scratch} reg
@@ -64,7 +66,7 @@ class LFIAsmParser : public MCAsmParserExtension {
     return false;
   }
 
-  /// ::= {.unscratch}
+  /// ::= {.scratch_clear}
   bool ParseUnscratch(StringRef Directive, SMLoc Loc) {
     getParser().checkForValidSection();
     if (getLexer().isNot(AsmToken::EndOfStatement))
@@ -75,6 +77,30 @@ class LFIAsmParser : public MCAsmParserExtension {
 
     return false;
   }
+
+  /// ::= {.no_expand}
+  bool ParseExpandDisable(StringRef Directive, SMLoc Loc) {
+    getParser().checkForValidSection();
+    if (getLexer().isNot(AsmToken::EndOfStatement))
+      return TokError("unexpected token in '.no_expand' directive");
+    Lex();
+
+    Expander->disable();
+
+    return false;
+  }
+
+  /// ::= {.expand}
+  bool ParseExpandEnable(StringRef Directive, SMLoc Loc) {
+    getParser().checkForValidSection();
+    if (getLexer().isNot(AsmToken::EndOfStatement))
+      return TokError("unexpected token in '.expand' directive");
+    Lex();
+
+    Expander->enable();
+
+    return false;
+  }
 };
 
 namespace llvm {
diff --git a/llvm/test/MC/AArch64/LFI/other.s b/llvm/test/MC/AArch64/LFI/other.s
new file mode 100644
index 0000000000000..c63ee1cd5d88f
--- /dev/null
+++ b/llvm/test/MC/AArch64/LFI/other.s
@@ -0,0 +1,6 @@
+// RUN: llvm-mc -filetype asm -triple aarch64_lfi %s | FileCheck %s
+
+.no_expand
+ldr x0, [x1]
+// CHECK: ldr x0, [x1]
+.expand

From 284413019bf53960846bc73700dcc9b73c4c4579 Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Thu, 21 Aug 2025 13:19:05 -0700
Subject: [PATCH 15/23] [LFI] Make sure rewriter is enabled by default

---
 llvm/include/llvm/MC/MCLFIExpander.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/include/llvm/MC/MCLFIExpander.h b/llvm/include/llvm/MC/MCLFIExpander.h
index d77383c2576f5..7ccb5963e8439 100644
--- a/llvm/include/llvm/MC/MCLFIExpander.h
+++ b/llvm/include/llvm/MC/MCLFIExpander.h
@@ -31,7 +31,7 @@ class MCLFIExpander {
 private:
   SmallVector<MCRegister, 2> ScratchRegs;
   MCContext &Ctx;
-  bool Enabled;
+  bool Enabled = true;
 
 protected:
   std::unique_ptr<MCInstrInfo> InstInfo;

From 0c514ef819a0c18a61193c6e99e73f87b20945d4 Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Thu, 21 Aug 2025 13:36:41 -0700
Subject: [PATCH 16/23] [LFI] [Compiler-RT] Disable use of LFI reserved regs

---
 compiler-rt/lib/hwasan/hwasan_interceptors.cpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/compiler-rt/lib/hwasan/hwasan_interceptors.cpp b/compiler-rt/lib/hwasan/hwasan_interceptors.cpp
index c10b5c158548e..fbe9f04db3669 100644
--- a/compiler-rt/lib/hwasan/hwasan_interceptors.cpp
+++ b/compiler-rt/lib/hwasan/hwasan_interceptors.cpp
@@ -382,7 +382,9 @@ InternalLongjmp(__hw_register_buf env, int retval) {
       "ldp	x21, x22, [%0, #2<<3];"
       "ldp	x23, x24, [%0, #4<<3];"
       "ldp	x25, x26, [%0, #6<<3];"
+#ifndef __LFI__
       "ldp	x27, x28, [%0, #8<<3];"
+#endif
       "ldp	x29, x30, [%0, #10<<3];"
       "ldp	 d8,  d9, [%0, #14<<3];"
       "ldp	d10, d11, [%0, #16<<3];"

From 91c1f4586a829936da5010081ccaf16d77d84853 Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Thu, 21 Aug 2025 14:24:56 -0700
Subject: [PATCH 17/23] [LFI] [AArch64] Delete uses of reserved regs by default

---
 .../MCTargetDesc/AArch64MCLFIExpander.cpp     | 40 ++++++++++++++-----
 llvm/test/MC/AArch64/LFI/reserved.s           | 19 ++++++---
 2 files changed, 45 insertions(+), 14 deletions(-)

diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
index b74dcccc8279c..e5e3dce2d46c4 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
@@ -22,12 +22,17 @@
 #include "llvm/MC/MCObjectFileInfo.h"
 #include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCStreamer.h"
+#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 
 using namespace llvm;
 
 #define DEBUG_TYPE "lfi"
 
+static cl::opt<bool> AArch64LFIErrorReserved("aarch64-lfi-error-reserved",
+    cl::Hidden, cl::init(false),
+    cl::desc("Produce errors for uses of LFI reserved registers"));
+
 static MCRegister LFIAddrReg = AArch64::X28;
 static MCRegister LFIBaseReg = AArch64::X27;
 static MCRegister LFIScratchReg = AArch64::X26;
@@ -167,20 +172,26 @@ bool AArch64::AArch64MCLFIExpander::mayModifyLR(const MCInst &Inst) {
   return mayModifyRegister(Inst, AArch64::LR);
 }
 
-void AArch64::AArch64MCLFIExpander::expandLRModification(
-    const MCInst &Inst, MCStreamer &Out, const MCSubtargetInfo &STI) {
-  MCRegister Scratch = getScratch();
+static MCInst replaceReg(const MCInst &Inst,
+                         MCRegister Dest, MCRegister Src) {
   MCInst New;
-  New.setLoc(Inst.getLoc());
   New.setOpcode(Inst.getOpcode());
+  New.setLoc(Inst.getLoc());
   for (unsigned I = 0; I < Inst.getNumOperands(); ++I) {
     const MCOperand &Op = Inst.getOperand(I);
-    if (Op.isReg() && Op.getReg() == AArch64::LR) {
-      New.addOperand(MCOperand::createReg(Scratch));
+    if (Op.isReg() && Op.getReg() == Src) {
+      New.addOperand(MCOperand::createReg(Dest));
     } else {
       New.addOperand(Op);
     }
   }
+  return New;
+}
+
+void AArch64::AArch64MCLFIExpander::expandLRModification(
+    const MCInst &Inst, MCStreamer &Out, const MCSubtargetInfo &STI) {
+  MCRegister Scratch = getScratch();
+  MCInst New = replaceReg(Inst, Scratch, AArch64::LR);
   if (mayLoad(New) || mayStore(New))
     expandLoadStore(New, Out, STI);
   else
@@ -459,9 +470,20 @@ void AArch64::AArch64MCLFIExpander::doExpandInst(const MCInst &Inst, MCStreamer
 
   // Bail out with an error. In the future, we could consider automatically
   // rewriting uses of reserved LFI registers.
-  if (mayModifyReserved(Inst))
-    return Out.getContext().reportError(
-        Inst.getLoc(), "illegal modification of reserved LFI register");
+  if (mayModifyReserved(Inst)) {
+    if (AArch64LFIErrorReserved)
+      return Out.getContext().reportError(
+          Inst.getLoc(), "illegal modification of reserved LFI register");
+    Out.getContext().reportWarning(
+        Inst.getLoc(), "deleting modification of reserved LFI register");
+    MCInst New = replaceReg(Inst, AArch64::XZR, LFIBaseReg);
+    if (mayModifyReserved(New)) {
+      MCRegister Scratch = getScratch();
+      New = replaceReg(New, Scratch, LFIAddrReg);
+    }
+    assert(!mayModifyReserved(New));
+    return doExpandInst(New, Out, STI);
+  }
 
   if (mayModifyStack(Inst))
     return expandStackModification(Inst, Out, STI);
diff --git a/llvm/test/MC/AArch64/LFI/reserved.s b/llvm/test/MC/AArch64/LFI/reserved.s
index 7b9ffb83474c0..e387ec5ec21e2 100644
--- a/llvm/test/MC/AArch64/LFI/reserved.s
+++ b/llvm/test/MC/AArch64/LFI/reserved.s
@@ -1,12 +1,21 @@
-// RUN: not llvm-mc -filetype asm -triple aarch64_lfi %s 2> %t
+// RUN: llvm-mc -filetype asm -triple aarch64_lfi %s 2> %t
 // RUN: FileCheck --check-prefix=CHECK-ERROR %s <%t
 
 mov x28, x0
-// CHECK-ERROR: error: illegal modification of reserved LFI register
+// CHECK:       mov x26, x0
+// CHECK-ERROR: warning: deleting modification of reserved LFI register
 // CHECK-ERROR:        mov x28, x0
 // CHECK-ERROR:        ^
 
-ldr x27, [sp]
-// CHECK-ERROR: error: illegal modification of reserved LFI register
-// CHECK-ERROR:        ldr x27, [sp]
+ldr x27, [x0]
+// CHECK:       ldr xzr, [x27, w0, uxtw]
+// CHECK-ERROR: warning: deleting modification of reserved LFI register
+// CHECK-ERROR:        ldr x27, [x0]
+// CHECK-ERROR:        ^
+
+ldp x27, x28, [x0]
+// CHECK:       add x28, x27, w0, uxtw
+// CHECK-NEXT:  ldp xzr, x26, [x28]
+// CHECK-ERROR: warning: deleting modification of reserved LFI register
+// CHECK-ERROR:        ldp x27, x28, [x0]
 // CHECK-ERROR:        ^

From 455ddb425600f610747423602270b22d30162e64 Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Thu, 21 Aug 2025 15:06:00 -0700
Subject: [PATCH 18/23] [LFI] Switch to +lfi-stores and +lfi-jumps feature

Removes the aarch64_lfi_stores and aarch64_lfi_jumps target triples and
replaces them with +lfi-stores and +lfi-jumps subtarget features for
AArch64.
---
 llvm/include/llvm/TargetParser/Triple.h       | 24 +------------------
 llvm/lib/Target/AArch64/AArch64Features.td    | 10 ++++++++
 llvm/lib/Target/AArch64/AArch64Subtarget.h    |  2 ++
 .../MCTargetDesc/AArch64MCLFIExpander.cpp     | 17 +++++++++++++
 llvm/lib/TargetParser/Triple.cpp              | 12 ----------
 5 files changed, 30 insertions(+), 35 deletions(-)

diff --git a/llvm/include/llvm/TargetParser/Triple.h b/llvm/include/llvm/TargetParser/Triple.h
index 642c9e7a04436..7806b780fc5f1 100644
--- a/llvm/include/llvm/TargetParser/Triple.h
+++ b/llvm/include/llvm/TargetParser/Triple.h
@@ -149,8 +149,6 @@ class Triple {
     AArch64SubArch_arm64ec,
 
     AArch64SubArch_lfi,
-    AArch64SubArch_lfi_stores,
-    AArch64SubArch_lfi_jumps,
 
     KalimbaSubArch_v3,
     KalimbaSubArch_v4,
@@ -684,32 +682,12 @@ class Triple {
     return isAArch64LFI();
   }
 
-  /// Checks if we're targeting any subarch of AArch64 LFI.
+  /// Checks if we're targeting the AArch64 LFI subarch.
   bool isAArch64LFI() const {
-    return getArch() == Triple::aarch64 &&
-           (getSubArch() == Triple::AArch64SubArch_lfi ||
-            getSubArch() == Triple::AArch64SubArch_lfi_stores ||
-            getSubArch() == Triple::AArch64SubArch_lfi_jumps);
-  }
-
-  /// Checks if we're targeting AArch64 LFI.
-  bool isAArch64LFIFull() const {
     return getArch() == Triple::aarch64 &&
            getSubArch() == Triple::AArch64SubArch_lfi;
   }
 
-  /// Checks if we're targeting AArch64 LFI stores-only.
-  bool isAArch64LFIStores() const {
-    return getArch() == Triple::aarch64 &&
-           getSubArch() == Triple::AArch64SubArch_lfi_stores;
-  }
-
-  /// Checks if we're targeting AArch64 LFI jumps-only.
-  bool isAArch64LFIJumps() const {
-    return getArch() == Triple::aarch64 &&
-           getSubArch() == Triple::AArch64SubArch_lfi_jumps;
-  }
-
   bool isWindowsCoreCLREnvironment() const {
     return isOSWindows() && getEnvironment() == Triple::CoreCLR;
   }
diff --git a/llvm/lib/Target/AArch64/AArch64Features.td b/llvm/lib/Target/AArch64/AArch64Features.td
index d47dcfe92ff19..8364ba46fa53d 100644
--- a/llvm/lib/Target/AArch64/AArch64Features.td
+++ b/llvm/lib/Target/AArch64/AArch64Features.td
@@ -948,3 +948,13 @@ def FeatureHardenSlsNoComdat : SubtargetFeature<"harden-sls-nocomdat",
 // Only intended to be used by disassemblers.
 def FeatureAll
     : SubtargetFeature<"all", "IsAll", "true", "Enable all instructions">;
+
+//===----------------------------------------------------------------------===//
+// LFI
+//===----------------------------------------------------------------------===//
+
+def FeatureLFIStores : SubtargetFeature<"lfi-stores", "LFIStores", "true",
+                                        "Isolate only store+jump instructions">;
+
+def FeatureLFIJumps : SubtargetFeature<"lfi-jumps", "LFIJumps", "true",
+                                       "Isolate only jump instructions">;
diff --git a/llvm/lib/Target/AArch64/AArch64Subtarget.h b/llvm/lib/Target/AArch64/AArch64Subtarget.h
index cbc60a2e52567..f7875ebdfb6a4 100644
--- a/llvm/lib/Target/AArch64/AArch64Subtarget.h
+++ b/llvm/lib/Target/AArch64/AArch64Subtarget.h
@@ -289,6 +289,8 @@ class AArch64Subtarget final : public AArch64GenSubtargetInfo {
   bool isTargetFuchsia() const { return TargetTriple.isOSFuchsia(); }
   bool isWindowsArm64EC() const { return TargetTriple.isWindowsArm64EC(); }
   bool isAArch64LFI() const { return TargetTriple.isAArch64LFI(); }
+  bool isLFIStores() const { return LFIStores; }
+  bool isLFIJumps() const { return LFIJumps; }
 
   bool isTargetCOFF() const { return TargetTriple.isOSBinFormatCOFF(); }
   bool isTargetELF() const { return TargetTriple.isOSBinFormatELF(); }
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
index e5e3dce2d46c4..4da8a4bbc7b68 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
@@ -22,6 +22,7 @@
 #include "llvm/MC/MCObjectFileInfo.h"
 #include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCStreamer.h"
+#include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 
@@ -37,6 +38,17 @@ static MCRegister LFIAddrReg = AArch64::X28;
 static MCRegister LFIBaseReg = AArch64::X27;
 static MCRegister LFIScratchReg = AArch64::X26;
 
+static bool featureStores(const MCSubtargetInfo &STI) {
+  const auto Stores = FeatureBitset({AArch64::FeatureLFIStores});
+  return (STI.getFeatureBits() & Stores) == Stores;
+}
+
+static bool featureJumps(const MCSubtargetInfo &STI) {
+  const auto Jumps = FeatureBitset({AArch64::FeatureLFIJumps});
+  return (STI.getFeatureBits() & Jumps) == Jumps;
+}
+
+
 bool AArch64::AArch64MCLFIExpander::isValidScratchRegister(MCRegister Reg) const {
   return Reg != AArch64::SP;
 }
@@ -361,6 +373,11 @@ void AArch64::AArch64MCLFIExpander::expandLoadStoreRoW(const MCInst &Inst, MemIn
 void AArch64::AArch64MCLFIExpander::expandLoadStore(const MCInst &Inst,
                                                     MCStreamer &Out,
                                                     const MCSubtargetInfo &STI) {
+  if (featureJumps(STI))
+    return Out.emitInstruction(Inst, STI);
+  if (featureStores(STI) && !mayStore(Inst))
+    return Out.emitInstruction(Inst, STI);
+
   auto MII = getLoadInfo(Inst);
   if (!MII.has_value()) {
     MII = getStoreInfo(Inst);
diff --git a/llvm/lib/TargetParser/Triple.cpp b/llvm/lib/TargetParser/Triple.cpp
index 6a18790ca6e83..b5d5889363801 100644
--- a/llvm/lib/TargetParser/Triple.cpp
+++ b/llvm/lib/TargetParser/Triple.cpp
@@ -114,10 +114,6 @@ StringRef Triple::getArchName(ArchType Kind, SubArchType SubArch) {
       return "arm64e";
     if (SubArch == AArch64SubArch_lfi)
       return "aarch64_lfi";
-    if (SubArch == AArch64SubArch_lfi_stores)
-      return "aarch64_lfi_stores";
-    if (SubArch == AArch64SubArch_lfi_jumps)
-      return "aarch64_lfi_jumps";
     break;
   case Triple::spirv:
     switch (SubArch) {
@@ -582,8 +578,6 @@ static Triple::ArchType parseArch(StringRef ArchName) {
           .Case("arm64e", Triple::aarch64)
           .Case("arm64ec", Triple::aarch64)
           .Case("aarch64_lfi", Triple::aarch64)
-          .Case("aarch64_lfi_stores", Triple::aarch64)
-          .Case("aarch64_lfi_jumps", Triple::aarch64)
           .Case("arm", Triple::arm)
           .Case("armeb", Triple::armeb)
           .Case("thumb", Triple::thumb)
@@ -809,12 +803,6 @@ static Triple::SubArchType parseSubArch(StringRef SubArchName) {
   if (SubArchName == "aarch64_lfi")
     return Triple::AArch64SubArch_lfi;
 
-  if (SubArchName == "aarch64_lfi_stores")
-    return Triple::AArch64SubArch_lfi_stores;
-
-  if (SubArchName == "aarch64_lfi_jumps")
-    return Triple::AArch64SubArch_lfi_jumps;
-
   if (SubArchName.starts_with("spirv"))
     return StringSwitch<Triple::SubArchType>(SubArchName)
         .EndsWith("v1.0", Triple::SPIRVSubArch_v10)

From d4933b462c5e9e8897d644097acd66954f13ebb6 Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Thu, 21 Aug 2025 17:15:37 -0700
Subject: [PATCH 19/23] [LFI] [AArch64] Add +lfi-tls-reg feature

---
 llvm/lib/Target/AArch64/AArch64Features.td    |  3 +++
 .../Target/AArch64/AArch64RegisterInfo.cpp    |  3 +++
 .../MCTargetDesc/AArch64MCLFIExpander.cpp     | 20 +++++++++----------
 llvm/test/MC/AArch64/LFI/tls-reg.s            | 13 ++++++++++++
 4 files changed, 28 insertions(+), 11 deletions(-)
 create mode 100644 llvm/test/MC/AArch64/LFI/tls-reg.s

diff --git a/llvm/lib/Target/AArch64/AArch64Features.td b/llvm/lib/Target/AArch64/AArch64Features.td
index 8364ba46fa53d..e909152b20035 100644
--- a/llvm/lib/Target/AArch64/AArch64Features.td
+++ b/llvm/lib/Target/AArch64/AArch64Features.td
@@ -958,3 +958,6 @@ def FeatureLFIStores : SubtargetFeature<"lfi-stores", "LFIStores", "true",
 
 def FeatureLFIJumps : SubtargetFeature<"lfi-jumps", "LFIJumps", "true",
                                        "Isolate only jump instructions">;
+
+def FeatureLFITLSReg : SubtargetFeature<"lfi-tls-reg", "LFITLSReg", "true",
+                                        "Reserve a register for thread-local LFI data">;
diff --git a/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp b/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
index 22f845e1fa798..a52465c07840a 100644
--- a/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
@@ -464,6 +464,9 @@ AArch64RegisterInfo::getStrictlyReservedRegs(const MachineFunction &MF) const {
     markSuperRegs(Reserved, AArch64::W28);
     markSuperRegs(Reserved, AArch64::W27);
     markSuperRegs(Reserved, AArch64::W26);
+    auto TLSReg = FeatureBitset({AArch64::FeatureLFITLSReg});
+    if ((MF.getSubtarget<AArch64Subtarget>().getFeatureBits() & TLSReg) == TLSReg)
+      markSuperRegs(Reserved, AArch64::W25);
     if (!MF.getProperties().hasProperty(MachineFunctionProperties::Property::NoVRegs)) {
       markSuperRegs(Reserved, AArch64::LR);
       markSuperRegs(Reserved, AArch64::W30);
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
index 4da8a4bbc7b68..fdb25f3f0ead0 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
@@ -37,18 +37,12 @@ static cl::opt<bool> AArch64LFIErrorReserved("aarch64-lfi-error-reserved",
 static MCRegister LFIAddrReg = AArch64::X28;
 static MCRegister LFIBaseReg = AArch64::X27;
 static MCRegister LFIScratchReg = AArch64::X26;
+static MCRegister LFITLSReg = AArch64::X25;
 
-static bool featureStores(const MCSubtargetInfo &STI) {
-  const auto Stores = FeatureBitset({AArch64::FeatureLFIStores});
-  return (STI.getFeatureBits() & Stores) == Stores;
+static bool hasFeature(const FeatureBitset Feature, const MCSubtargetInfo &STI) {
+  return (STI.getFeatureBits() & Feature) == Feature;
 }
 
-static bool featureJumps(const MCSubtargetInfo &STI) {
-  const auto Jumps = FeatureBitset({AArch64::FeatureLFIJumps});
-  return (STI.getFeatureBits() & Jumps) == Jumps;
-}
-
-
 bool AArch64::AArch64MCLFIExpander::isValidScratchRegister(MCRegister Reg) const {
   return Reg != AArch64::SP;
 }
@@ -373,9 +367,9 @@ void AArch64::AArch64MCLFIExpander::expandLoadStoreRoW(const MCInst &Inst, MemIn
 void AArch64::AArch64MCLFIExpander::expandLoadStore(const MCInst &Inst,
                                                     MCStreamer &Out,
                                                     const MCSubtargetInfo &STI) {
-  if (featureJumps(STI))
+  if (hasFeature(FeatureBitset({AArch64::FeatureLFIJumps}), STI))
     return Out.emitInstruction(Inst, STI);
-  if (featureStores(STI) && !mayStore(Inst))
+  if (hasFeature(FeatureBitset({AArch64::FeatureLFIStores}), STI) && !mayStore(Inst))
     return Out.emitInstruction(Inst, STI);
 
   auto MII = getLoadInfo(Inst);
@@ -427,6 +421,10 @@ static void emitSwap(MCRegister Reg1, MCRegister Reg2, MCStreamer &Out, const MC
 void AArch64::AArch64MCLFIExpander::expandTLSRead(const MCInst &Inst, MCStreamer &Out,
                    const MCSubtargetInfo &STI) {
   MCRegister Reg = Inst.getOperand(0).getReg();
+
+  if (hasFeature(FeatureBitset({AArch64::FeatureLFITLSReg}), STI))
+    return emit(AArch64::LDRXui, Reg, LFITLSReg, 0, Out, STI);
+
   if (Reg == AArch64::X0) {
     emitLFICall(LFITLSRead, Out, STI);
   } else {
diff --git a/llvm/test/MC/AArch64/LFI/tls-reg.s b/llvm/test/MC/AArch64/LFI/tls-reg.s
new file mode 100644
index 0000000000000..b079f8316f89e
--- /dev/null
+++ b/llvm/test/MC/AArch64/LFI/tls-reg.s
@@ -0,0 +1,13 @@
+// RUN: llvm-mc -filetype asm -triple aarch64_lfi -mattr +lfi-tls-reg %s | FileCheck %s
+
+msr tpidr_el0, x0
+// CHECK:      mov x26, x30
+// CHECK-NEXT: ldr x30, [x27, #16]
+// CHECK-NEXT: blr x30
+// CHECK-NEXT: add x30, x27, w26, uxtw
+
+mrs x0, tpidr_el0
+// CHECK:      ldr x0, [x25]
+
+mrs x1, tpidr_el0
+// CHECK:      ldr x1, [x25]

From 684253eb4e77d89b8d98677c95684d4459dfb170 Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Fri, 22 Aug 2025 15:42:14 -0700
Subject: [PATCH 20/23] [LFI] [AArch64] Handle special case pre/post index

Also changes the instruction size estimate to include an additional
guard instruction for loads/stores so that branch relaxation correctly
resizes branches even before the expander is run.
---
 llvm/lib/Target/AArch64/AArch64InstrInfo.cpp  |  17 +-
 .../MCTargetDesc/AArch64InstPrinter.cpp       | 360 ------------------
 .../MCTargetDesc/AArch64MCLFIExpander.cpp     |  20 +-
 .../Target/AArch64/Utils/AArch64BaseInfo.cpp  | 352 +++++++++++++++++
 .../Target/AArch64/Utils/AArch64BaseInfo.h    |  10 +
 llvm/test/MC/AArch64/LFI/mem.s                |  14 +
 6 files changed, 398 insertions(+), 375 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index 0f2b969fba35c..b6f5e410e32d8 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -34,6 +34,7 @@
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/RegisterScavenging.h"
 #include "llvm/CodeGen/StackMaps.h"
+#include "llvm/CodeGen/TargetOpcodes.h"
 #include "llvm/CodeGen/TargetRegisterInfo.h"
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
 #include "llvm/IR/DebugInfoMetadata.h"
@@ -106,6 +107,7 @@ unsigned AArch64InstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
   unsigned NumBytes = 0;
   const MCInstrDesc &Desc = MI.getDesc();
 
+  const auto &STI = MF->getSubtarget<AArch64Subtarget>();
   if (!MI.isBundle() && isTailCallReturnInst(MI)) {
     NumBytes = Desc.getSize() ? Desc.getSize() : 4;
 
@@ -113,7 +115,6 @@ unsigned AArch64InstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
     if (!MFI->shouldSignReturnAddress(MF))
       return NumBytes;
 
-    const auto &STI = MF->getSubtarget<AArch64Subtarget>();
     auto Method = STI.getAuthenticatedLRCheckMethod(*MF);
     NumBytes += AArch64PAuth::getCheckerSizeInBytes(Method);
     return NumBytes;
@@ -124,8 +125,10 @@ unsigned AArch64InstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
   // Specific cases handle instructions of variable sizes
   switch (Desc.getOpcode()) {
   default:
-    if (Desc.getSize())
-      return Desc.getSize();
+    if (Desc.getSize()) {
+      NumBytes = Desc.getSize();
+      break;
+    }
 
     // Anything not explicitly designated otherwise (i.e. pseudo-instructions
     // with fixed constant size but not specified in .td file) is a normal
@@ -174,6 +177,14 @@ unsigned AArch64InstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
     break;
   }
 
+  if (STI.getTargetTriple().isAArch64LFI()) {
+    // Loads and stores are frequent and may be expanded to include an
+    // additional guard instruction, so we overestimate the size here to allow
+    // things like branch relaxation to be more accurate.
+    if (Desc.mayLoad() || Desc.mayStore())
+      NumBytes += 4;
+  }
+
   return NumBytes;
 }
 
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.cpp
index 875b505549f0a..e6dd82fcea387 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.cpp
@@ -420,366 +420,6 @@ static bool isTblTbxInstruction(unsigned Opcode, StringRef &Layout,
   }
 }
 
-struct LdStNInstrDesc {
-  unsigned Opcode;
-  const char *Mnemonic;
-  const char *Layout;
-  int ListOperand;
-  bool HasLane;
-  int NaturalOffset;
-};
-
-static const LdStNInstrDesc LdStNInstInfo[] = {
-  { AArch64::LD1i8,             "ld1",  ".b",     1, true,  0  },
-  { AArch64::LD1i16,            "ld1",  ".h",     1, true,  0  },
-  { AArch64::LD1i32,            "ld1",  ".s",     1, true,  0  },
-  { AArch64::LD1i64,            "ld1",  ".d",     1, true,  0  },
-  { AArch64::LD1i8_POST,        "ld1",  ".b",     2, true,  1  },
-  { AArch64::LD1i16_POST,       "ld1",  ".h",     2, true,  2  },
-  { AArch64::LD1i32_POST,       "ld1",  ".s",     2, true,  4  },
-  { AArch64::LD1i64_POST,       "ld1",  ".d",     2, true,  8  },
-  { AArch64::LD1Rv16b,          "ld1r", ".16b",   0, false, 0  },
-  { AArch64::LD1Rv8h,           "ld1r", ".8h",    0, false, 0  },
-  { AArch64::LD1Rv4s,           "ld1r", ".4s",    0, false, 0  },
-  { AArch64::LD1Rv2d,           "ld1r", ".2d",    0, false, 0  },
-  { AArch64::LD1Rv8b,           "ld1r", ".8b",    0, false, 0  },
-  { AArch64::LD1Rv4h,           "ld1r", ".4h",    0, false, 0  },
-  { AArch64::LD1Rv2s,           "ld1r", ".2s",    0, false, 0  },
-  { AArch64::LD1Rv1d,           "ld1r", ".1d",    0, false, 0  },
-  { AArch64::LD1Rv16b_POST,     "ld1r", ".16b",   1, false, 1  },
-  { AArch64::LD1Rv8h_POST,      "ld1r", ".8h",    1, false, 2  },
-  { AArch64::LD1Rv4s_POST,      "ld1r", ".4s",    1, false, 4  },
-  { AArch64::LD1Rv2d_POST,      "ld1r", ".2d",    1, false, 8  },
-  { AArch64::LD1Rv8b_POST,      "ld1r", ".8b",    1, false, 1  },
-  { AArch64::LD1Rv4h_POST,      "ld1r", ".4h",    1, false, 2  },
-  { AArch64::LD1Rv2s_POST,      "ld1r", ".2s",    1, false, 4  },
-  { AArch64::LD1Rv1d_POST,      "ld1r", ".1d",    1, false, 8  },
-  { AArch64::LD1Onev16b,        "ld1",  ".16b",   0, false, 0  },
-  { AArch64::LD1Onev8h,         "ld1",  ".8h",    0, false, 0  },
-  { AArch64::LD1Onev4s,         "ld1",  ".4s",    0, false, 0  },
-  { AArch64::LD1Onev2d,         "ld1",  ".2d",    0, false, 0  },
-  { AArch64::LD1Onev8b,         "ld1",  ".8b",    0, false, 0  },
-  { AArch64::LD1Onev4h,         "ld1",  ".4h",    0, false, 0  },
-  { AArch64::LD1Onev2s,         "ld1",  ".2s",    0, false, 0  },
-  { AArch64::LD1Onev1d,         "ld1",  ".1d",    0, false, 0  },
-  { AArch64::LD1Onev16b_POST,   "ld1",  ".16b",   1, false, 16 },
-  { AArch64::LD1Onev8h_POST,    "ld1",  ".8h",    1, false, 16 },
-  { AArch64::LD1Onev4s_POST,    "ld1",  ".4s",    1, false, 16 },
-  { AArch64::LD1Onev2d_POST,    "ld1",  ".2d",    1, false, 16 },
-  { AArch64::LD1Onev8b_POST,    "ld1",  ".8b",    1, false, 8  },
-  { AArch64::LD1Onev4h_POST,    "ld1",  ".4h",    1, false, 8  },
-  { AArch64::LD1Onev2s_POST,    "ld1",  ".2s",    1, false, 8  },
-  { AArch64::LD1Onev1d_POST,    "ld1",  ".1d",    1, false, 8  },
-  { AArch64::LD1Twov16b,        "ld1",  ".16b",   0, false, 0  },
-  { AArch64::LD1Twov8h,         "ld1",  ".8h",    0, false, 0  },
-  { AArch64::LD1Twov4s,         "ld1",  ".4s",    0, false, 0  },
-  { AArch64::LD1Twov2d,         "ld1",  ".2d",    0, false, 0  },
-  { AArch64::LD1Twov8b,         "ld1",  ".8b",    0, false, 0  },
-  { AArch64::LD1Twov4h,         "ld1",  ".4h",    0, false, 0  },
-  { AArch64::LD1Twov2s,         "ld1",  ".2s",    0, false, 0  },
-  { AArch64::LD1Twov1d,         "ld1",  ".1d",    0, false, 0  },
-  { AArch64::LD1Twov16b_POST,   "ld1",  ".16b",   1, false, 32 },
-  { AArch64::LD1Twov8h_POST,    "ld1",  ".8h",    1, false, 32 },
-  { AArch64::LD1Twov4s_POST,    "ld1",  ".4s",    1, false, 32 },
-  { AArch64::LD1Twov2d_POST,    "ld1",  ".2d",    1, false, 32 },
-  { AArch64::LD1Twov8b_POST,    "ld1",  ".8b",    1, false, 16 },
-  { AArch64::LD1Twov4h_POST,    "ld1",  ".4h",    1, false, 16 },
-  { AArch64::LD1Twov2s_POST,    "ld1",  ".2s",    1, false, 16 },
-  { AArch64::LD1Twov1d_POST,    "ld1",  ".1d",    1, false, 16 },
-  { AArch64::LD1Threev16b,      "ld1",  ".16b",   0, false, 0  },
-  { AArch64::LD1Threev8h,       "ld1",  ".8h",    0, false, 0  },
-  { AArch64::LD1Threev4s,       "ld1",  ".4s",    0, false, 0  },
-  { AArch64::LD1Threev2d,       "ld1",  ".2d",    0, false, 0  },
-  { AArch64::LD1Threev8b,       "ld1",  ".8b",    0, false, 0  },
-  { AArch64::LD1Threev4h,       "ld1",  ".4h",    0, false, 0  },
-  { AArch64::LD1Threev2s,       "ld1",  ".2s",    0, false, 0  },
-  { AArch64::LD1Threev1d,       "ld1",  ".1d",    0, false, 0  },
-  { AArch64::LD1Threev16b_POST, "ld1",  ".16b",   1, false, 48 },
-  { AArch64::LD1Threev8h_POST,  "ld1",  ".8h",    1, false, 48 },
-  { AArch64::LD1Threev4s_POST,  "ld1",  ".4s",    1, false, 48 },
-  { AArch64::LD1Threev2d_POST,  "ld1",  ".2d",    1, false, 48 },
-  { AArch64::LD1Threev8b_POST,  "ld1",  ".8b",    1, false, 24 },
-  { AArch64::LD1Threev4h_POST,  "ld1",  ".4h",    1, false, 24 },
-  { AArch64::LD1Threev2s_POST,  "ld1",  ".2s",    1, false, 24 },
-  { AArch64::LD1Threev1d_POST,  "ld1",  ".1d",    1, false, 24 },
-  { AArch64::LD1Fourv16b,       "ld1",  ".16b",   0, false, 0  },
-  { AArch64::LD1Fourv8h,        "ld1",  ".8h",    0, false, 0  },
-  { AArch64::LD1Fourv4s,        "ld1",  ".4s",    0, false, 0  },
-  { AArch64::LD1Fourv2d,        "ld1",  ".2d",    0, false, 0  },
-  { AArch64::LD1Fourv8b,        "ld1",  ".8b",    0, false, 0  },
-  { AArch64::LD1Fourv4h,        "ld1",  ".4h",    0, false, 0  },
-  { AArch64::LD1Fourv2s,        "ld1",  ".2s",    0, false, 0  },
-  { AArch64::LD1Fourv1d,        "ld1",  ".1d",    0, false, 0  },
-  { AArch64::LD1Fourv16b_POST,  "ld1",  ".16b",   1, false, 64 },
-  { AArch64::LD1Fourv8h_POST,   "ld1",  ".8h",    1, false, 64 },
-  { AArch64::LD1Fourv4s_POST,   "ld1",  ".4s",    1, false, 64 },
-  { AArch64::LD1Fourv2d_POST,   "ld1",  ".2d",    1, false, 64 },
-  { AArch64::LD1Fourv8b_POST,   "ld1",  ".8b",    1, false, 32 },
-  { AArch64::LD1Fourv4h_POST,   "ld1",  ".4h",    1, false, 32 },
-  { AArch64::LD1Fourv2s_POST,   "ld1",  ".2s",    1, false, 32 },
-  { AArch64::LD1Fourv1d_POST,   "ld1",  ".1d",    1, false, 32 },
-  { AArch64::LD2i8,             "ld2",  ".b",     1, true,  0  },
-  { AArch64::LD2i16,            "ld2",  ".h",     1, true,  0  },
-  { AArch64::LD2i32,            "ld2",  ".s",     1, true,  0  },
-  { AArch64::LD2i64,            "ld2",  ".d",     1, true,  0  },
-  { AArch64::LD2i8_POST,        "ld2",  ".b",     2, true,  2  },
-  { AArch64::LD2i16_POST,       "ld2",  ".h",     2, true,  4  },
-  { AArch64::LD2i32_POST,       "ld2",  ".s",     2, true,  8  },
-  { AArch64::LD2i64_POST,       "ld2",  ".d",     2, true,  16  },
-  { AArch64::LD2Rv16b,          "ld2r", ".16b",   0, false, 0  },
-  { AArch64::LD2Rv8h,           "ld2r", ".8h",    0, false, 0  },
-  { AArch64::LD2Rv4s,           "ld2r", ".4s",    0, false, 0  },
-  { AArch64::LD2Rv2d,           "ld2r", ".2d",    0, false, 0  },
-  { AArch64::LD2Rv8b,           "ld2r", ".8b",    0, false, 0  },
-  { AArch64::LD2Rv4h,           "ld2r", ".4h",    0, false, 0  },
-  { AArch64::LD2Rv2s,           "ld2r", ".2s",    0, false, 0  },
-  { AArch64::LD2Rv1d,           "ld2r", ".1d",    0, false, 0  },
-  { AArch64::LD2Rv16b_POST,     "ld2r", ".16b",   1, false, 2  },
-  { AArch64::LD2Rv8h_POST,      "ld2r", ".8h",    1, false, 4  },
-  { AArch64::LD2Rv4s_POST,      "ld2r", ".4s",    1, false, 8  },
-  { AArch64::LD2Rv2d_POST,      "ld2r", ".2d",    1, false, 16 },
-  { AArch64::LD2Rv8b_POST,      "ld2r", ".8b",    1, false, 2  },
-  { AArch64::LD2Rv4h_POST,      "ld2r", ".4h",    1, false, 4  },
-  { AArch64::LD2Rv2s_POST,      "ld2r", ".2s",    1, false, 8  },
-  { AArch64::LD2Rv1d_POST,      "ld2r", ".1d",    1, false, 16 },
-  { AArch64::LD2Twov16b,        "ld2",  ".16b",   0, false, 0  },
-  { AArch64::LD2Twov8h,         "ld2",  ".8h",    0, false, 0  },
-  { AArch64::LD2Twov4s,         "ld2",  ".4s",    0, false, 0  },
-  { AArch64::LD2Twov2d,         "ld2",  ".2d",    0, false, 0  },
-  { AArch64::LD2Twov8b,         "ld2",  ".8b",    0, false, 0  },
-  { AArch64::LD2Twov4h,         "ld2",  ".4h",    0, false, 0  },
-  { AArch64::LD2Twov2s,         "ld2",  ".2s",    0, false, 0  },
-  { AArch64::LD2Twov16b_POST,   "ld2",  ".16b",   1, false, 32 },
-  { AArch64::LD2Twov8h_POST,    "ld2",  ".8h",    1, false, 32 },
-  { AArch64::LD2Twov4s_POST,    "ld2",  ".4s",    1, false, 32 },
-  { AArch64::LD2Twov2d_POST,    "ld2",  ".2d",    1, false, 32 },
-  { AArch64::LD2Twov8b_POST,    "ld2",  ".8b",    1, false, 16 },
-  { AArch64::LD2Twov4h_POST,    "ld2",  ".4h",    1, false, 16 },
-  { AArch64::LD2Twov2s_POST,    "ld2",  ".2s",    1, false, 16 },
-  { AArch64::LD3i8,             "ld3",  ".b",     1, true,  0  },
-  { AArch64::LD3i16,            "ld3",  ".h",     1, true,  0  },
-  { AArch64::LD3i32,            "ld3",  ".s",     1, true,  0  },
-  { AArch64::LD3i64,            "ld3",  ".d",     1, true,  0  },
-  { AArch64::LD3i8_POST,        "ld3",  ".b",     2, true,  3  },
-  { AArch64::LD3i16_POST,       "ld3",  ".h",     2, true,  6  },
-  { AArch64::LD3i32_POST,       "ld3",  ".s",     2, true,  12 },
-  { AArch64::LD3i64_POST,       "ld3",  ".d",     2, true,  24 },
-  { AArch64::LD3Rv16b,          "ld3r", ".16b",   0, false, 0  },
-  { AArch64::LD3Rv8h,           "ld3r", ".8h",    0, false, 0  },
-  { AArch64::LD3Rv4s,           "ld3r", ".4s",    0, false, 0  },
-  { AArch64::LD3Rv2d,           "ld3r", ".2d",    0, false, 0  },
-  { AArch64::LD3Rv8b,           "ld3r", ".8b",    0, false, 0  },
-  { AArch64::LD3Rv4h,           "ld3r", ".4h",    0, false, 0  },
-  { AArch64::LD3Rv2s,           "ld3r", ".2s",    0, false, 0  },
-  { AArch64::LD3Rv1d,           "ld3r", ".1d",    0, false, 0  },
-  { AArch64::LD3Rv16b_POST,     "ld3r", ".16b",   1, false, 3  },
-  { AArch64::LD3Rv8h_POST,      "ld3r", ".8h",    1, false, 6  },
-  { AArch64::LD3Rv4s_POST,      "ld3r", ".4s",    1, false, 12 },
-  { AArch64::LD3Rv2d_POST,      "ld3r", ".2d",    1, false, 24 },
-  { AArch64::LD3Rv8b_POST,      "ld3r", ".8b",    1, false, 3  },
-  { AArch64::LD3Rv4h_POST,      "ld3r", ".4h",    1, false, 6  },
-  { AArch64::LD3Rv2s_POST,      "ld3r", ".2s",    1, false, 12 },
-  { AArch64::LD3Rv1d_POST,      "ld3r", ".1d",    1, false, 24 },
-  { AArch64::LD3Threev16b,      "ld3",  ".16b",   0, false, 0  },
-  { AArch64::LD3Threev8h,       "ld3",  ".8h",    0, false, 0  },
-  { AArch64::LD3Threev4s,       "ld3",  ".4s",    0, false, 0  },
-  { AArch64::LD3Threev2d,       "ld3",  ".2d",    0, false, 0  },
-  { AArch64::LD3Threev8b,       "ld3",  ".8b",    0, false, 0  },
-  { AArch64::LD3Threev4h,       "ld3",  ".4h",    0, false, 0  },
-  { AArch64::LD3Threev2s,       "ld3",  ".2s",    0, false, 0  },
-  { AArch64::LD3Threev16b_POST, "ld3",  ".16b",   1, false, 48 },
-  { AArch64::LD3Threev8h_POST,  "ld3",  ".8h",    1, false, 48 },
-  { AArch64::LD3Threev4s_POST,  "ld3",  ".4s",    1, false, 48 },
-  { AArch64::LD3Threev2d_POST,  "ld3",  ".2d",    1, false, 48 },
-  { AArch64::LD3Threev8b_POST,  "ld3",  ".8b",    1, false, 24 },
-  { AArch64::LD3Threev4h_POST,  "ld3",  ".4h",    1, false, 24 },
-  { AArch64::LD3Threev2s_POST,  "ld3",  ".2s",    1, false, 24 },
-  { AArch64::LD4i8,             "ld4",  ".b",     1, true,  0  },
-  { AArch64::LD4i16,            "ld4",  ".h",     1, true,  0  },
-  { AArch64::LD4i32,            "ld4",  ".s",     1, true,  0  },
-  { AArch64::LD4i64,            "ld4",  ".d",     1, true,  0  },
-  { AArch64::LD4i8_POST,        "ld4",  ".b",     2, true,  4  },
-  { AArch64::LD4i16_POST,       "ld4",  ".h",     2, true,  8  },
-  { AArch64::LD4i32_POST,       "ld4",  ".s",     2, true,  16 },
-  { AArch64::LD4i64_POST,       "ld4",  ".d",     2, true,  32 },
-  { AArch64::LD4Rv16b,          "ld4r", ".16b",   0, false, 0  },
-  { AArch64::LD4Rv8h,           "ld4r", ".8h",    0, false, 0  },
-  { AArch64::LD4Rv4s,           "ld4r", ".4s",    0, false, 0  },
-  { AArch64::LD4Rv2d,           "ld4r", ".2d",    0, false, 0  },
-  { AArch64::LD4Rv8b,           "ld4r", ".8b",    0, false, 0  },
-  { AArch64::LD4Rv4h,           "ld4r", ".4h",    0, false, 0  },
-  { AArch64::LD4Rv2s,           "ld4r", ".2s",    0, false, 0  },
-  { AArch64::LD4Rv1d,           "ld4r", ".1d",    0, false, 0  },
-  { AArch64::LD4Rv16b_POST,     "ld4r", ".16b",   1, false, 4  },
-  { AArch64::LD4Rv8h_POST,      "ld4r", ".8h",    1, false, 8  },
-  { AArch64::LD4Rv4s_POST,      "ld4r", ".4s",    1, false, 16 },
-  { AArch64::LD4Rv2d_POST,      "ld4r", ".2d",    1, false, 32 },
-  { AArch64::LD4Rv8b_POST,      "ld4r", ".8b",    1, false, 4  },
-  { AArch64::LD4Rv4h_POST,      "ld4r", ".4h",    1, false, 8  },
-  { AArch64::LD4Rv2s_POST,      "ld4r", ".2s",    1, false, 16 },
-  { AArch64::LD4Rv1d_POST,      "ld4r", ".1d",    1, false, 32 },
-  { AArch64::LD4Fourv16b,       "ld4",  ".16b",   0, false, 0  },
-  { AArch64::LD4Fourv8h,        "ld4",  ".8h",    0, false, 0  },
-  { AArch64::LD4Fourv4s,        "ld4",  ".4s",    0, false, 0  },
-  { AArch64::LD4Fourv2d,        "ld4",  ".2d",    0, false, 0  },
-  { AArch64::LD4Fourv8b,        "ld4",  ".8b",    0, false, 0  },
-  { AArch64::LD4Fourv4h,        "ld4",  ".4h",    0, false, 0  },
-  { AArch64::LD4Fourv2s,        "ld4",  ".2s",    0, false, 0  },
-  { AArch64::LD4Fourv16b_POST,  "ld4",  ".16b",   1, false, 64 },
-  { AArch64::LD4Fourv8h_POST,   "ld4",  ".8h",    1, false, 64 },
-  { AArch64::LD4Fourv4s_POST,   "ld4",  ".4s",    1, false, 64 },
-  { AArch64::LD4Fourv2d_POST,   "ld4",  ".2d",    1, false, 64 },
-  { AArch64::LD4Fourv8b_POST,   "ld4",  ".8b",    1, false, 32 },
-  { AArch64::LD4Fourv4h_POST,   "ld4",  ".4h",    1, false, 32 },
-  { AArch64::LD4Fourv2s_POST,   "ld4",  ".2s",    1, false, 32 },
-  { AArch64::ST1i8,             "st1",  ".b",     0, true,  0  },
-  { AArch64::ST1i16,            "st1",  ".h",     0, true,  0  },
-  { AArch64::ST1i32,            "st1",  ".s",     0, true,  0  },
-  { AArch64::ST1i64,            "st1",  ".d",     0, true,  0  },
-  { AArch64::ST1i8_POST,        "st1",  ".b",     1, true,  1  },
-  { AArch64::ST1i16_POST,       "st1",  ".h",     1, true,  2  },
-  { AArch64::ST1i32_POST,       "st1",  ".s",     1, true,  4  },
-  { AArch64::ST1i64_POST,       "st1",  ".d",     1, true,  8  },
-  { AArch64::ST1Onev16b,        "st1",  ".16b",   0, false, 0  },
-  { AArch64::ST1Onev8h,         "st1",  ".8h",    0, false, 0  },
-  { AArch64::ST1Onev4s,         "st1",  ".4s",    0, false, 0  },
-  { AArch64::ST1Onev2d,         "st1",  ".2d",    0, false, 0  },
-  { AArch64::ST1Onev8b,         "st1",  ".8b",    0, false, 0  },
-  { AArch64::ST1Onev4h,         "st1",  ".4h",    0, false, 0  },
-  { AArch64::ST1Onev2s,         "st1",  ".2s",    0, false, 0  },
-  { AArch64::ST1Onev1d,         "st1",  ".1d",    0, false, 0  },
-  { AArch64::ST1Onev16b_POST,   "st1",  ".16b",   1, false, 16 },
-  { AArch64::ST1Onev8h_POST,    "st1",  ".8h",    1, false, 16 },
-  { AArch64::ST1Onev4s_POST,    "st1",  ".4s",    1, false, 16 },
-  { AArch64::ST1Onev2d_POST,    "st1",  ".2d",    1, false, 16 },
-  { AArch64::ST1Onev8b_POST,    "st1",  ".8b",    1, false, 8  },
-  { AArch64::ST1Onev4h_POST,    "st1",  ".4h",    1, false, 8  },
-  { AArch64::ST1Onev2s_POST,    "st1",  ".2s",    1, false, 8  },
-  { AArch64::ST1Onev1d_POST,    "st1",  ".1d",    1, false, 8  },
-  { AArch64::ST1Twov16b,        "st1",  ".16b",   0, false, 0  },
-  { AArch64::ST1Twov8h,         "st1",  ".8h",    0, false, 0  },
-  { AArch64::ST1Twov4s,         "st1",  ".4s",    0, false, 0  },
-  { AArch64::ST1Twov2d,         "st1",  ".2d",    0, false, 0  },
-  { AArch64::ST1Twov8b,         "st1",  ".8b",    0, false, 0  },
-  { AArch64::ST1Twov4h,         "st1",  ".4h",    0, false, 0  },
-  { AArch64::ST1Twov2s,         "st1",  ".2s",    0, false, 0  },
-  { AArch64::ST1Twov1d,         "st1",  ".1d",    0, false, 0  },
-  { AArch64::ST1Twov16b_POST,   "st1",  ".16b",   1, false, 32 },
-  { AArch64::ST1Twov8h_POST,    "st1",  ".8h",    1, false, 32 },
-  { AArch64::ST1Twov4s_POST,    "st1",  ".4s",    1, false, 32 },
-  { AArch64::ST1Twov2d_POST,    "st1",  ".2d",    1, false, 32 },
-  { AArch64::ST1Twov8b_POST,    "st1",  ".8b",    1, false, 16 },
-  { AArch64::ST1Twov4h_POST,    "st1",  ".4h",    1, false, 16 },
-  { AArch64::ST1Twov2s_POST,    "st1",  ".2s",    1, false, 16 },
-  { AArch64::ST1Twov1d_POST,    "st1",  ".1d",    1, false, 16 },
-  { AArch64::ST1Threev16b,      "st1",  ".16b",   0, false, 0  },
-  { AArch64::ST1Threev8h,       "st1",  ".8h",    0, false, 0  },
-  { AArch64::ST1Threev4s,       "st1",  ".4s",    0, false, 0  },
-  { AArch64::ST1Threev2d,       "st1",  ".2d",    0, false, 0  },
-  { AArch64::ST1Threev8b,       "st1",  ".8b",    0, false, 0  },
-  { AArch64::ST1Threev4h,       "st1",  ".4h",    0, false, 0  },
-  { AArch64::ST1Threev2s,       "st1",  ".2s",    0, false, 0  },
-  { AArch64::ST1Threev1d,       "st1",  ".1d",    0, false, 0  },
-  { AArch64::ST1Threev16b_POST, "st1",  ".16b",   1, false, 48 },
-  { AArch64::ST1Threev8h_POST,  "st1",  ".8h",    1, false, 48 },
-  { AArch64::ST1Threev4s_POST,  "st1",  ".4s",    1, false, 48 },
-  { AArch64::ST1Threev2d_POST,  "st1",  ".2d",    1, false, 48 },
-  { AArch64::ST1Threev8b_POST,  "st1",  ".8b",    1, false, 24 },
-  { AArch64::ST1Threev4h_POST,  "st1",  ".4h",    1, false, 24 },
-  { AArch64::ST1Threev2s_POST,  "st1",  ".2s",    1, false, 24 },
-  { AArch64::ST1Threev1d_POST,  "st1",  ".1d",    1, false, 24 },
-  { AArch64::ST1Fourv16b,       "st1",  ".16b",   0, false, 0  },
-  { AArch64::ST1Fourv8h,        "st1",  ".8h",    0, false, 0  },
-  { AArch64::ST1Fourv4s,        "st1",  ".4s",    0, false, 0  },
-  { AArch64::ST1Fourv2d,        "st1",  ".2d",    0, false, 0  },
-  { AArch64::ST1Fourv8b,        "st1",  ".8b",    0, false, 0  },
-  { AArch64::ST1Fourv4h,        "st1",  ".4h",    0, false, 0  },
-  { AArch64::ST1Fourv2s,        "st1",  ".2s",    0, false, 0  },
-  { AArch64::ST1Fourv1d,        "st1",  ".1d",    0, false, 0  },
-  { AArch64::ST1Fourv16b_POST,  "st1",  ".16b",   1, false, 64 },
-  { AArch64::ST1Fourv8h_POST,   "st1",  ".8h",    1, false, 64 },
-  { AArch64::ST1Fourv4s_POST,   "st1",  ".4s",    1, false, 64 },
-  { AArch64::ST1Fourv2d_POST,   "st1",  ".2d",    1, false, 64 },
-  { AArch64::ST1Fourv8b_POST,   "st1",  ".8b",    1, false, 32 },
-  { AArch64::ST1Fourv4h_POST,   "st1",  ".4h",    1, false, 32 },
-  { AArch64::ST1Fourv2s_POST,   "st1",  ".2s",    1, false, 32 },
-  { AArch64::ST1Fourv1d_POST,   "st1",  ".1d",    1, false, 32 },
-  { AArch64::ST2i8,             "st2",  ".b",     0, true,  0  },
-  { AArch64::ST2i16,            "st2",  ".h",     0, true,  0  },
-  { AArch64::ST2i32,            "st2",  ".s",     0, true,  0  },
-  { AArch64::ST2i64,            "st2",  ".d",     0, true,  0  },
-  { AArch64::ST2i8_POST,        "st2",  ".b",     1, true,  2  },
-  { AArch64::ST2i16_POST,       "st2",  ".h",     1, true,  4  },
-  { AArch64::ST2i32_POST,       "st2",  ".s",     1, true,  8  },
-  { AArch64::ST2i64_POST,       "st2",  ".d",     1, true,  16 },
-  { AArch64::ST2Twov16b,        "st2",  ".16b",   0, false, 0  },
-  { AArch64::ST2Twov8h,         "st2",  ".8h",    0, false, 0  },
-  { AArch64::ST2Twov4s,         "st2",  ".4s",    0, false, 0  },
-  { AArch64::ST2Twov2d,         "st2",  ".2d",    0, false, 0  },
-  { AArch64::ST2Twov8b,         "st2",  ".8b",    0, false, 0  },
-  { AArch64::ST2Twov4h,         "st2",  ".4h",    0, false, 0  },
-  { AArch64::ST2Twov2s,         "st2",  ".2s",    0, false, 0  },
-  { AArch64::ST2Twov16b_POST,   "st2",  ".16b",   1, false, 32 },
-  { AArch64::ST2Twov8h_POST,    "st2",  ".8h",    1, false, 32 },
-  { AArch64::ST2Twov4s_POST,    "st2",  ".4s",    1, false, 32 },
-  { AArch64::ST2Twov2d_POST,    "st2",  ".2d",    1, false, 32 },
-  { AArch64::ST2Twov8b_POST,    "st2",  ".8b",    1, false, 16 },
-  { AArch64::ST2Twov4h_POST,    "st2",  ".4h",    1, false, 16 },
-  { AArch64::ST2Twov2s_POST,    "st2",  ".2s",    1, false, 16 },
-  { AArch64::ST3i8,             "st3",  ".b",     0, true,  0  },
-  { AArch64::ST3i16,            "st3",  ".h",     0, true,  0  },
-  { AArch64::ST3i32,            "st3",  ".s",     0, true,  0  },
-  { AArch64::ST3i64,            "st3",  ".d",     0, true,  0  },
-  { AArch64::ST3i8_POST,        "st3",  ".b",     1, true,  3  },
-  { AArch64::ST3i16_POST,       "st3",  ".h",     1, true,  6  },
-  { AArch64::ST3i32_POST,       "st3",  ".s",     1, true,  12 },
-  { AArch64::ST3i64_POST,       "st3",  ".d",     1, true,  24 },
-  { AArch64::ST3Threev16b,      "st3",  ".16b",   0, false, 0  },
-  { AArch64::ST3Threev8h,       "st3",  ".8h",    0, false, 0  },
-  { AArch64::ST3Threev4s,       "st3",  ".4s",    0, false, 0  },
-  { AArch64::ST3Threev2d,       "st3",  ".2d",    0, false, 0  },
-  { AArch64::ST3Threev8b,       "st3",  ".8b",    0, false, 0  },
-  { AArch64::ST3Threev4h,       "st3",  ".4h",    0, false, 0  },
-  { AArch64::ST3Threev2s,       "st3",  ".2s",    0, false, 0  },
-  { AArch64::ST3Threev16b_POST, "st3",  ".16b",   1, false, 48 },
-  { AArch64::ST3Threev8h_POST,  "st3",  ".8h",    1, false, 48 },
-  { AArch64::ST3Threev4s_POST,  "st3",  ".4s",    1, false, 48 },
-  { AArch64::ST3Threev2d_POST,  "st3",  ".2d",    1, false, 48 },
-  { AArch64::ST3Threev8b_POST,  "st3",  ".8b",    1, false, 24 },
-  { AArch64::ST3Threev4h_POST,  "st3",  ".4h",    1, false, 24 },
-  { AArch64::ST3Threev2s_POST,  "st3",  ".2s",    1, false, 24 },
-  { AArch64::ST4i8,             "st4",  ".b",     0, true,  0  },
-  { AArch64::ST4i16,            "st4",  ".h",     0, true,  0  },
-  { AArch64::ST4i32,            "st4",  ".s",     0, true,  0  },
-  { AArch64::ST4i64,            "st4",  ".d",     0, true,  0  },
-  { AArch64::ST4i8_POST,        "st4",  ".b",     1, true,  4  },
-  { AArch64::ST4i16_POST,       "st4",  ".h",     1, true,  8  },
-  { AArch64::ST4i32_POST,       "st4",  ".s",     1, true,  16 },
-  { AArch64::ST4i64_POST,       "st4",  ".d",     1, true,  32 },
-  { AArch64::ST4Fourv16b,       "st4",  ".16b",   0, false, 0  },
-  { AArch64::ST4Fourv8h,        "st4",  ".8h",    0, false, 0  },
-  { AArch64::ST4Fourv4s,        "st4",  ".4s",    0, false, 0  },
-  { AArch64::ST4Fourv2d,        "st4",  ".2d",    0, false, 0  },
-  { AArch64::ST4Fourv8b,        "st4",  ".8b",    0, false, 0  },
-  { AArch64::ST4Fourv4h,        "st4",  ".4h",    0, false, 0  },
-  { AArch64::ST4Fourv2s,        "st4",  ".2s",    0, false, 0  },
-  { AArch64::ST4Fourv16b_POST,  "st4",  ".16b",   1, false, 64 },
-  { AArch64::ST4Fourv8h_POST,   "st4",  ".8h",    1, false, 64 },
-  { AArch64::ST4Fourv4s_POST,   "st4",  ".4s",    1, false, 64 },
-  { AArch64::ST4Fourv2d_POST,   "st4",  ".2d",    1, false, 64 },
-  { AArch64::ST4Fourv8b_POST,   "st4",  ".8b",    1, false, 32 },
-  { AArch64::ST4Fourv4h_POST,   "st4",  ".4h",    1, false, 32 },
-  { AArch64::ST4Fourv2s_POST,   "st4",  ".2s",    1, false, 32 },
-};
-
-static const LdStNInstrDesc *getLdStNInstrDesc(unsigned Opcode) {
-  for (const auto &Info : LdStNInstInfo)
-    if (Info.Opcode == Opcode)
-      return &Info;
-
-  return nullptr;
-}
-
 void AArch64AppleInstPrinter::printInst(const MCInst *MI, uint64_t Address,
                                         StringRef Annot,
                                         const MCSubtargetInfo &STI,
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
index fdb25f3f0ead0..58597f506e365 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
@@ -277,18 +277,14 @@ void AArch64::AArch64MCLFIExpander::expandLoadStoreBasic(const MCInst &Inst, Mem
       // it is encoded as a post-index addressing with XZR register operand.
       // e.g., LD3Threev3d_POST can only have #48 as its operand and its offset
       // MachineOperand holds XZR, which is a *Register* kind, not Imm.
-
-      // TODO: handle this case: ld3 { v0.4s, v1.4s, v2.4s }, [x0], #48
-      // MCRegister OffReg = OffsetMO.getReg();
-      // if (OffReg == AArch64::XZR) {
-      //   const LdStNInstrDesc *Info = getLdStNInstrDesc(Inst.getOpcode());
-      //   assert(Info && Info->NaturalOffset >= 0);
-      //   return emit(AArch64::ADDXri, Base, Base, Info->NaturalOffset, 0, Out, STI);
-      // } else {
-      //   assert(OffReg != AArch64::WZR);
-      //   return emit(AArch64::ADDXrs, Base, Base, OffsetMO.getReg(), 0, Out, STI);
-      // }
-      return;
+      MCRegister OffReg = OffsetMO.getReg();
+      if (OffReg == AArch64::XZR) {
+        const LdStNInstrDesc *Info = getLdStNInstrDesc(Inst.getOpcode());
+        assert(Info && Info->NaturalOffset >= 0);
+        return emit(AArch64::ADDXri, Base, Base, Info->NaturalOffset, 0, Out, STI);
+      }
+      assert(OffReg != AArch64::WZR);
+      return emit(AArch64::ADDXrs, Base, Base, OffsetMO.getReg(), 0, Out, STI);
     } else {
       auto Offset = Inst.getOperand(MII.OffsetIdx).getImm() * getPrePostScale(Inst.getOpcode());
       if (Offset >= 0)
diff --git a/llvm/lib/Target/AArch64/Utils/AArch64BaseInfo.cpp b/llvm/lib/Target/AArch64/Utils/AArch64BaseInfo.cpp
index 7767028c20bcc..756fdca32dcc6 100644
--- a/llvm/lib/Target/AArch64/Utils/AArch64BaseInfo.cpp
+++ b/llvm/lib/Target/AArch64/Utils/AArch64BaseInfo.cpp
@@ -190,4 +190,356 @@ namespace llvm {
 #define GET_SVCRsList_IMPL
 #include "AArch64GenSystemOperands.inc"
   }
+
+static const LdStNInstrDesc LdStNInstInfo[] = {
+  { AArch64::LD1i8,             "ld1",  ".b",     1, true,  0  },
+  { AArch64::LD1i16,            "ld1",  ".h",     1, true,  0  },
+  { AArch64::LD1i32,            "ld1",  ".s",     1, true,  0  },
+  { AArch64::LD1i64,            "ld1",  ".d",     1, true,  0  },
+  { AArch64::LD1i8_POST,        "ld1",  ".b",     2, true,  1  },
+  { AArch64::LD1i16_POST,       "ld1",  ".h",     2, true,  2  },
+  { AArch64::LD1i32_POST,       "ld1",  ".s",     2, true,  4  },
+  { AArch64::LD1i64_POST,       "ld1",  ".d",     2, true,  8  },
+  { AArch64::LD1Rv16b,          "ld1r", ".16b",   0, false, 0  },
+  { AArch64::LD1Rv8h,           "ld1r", ".8h",    0, false, 0  },
+  { AArch64::LD1Rv4s,           "ld1r", ".4s",    0, false, 0  },
+  { AArch64::LD1Rv2d,           "ld1r", ".2d",    0, false, 0  },
+  { AArch64::LD1Rv8b,           "ld1r", ".8b",    0, false, 0  },
+  { AArch64::LD1Rv4h,           "ld1r", ".4h",    0, false, 0  },
+  { AArch64::LD1Rv2s,           "ld1r", ".2s",    0, false, 0  },
+  { AArch64::LD1Rv1d,           "ld1r", ".1d",    0, false, 0  },
+  { AArch64::LD1Rv16b_POST,     "ld1r", ".16b",   1, false, 1  },
+  { AArch64::LD1Rv8h_POST,      "ld1r", ".8h",    1, false, 2  },
+  { AArch64::LD1Rv4s_POST,      "ld1r", ".4s",    1, false, 4  },
+  { AArch64::LD1Rv2d_POST,      "ld1r", ".2d",    1, false, 8  },
+  { AArch64::LD1Rv8b_POST,      "ld1r", ".8b",    1, false, 1  },
+  { AArch64::LD1Rv4h_POST,      "ld1r", ".4h",    1, false, 2  },
+  { AArch64::LD1Rv2s_POST,      "ld1r", ".2s",    1, false, 4  },
+  { AArch64::LD1Rv1d_POST,      "ld1r", ".1d",    1, false, 8  },
+  { AArch64::LD1Onev16b,        "ld1",  ".16b",   0, false, 0  },
+  { AArch64::LD1Onev8h,         "ld1",  ".8h",    0, false, 0  },
+  { AArch64::LD1Onev4s,         "ld1",  ".4s",    0, false, 0  },
+  { AArch64::LD1Onev2d,         "ld1",  ".2d",    0, false, 0  },
+  { AArch64::LD1Onev8b,         "ld1",  ".8b",    0, false, 0  },
+  { AArch64::LD1Onev4h,         "ld1",  ".4h",    0, false, 0  },
+  { AArch64::LD1Onev2s,         "ld1",  ".2s",    0, false, 0  },
+  { AArch64::LD1Onev1d,         "ld1",  ".1d",    0, false, 0  },
+  { AArch64::LD1Onev16b_POST,   "ld1",  ".16b",   1, false, 16 },
+  { AArch64::LD1Onev8h_POST,    "ld1",  ".8h",    1, false, 16 },
+  { AArch64::LD1Onev4s_POST,    "ld1",  ".4s",    1, false, 16 },
+  { AArch64::LD1Onev2d_POST,    "ld1",  ".2d",    1, false, 16 },
+  { AArch64::LD1Onev8b_POST,    "ld1",  ".8b",    1, false, 8  },
+  { AArch64::LD1Onev4h_POST,    "ld1",  ".4h",    1, false, 8  },
+  { AArch64::LD1Onev2s_POST,    "ld1",  ".2s",    1, false, 8  },
+  { AArch64::LD1Onev1d_POST,    "ld1",  ".1d",    1, false, 8  },
+  { AArch64::LD1Twov16b,        "ld1",  ".16b",   0, false, 0  },
+  { AArch64::LD1Twov8h,         "ld1",  ".8h",    0, false, 0  },
+  { AArch64::LD1Twov4s,         "ld1",  ".4s",    0, false, 0  },
+  { AArch64::LD1Twov2d,         "ld1",  ".2d",    0, false, 0  },
+  { AArch64::LD1Twov8b,         "ld1",  ".8b",    0, false, 0  },
+  { AArch64::LD1Twov4h,         "ld1",  ".4h",    0, false, 0  },
+  { AArch64::LD1Twov2s,         "ld1",  ".2s",    0, false, 0  },
+  { AArch64::LD1Twov1d,         "ld1",  ".1d",    0, false, 0  },
+  { AArch64::LD1Twov16b_POST,   "ld1",  ".16b",   1, false, 32 },
+  { AArch64::LD1Twov8h_POST,    "ld1",  ".8h",    1, false, 32 },
+  { AArch64::LD1Twov4s_POST,    "ld1",  ".4s",    1, false, 32 },
+  { AArch64::LD1Twov2d_POST,    "ld1",  ".2d",    1, false, 32 },
+  { AArch64::LD1Twov8b_POST,    "ld1",  ".8b",    1, false, 16 },
+  { AArch64::LD1Twov4h_POST,    "ld1",  ".4h",    1, false, 16 },
+  { AArch64::LD1Twov2s_POST,    "ld1",  ".2s",    1, false, 16 },
+  { AArch64::LD1Twov1d_POST,    "ld1",  ".1d",    1, false, 16 },
+  { AArch64::LD1Threev16b,      "ld1",  ".16b",   0, false, 0  },
+  { AArch64::LD1Threev8h,       "ld1",  ".8h",    0, false, 0  },
+  { AArch64::LD1Threev4s,       "ld1",  ".4s",    0, false, 0  },
+  { AArch64::LD1Threev2d,       "ld1",  ".2d",    0, false, 0  },
+  { AArch64::LD1Threev8b,       "ld1",  ".8b",    0, false, 0  },
+  { AArch64::LD1Threev4h,       "ld1",  ".4h",    0, false, 0  },
+  { AArch64::LD1Threev2s,       "ld1",  ".2s",    0, false, 0  },
+  { AArch64::LD1Threev1d,       "ld1",  ".1d",    0, false, 0  },
+  { AArch64::LD1Threev16b_POST, "ld1",  ".16b",   1, false, 48 },
+  { AArch64::LD1Threev8h_POST,  "ld1",  ".8h",    1, false, 48 },
+  { AArch64::LD1Threev4s_POST,  "ld1",  ".4s",    1, false, 48 },
+  { AArch64::LD1Threev2d_POST,  "ld1",  ".2d",    1, false, 48 },
+  { AArch64::LD1Threev8b_POST,  "ld1",  ".8b",    1, false, 24 },
+  { AArch64::LD1Threev4h_POST,  "ld1",  ".4h",    1, false, 24 },
+  { AArch64::LD1Threev2s_POST,  "ld1",  ".2s",    1, false, 24 },
+  { AArch64::LD1Threev1d_POST,  "ld1",  ".1d",    1, false, 24 },
+  { AArch64::LD1Fourv16b,       "ld1",  ".16b",   0, false, 0  },
+  { AArch64::LD1Fourv8h,        "ld1",  ".8h",    0, false, 0  },
+  { AArch64::LD1Fourv4s,        "ld1",  ".4s",    0, false, 0  },
+  { AArch64::LD1Fourv2d,        "ld1",  ".2d",    0, false, 0  },
+  { AArch64::LD1Fourv8b,        "ld1",  ".8b",    0, false, 0  },
+  { AArch64::LD1Fourv4h,        "ld1",  ".4h",    0, false, 0  },
+  { AArch64::LD1Fourv2s,        "ld1",  ".2s",    0, false, 0  },
+  { AArch64::LD1Fourv1d,        "ld1",  ".1d",    0, false, 0  },
+  { AArch64::LD1Fourv16b_POST,  "ld1",  ".16b",   1, false, 64 },
+  { AArch64::LD1Fourv8h_POST,   "ld1",  ".8h",    1, false, 64 },
+  { AArch64::LD1Fourv4s_POST,   "ld1",  ".4s",    1, false, 64 },
+  { AArch64::LD1Fourv2d_POST,   "ld1",  ".2d",    1, false, 64 },
+  { AArch64::LD1Fourv8b_POST,   "ld1",  ".8b",    1, false, 32 },
+  { AArch64::LD1Fourv4h_POST,   "ld1",  ".4h",    1, false, 32 },
+  { AArch64::LD1Fourv2s_POST,   "ld1",  ".2s",    1, false, 32 },
+  { AArch64::LD1Fourv1d_POST,   "ld1",  ".1d",    1, false, 32 },
+  { AArch64::LD2i8,             "ld2",  ".b",     1, true,  0  },
+  { AArch64::LD2i16,            "ld2",  ".h",     1, true,  0  },
+  { AArch64::LD2i32,            "ld2",  ".s",     1, true,  0  },
+  { AArch64::LD2i64,            "ld2",  ".d",     1, true,  0  },
+  { AArch64::LD2i8_POST,        "ld2",  ".b",     2, true,  2  },
+  { AArch64::LD2i16_POST,       "ld2",  ".h",     2, true,  4  },
+  { AArch64::LD2i32_POST,       "ld2",  ".s",     2, true,  8  },
+  { AArch64::LD2i64_POST,       "ld2",  ".d",     2, true,  16  },
+  { AArch64::LD2Rv16b,          "ld2r", ".16b",   0, false, 0  },
+  { AArch64::LD2Rv8h,           "ld2r", ".8h",    0, false, 0  },
+  { AArch64::LD2Rv4s,           "ld2r", ".4s",    0, false, 0  },
+  { AArch64::LD2Rv2d,           "ld2r", ".2d",    0, false, 0  },
+  { AArch64::LD2Rv8b,           "ld2r", ".8b",    0, false, 0  },
+  { AArch64::LD2Rv4h,           "ld2r", ".4h",    0, false, 0  },
+  { AArch64::LD2Rv2s,           "ld2r", ".2s",    0, false, 0  },
+  { AArch64::LD2Rv1d,           "ld2r", ".1d",    0, false, 0  },
+  { AArch64::LD2Rv16b_POST,     "ld2r", ".16b",   1, false, 2  },
+  { AArch64::LD2Rv8h_POST,      "ld2r", ".8h",    1, false, 4  },
+  { AArch64::LD2Rv4s_POST,      "ld2r", ".4s",    1, false, 8  },
+  { AArch64::LD2Rv2d_POST,      "ld2r", ".2d",    1, false, 16 },
+  { AArch64::LD2Rv8b_POST,      "ld2r", ".8b",    1, false, 2  },
+  { AArch64::LD2Rv4h_POST,      "ld2r", ".4h",    1, false, 4  },
+  { AArch64::LD2Rv2s_POST,      "ld2r", ".2s",    1, false, 8  },
+  { AArch64::LD2Rv1d_POST,      "ld2r", ".1d",    1, false, 16 },
+  { AArch64::LD2Twov16b,        "ld2",  ".16b",   0, false, 0  },
+  { AArch64::LD2Twov8h,         "ld2",  ".8h",    0, false, 0  },
+  { AArch64::LD2Twov4s,         "ld2",  ".4s",    0, false, 0  },
+  { AArch64::LD2Twov2d,         "ld2",  ".2d",    0, false, 0  },
+  { AArch64::LD2Twov8b,         "ld2",  ".8b",    0, false, 0  },
+  { AArch64::LD2Twov4h,         "ld2",  ".4h",    0, false, 0  },
+  { AArch64::LD2Twov2s,         "ld2",  ".2s",    0, false, 0  },
+  { AArch64::LD2Twov16b_POST,   "ld2",  ".16b",   1, false, 32 },
+  { AArch64::LD2Twov8h_POST,    "ld2",  ".8h",    1, false, 32 },
+  { AArch64::LD2Twov4s_POST,    "ld2",  ".4s",    1, false, 32 },
+  { AArch64::LD2Twov2d_POST,    "ld2",  ".2d",    1, false, 32 },
+  { AArch64::LD2Twov8b_POST,    "ld2",  ".8b",    1, false, 16 },
+  { AArch64::LD2Twov4h_POST,    "ld2",  ".4h",    1, false, 16 },
+  { AArch64::LD2Twov2s_POST,    "ld2",  ".2s",    1, false, 16 },
+  { AArch64::LD3i8,             "ld3",  ".b",     1, true,  0  },
+  { AArch64::LD3i16,            "ld3",  ".h",     1, true,  0  },
+  { AArch64::LD3i32,            "ld3",  ".s",     1, true,  0  },
+  { AArch64::LD3i64,            "ld3",  ".d",     1, true,  0  },
+  { AArch64::LD3i8_POST,        "ld3",  ".b",     2, true,  3  },
+  { AArch64::LD3i16_POST,       "ld3",  ".h",     2, true,  6  },
+  { AArch64::LD3i32_POST,       "ld3",  ".s",     2, true,  12 },
+  { AArch64::LD3i64_POST,       "ld3",  ".d",     2, true,  24 },
+  { AArch64::LD3Rv16b,          "ld3r", ".16b",   0, false, 0  },
+  { AArch64::LD3Rv8h,           "ld3r", ".8h",    0, false, 0  },
+  { AArch64::LD3Rv4s,           "ld3r", ".4s",    0, false, 0  },
+  { AArch64::LD3Rv2d,           "ld3r", ".2d",    0, false, 0  },
+  { AArch64::LD3Rv8b,           "ld3r", ".8b",    0, false, 0  },
+  { AArch64::LD3Rv4h,           "ld3r", ".4h",    0, false, 0  },
+  { AArch64::LD3Rv2s,           "ld3r", ".2s",    0, false, 0  },
+  { AArch64::LD3Rv1d,           "ld3r", ".1d",    0, false, 0  },
+  { AArch64::LD3Rv16b_POST,     "ld3r", ".16b",   1, false, 3  },
+  { AArch64::LD3Rv8h_POST,      "ld3r", ".8h",    1, false, 6  },
+  { AArch64::LD3Rv4s_POST,      "ld3r", ".4s",    1, false, 12 },
+  { AArch64::LD3Rv2d_POST,      "ld3r", ".2d",    1, false, 24 },
+  { AArch64::LD3Rv8b_POST,      "ld3r", ".8b",    1, false, 3  },
+  { AArch64::LD3Rv4h_POST,      "ld3r", ".4h",    1, false, 6  },
+  { AArch64::LD3Rv2s_POST,      "ld3r", ".2s",    1, false, 12 },
+  { AArch64::LD3Rv1d_POST,      "ld3r", ".1d",    1, false, 24 },
+  { AArch64::LD3Threev16b,      "ld3",  ".16b",   0, false, 0  },
+  { AArch64::LD3Threev8h,       "ld3",  ".8h",    0, false, 0  },
+  { AArch64::LD3Threev4s,       "ld3",  ".4s",    0, false, 0  },
+  { AArch64::LD3Threev2d,       "ld3",  ".2d",    0, false, 0  },
+  { AArch64::LD3Threev8b,       "ld3",  ".8b",    0, false, 0  },
+  { AArch64::LD3Threev4h,       "ld3",  ".4h",    0, false, 0  },
+  { AArch64::LD3Threev2s,       "ld3",  ".2s",    0, false, 0  },
+  { AArch64::LD3Threev16b_POST, "ld3",  ".16b",   1, false, 48 },
+  { AArch64::LD3Threev8h_POST,  "ld3",  ".8h",    1, false, 48 },
+  { AArch64::LD3Threev4s_POST,  "ld3",  ".4s",    1, false, 48 },
+  { AArch64::LD3Threev2d_POST,  "ld3",  ".2d",    1, false, 48 },
+  { AArch64::LD3Threev8b_POST,  "ld3",  ".8b",    1, false, 24 },
+  { AArch64::LD3Threev4h_POST,  "ld3",  ".4h",    1, false, 24 },
+  { AArch64::LD3Threev2s_POST,  "ld3",  ".2s",    1, false, 24 },
+  { AArch64::LD4i8,             "ld4",  ".b",     1, true,  0  },
+  { AArch64::LD4i16,            "ld4",  ".h",     1, true,  0  },
+  { AArch64::LD4i32,            "ld4",  ".s",     1, true,  0  },
+  { AArch64::LD4i64,            "ld4",  ".d",     1, true,  0  },
+  { AArch64::LD4i8_POST,        "ld4",  ".b",     2, true,  4  },
+  { AArch64::LD4i16_POST,       "ld4",  ".h",     2, true,  8  },
+  { AArch64::LD4i32_POST,       "ld4",  ".s",     2, true,  16 },
+  { AArch64::LD4i64_POST,       "ld4",  ".d",     2, true,  32 },
+  { AArch64::LD4Rv16b,          "ld4r", ".16b",   0, false, 0  },
+  { AArch64::LD4Rv8h,           "ld4r", ".8h",    0, false, 0  },
+  { AArch64::LD4Rv4s,           "ld4r", ".4s",    0, false, 0  },
+  { AArch64::LD4Rv2d,           "ld4r", ".2d",    0, false, 0  },
+  { AArch64::LD4Rv8b,           "ld4r", ".8b",    0, false, 0  },
+  { AArch64::LD4Rv4h,           "ld4r", ".4h",    0, false, 0  },
+  { AArch64::LD4Rv2s,           "ld4r", ".2s",    0, false, 0  },
+  { AArch64::LD4Rv1d,           "ld4r", ".1d",    0, false, 0  },
+  { AArch64::LD4Rv16b_POST,     "ld4r", ".16b",   1, false, 4  },
+  { AArch64::LD4Rv8h_POST,      "ld4r", ".8h",    1, false, 8  },
+  { AArch64::LD4Rv4s_POST,      "ld4r", ".4s",    1, false, 16 },
+  { AArch64::LD4Rv2d_POST,      "ld4r", ".2d",    1, false, 32 },
+  { AArch64::LD4Rv8b_POST,      "ld4r", ".8b",    1, false, 4  },
+  { AArch64::LD4Rv4h_POST,      "ld4r", ".4h",    1, false, 8  },
+  { AArch64::LD4Rv2s_POST,      "ld4r", ".2s",    1, false, 16 },
+  { AArch64::LD4Rv1d_POST,      "ld4r", ".1d",    1, false, 32 },
+  { AArch64::LD4Fourv16b,       "ld4",  ".16b",   0, false, 0  },
+  { AArch64::LD4Fourv8h,        "ld4",  ".8h",    0, false, 0  },
+  { AArch64::LD4Fourv4s,        "ld4",  ".4s",    0, false, 0  },
+  { AArch64::LD4Fourv2d,        "ld4",  ".2d",    0, false, 0  },
+  { AArch64::LD4Fourv8b,        "ld4",  ".8b",    0, false, 0  },
+  { AArch64::LD4Fourv4h,        "ld4",  ".4h",    0, false, 0  },
+  { AArch64::LD4Fourv2s,        "ld4",  ".2s",    0, false, 0  },
+  { AArch64::LD4Fourv16b_POST,  "ld4",  ".16b",   1, false, 64 },
+  { AArch64::LD4Fourv8h_POST,   "ld4",  ".8h",    1, false, 64 },
+  { AArch64::LD4Fourv4s_POST,   "ld4",  ".4s",    1, false, 64 },
+  { AArch64::LD4Fourv2d_POST,   "ld4",  ".2d",    1, false, 64 },
+  { AArch64::LD4Fourv8b_POST,   "ld4",  ".8b",    1, false, 32 },
+  { AArch64::LD4Fourv4h_POST,   "ld4",  ".4h",    1, false, 32 },
+  { AArch64::LD4Fourv2s_POST,   "ld4",  ".2s",    1, false, 32 },
+  { AArch64::ST1i8,             "st1",  ".b",     0, true,  0  },
+  { AArch64::ST1i16,            "st1",  ".h",     0, true,  0  },
+  { AArch64::ST1i32,            "st1",  ".s",     0, true,  0  },
+  { AArch64::ST1i64,            "st1",  ".d",     0, true,  0  },
+  { AArch64::ST1i8_POST,        "st1",  ".b",     1, true,  1  },
+  { AArch64::ST1i16_POST,       "st1",  ".h",     1, true,  2  },
+  { AArch64::ST1i32_POST,       "st1",  ".s",     1, true,  4  },
+  { AArch64::ST1i64_POST,       "st1",  ".d",     1, true,  8  },
+  { AArch64::ST1Onev16b,        "st1",  ".16b",   0, false, 0  },
+  { AArch64::ST1Onev8h,         "st1",  ".8h",    0, false, 0  },
+  { AArch64::ST1Onev4s,         "st1",  ".4s",    0, false, 0  },
+  { AArch64::ST1Onev2d,         "st1",  ".2d",    0, false, 0  },
+  { AArch64::ST1Onev8b,         "st1",  ".8b",    0, false, 0  },
+  { AArch64::ST1Onev4h,         "st1",  ".4h",    0, false, 0  },
+  { AArch64::ST1Onev2s,         "st1",  ".2s",    0, false, 0  },
+  { AArch64::ST1Onev1d,         "st1",  ".1d",    0, false, 0  },
+  { AArch64::ST1Onev16b_POST,   "st1",  ".16b",   1, false, 16 },
+  { AArch64::ST1Onev8h_POST,    "st1",  ".8h",    1, false, 16 },
+  { AArch64::ST1Onev4s_POST,    "st1",  ".4s",    1, false, 16 },
+  { AArch64::ST1Onev2d_POST,    "st1",  ".2d",    1, false, 16 },
+  { AArch64::ST1Onev8b_POST,    "st1",  ".8b",    1, false, 8  },
+  { AArch64::ST1Onev4h_POST,    "st1",  ".4h",    1, false, 8  },
+  { AArch64::ST1Onev2s_POST,    "st1",  ".2s",    1, false, 8  },
+  { AArch64::ST1Onev1d_POST,    "st1",  ".1d",    1, false, 8  },
+  { AArch64::ST1Twov16b,        "st1",  ".16b",   0, false, 0  },
+  { AArch64::ST1Twov8h,         "st1",  ".8h",    0, false, 0  },
+  { AArch64::ST1Twov4s,         "st1",  ".4s",    0, false, 0  },
+  { AArch64::ST1Twov2d,         "st1",  ".2d",    0, false, 0  },
+  { AArch64::ST1Twov8b,         "st1",  ".8b",    0, false, 0  },
+  { AArch64::ST1Twov4h,         "st1",  ".4h",    0, false, 0  },
+  { AArch64::ST1Twov2s,         "st1",  ".2s",    0, false, 0  },
+  { AArch64::ST1Twov1d,         "st1",  ".1d",    0, false, 0  },
+  { AArch64::ST1Twov16b_POST,   "st1",  ".16b",   1, false, 32 },
+  { AArch64::ST1Twov8h_POST,    "st1",  ".8h",    1, false, 32 },
+  { AArch64::ST1Twov4s_POST,    "st1",  ".4s",    1, false, 32 },
+  { AArch64::ST1Twov2d_POST,    "st1",  ".2d",    1, false, 32 },
+  { AArch64::ST1Twov8b_POST,    "st1",  ".8b",    1, false, 16 },
+  { AArch64::ST1Twov4h_POST,    "st1",  ".4h",    1, false, 16 },
+  { AArch64::ST1Twov2s_POST,    "st1",  ".2s",    1, false, 16 },
+  { AArch64::ST1Twov1d_POST,    "st1",  ".1d",    1, false, 16 },
+  { AArch64::ST1Threev16b,      "st1",  ".16b",   0, false, 0  },
+  { AArch64::ST1Threev8h,       "st1",  ".8h",    0, false, 0  },
+  { AArch64::ST1Threev4s,       "st1",  ".4s",    0, false, 0  },
+  { AArch64::ST1Threev2d,       "st1",  ".2d",    0, false, 0  },
+  { AArch64::ST1Threev8b,       "st1",  ".8b",    0, false, 0  },
+  { AArch64::ST1Threev4h,       "st1",  ".4h",    0, false, 0  },
+  { AArch64::ST1Threev2s,       "st1",  ".2s",    0, false, 0  },
+  { AArch64::ST1Threev1d,       "st1",  ".1d",    0, false, 0  },
+  { AArch64::ST1Threev16b_POST, "st1",  ".16b",   1, false, 48 },
+  { AArch64::ST1Threev8h_POST,  "st1",  ".8h",    1, false, 48 },
+  { AArch64::ST1Threev4s_POST,  "st1",  ".4s",    1, false, 48 },
+  { AArch64::ST1Threev2d_POST,  "st1",  ".2d",    1, false, 48 },
+  { AArch64::ST1Threev8b_POST,  "st1",  ".8b",    1, false, 24 },
+  { AArch64::ST1Threev4h_POST,  "st1",  ".4h",    1, false, 24 },
+  { AArch64::ST1Threev2s_POST,  "st1",  ".2s",    1, false, 24 },
+  { AArch64::ST1Threev1d_POST,  "st1",  ".1d",    1, false, 24 },
+  { AArch64::ST1Fourv16b,       "st1",  ".16b",   0, false, 0  },
+  { AArch64::ST1Fourv8h,        "st1",  ".8h",    0, false, 0  },
+  { AArch64::ST1Fourv4s,        "st1",  ".4s",    0, false, 0  },
+  { AArch64::ST1Fourv2d,        "st1",  ".2d",    0, false, 0  },
+  { AArch64::ST1Fourv8b,        "st1",  ".8b",    0, false, 0  },
+  { AArch64::ST1Fourv4h,        "st1",  ".4h",    0, false, 0  },
+  { AArch64::ST1Fourv2s,        "st1",  ".2s",    0, false, 0  },
+  { AArch64::ST1Fourv1d,        "st1",  ".1d",    0, false, 0  },
+  { AArch64::ST1Fourv16b_POST,  "st1",  ".16b",   1, false, 64 },
+  { AArch64::ST1Fourv8h_POST,   "st1",  ".8h",    1, false, 64 },
+  { AArch64::ST1Fourv4s_POST,   "st1",  ".4s",    1, false, 64 },
+  { AArch64::ST1Fourv2d_POST,   "st1",  ".2d",    1, false, 64 },
+  { AArch64::ST1Fourv8b_POST,   "st1",  ".8b",    1, false, 32 },
+  { AArch64::ST1Fourv4h_POST,   "st1",  ".4h",    1, false, 32 },
+  { AArch64::ST1Fourv2s_POST,   "st1",  ".2s",    1, false, 32 },
+  { AArch64::ST1Fourv1d_POST,   "st1",  ".1d",    1, false, 32 },
+  { AArch64::ST2i8,             "st2",  ".b",     0, true,  0  },
+  { AArch64::ST2i16,            "st2",  ".h",     0, true,  0  },
+  { AArch64::ST2i32,            "st2",  ".s",     0, true,  0  },
+  { AArch64::ST2i64,            "st2",  ".d",     0, true,  0  },
+  { AArch64::ST2i8_POST,        "st2",  ".b",     1, true,  2  },
+  { AArch64::ST2i16_POST,       "st2",  ".h",     1, true,  4  },
+  { AArch64::ST2i32_POST,       "st2",  ".s",     1, true,  8  },
+  { AArch64::ST2i64_POST,       "st2",  ".d",     1, true,  16 },
+  { AArch64::ST2Twov16b,        "st2",  ".16b",   0, false, 0  },
+  { AArch64::ST2Twov8h,         "st2",  ".8h",    0, false, 0  },
+  { AArch64::ST2Twov4s,         "st2",  ".4s",    0, false, 0  },
+  { AArch64::ST2Twov2d,         "st2",  ".2d",    0, false, 0  },
+  { AArch64::ST2Twov8b,         "st2",  ".8b",    0, false, 0  },
+  { AArch64::ST2Twov4h,         "st2",  ".4h",    0, false, 0  },
+  { AArch64::ST2Twov2s,         "st2",  ".2s",    0, false, 0  },
+  { AArch64::ST2Twov16b_POST,   "st2",  ".16b",   1, false, 32 },
+  { AArch64::ST2Twov8h_POST,    "st2",  ".8h",    1, false, 32 },
+  { AArch64::ST2Twov4s_POST,    "st2",  ".4s",    1, false, 32 },
+  { AArch64::ST2Twov2d_POST,    "st2",  ".2d",    1, false, 32 },
+  { AArch64::ST2Twov8b_POST,    "st2",  ".8b",    1, false, 16 },
+  { AArch64::ST2Twov4h_POST,    "st2",  ".4h",    1, false, 16 },
+  { AArch64::ST2Twov2s_POST,    "st2",  ".2s",    1, false, 16 },
+  { AArch64::ST3i8,             "st3",  ".b",     0, true,  0  },
+  { AArch64::ST3i16,            "st3",  ".h",     0, true,  0  },
+  { AArch64::ST3i32,            "st3",  ".s",     0, true,  0  },
+  { AArch64::ST3i64,            "st3",  ".d",     0, true,  0  },
+  { AArch64::ST3i8_POST,        "st3",  ".b",     1, true,  3  },
+  { AArch64::ST3i16_POST,       "st3",  ".h",     1, true,  6  },
+  { AArch64::ST3i32_POST,       "st3",  ".s",     1, true,  12 },
+  { AArch64::ST3i64_POST,       "st3",  ".d",     1, true,  24 },
+  { AArch64::ST3Threev16b,      "st3",  ".16b",   0, false, 0  },
+  { AArch64::ST3Threev8h,       "st3",  ".8h",    0, false, 0  },
+  { AArch64::ST3Threev4s,       "st3",  ".4s",    0, false, 0  },
+  { AArch64::ST3Threev2d,       "st3",  ".2d",    0, false, 0  },
+  { AArch64::ST3Threev8b,       "st3",  ".8b",    0, false, 0  },
+  { AArch64::ST3Threev4h,       "st3",  ".4h",    0, false, 0  },
+  { AArch64::ST3Threev2s,       "st3",  ".2s",    0, false, 0  },
+  { AArch64::ST3Threev16b_POST, "st3",  ".16b",   1, false, 48 },
+  { AArch64::ST3Threev8h_POST,  "st3",  ".8h",    1, false, 48 },
+  { AArch64::ST3Threev4s_POST,  "st3",  ".4s",    1, false, 48 },
+  { AArch64::ST3Threev2d_POST,  "st3",  ".2d",    1, false, 48 },
+  { AArch64::ST3Threev8b_POST,  "st3",  ".8b",    1, false, 24 },
+  { AArch64::ST3Threev4h_POST,  "st3",  ".4h",    1, false, 24 },
+  { AArch64::ST3Threev2s_POST,  "st3",  ".2s",    1, false, 24 },
+  { AArch64::ST4i8,             "st4",  ".b",     0, true,  0  },
+  { AArch64::ST4i16,            "st4",  ".h",     0, true,  0  },
+  { AArch64::ST4i32,            "st4",  ".s",     0, true,  0  },
+  { AArch64::ST4i64,            "st4",  ".d",     0, true,  0  },
+  { AArch64::ST4i8_POST,        "st4",  ".b",     1, true,  4  },
+  { AArch64::ST4i16_POST,       "st4",  ".h",     1, true,  8  },
+  { AArch64::ST4i32_POST,       "st4",  ".s",     1, true,  16 },
+  { AArch64::ST4i64_POST,       "st4",  ".d",     1, true,  32 },
+  { AArch64::ST4Fourv16b,       "st4",  ".16b",   0, false, 0  },
+  { AArch64::ST4Fourv8h,        "st4",  ".8h",    0, false, 0  },
+  { AArch64::ST4Fourv4s,        "st4",  ".4s",    0, false, 0  },
+  { AArch64::ST4Fourv2d,        "st4",  ".2d",    0, false, 0  },
+  { AArch64::ST4Fourv8b,        "st4",  ".8b",    0, false, 0  },
+  { AArch64::ST4Fourv4h,        "st4",  ".4h",    0, false, 0  },
+  { AArch64::ST4Fourv2s,        "st4",  ".2s",    0, false, 0  },
+  { AArch64::ST4Fourv16b_POST,  "st4",  ".16b",   1, false, 64 },
+  { AArch64::ST4Fourv8h_POST,   "st4",  ".8h",    1, false, 64 },
+  { AArch64::ST4Fourv4s_POST,   "st4",  ".4s",    1, false, 64 },
+  { AArch64::ST4Fourv2d_POST,   "st4",  ".2d",    1, false, 64 },
+  { AArch64::ST4Fourv8b_POST,   "st4",  ".8b",    1, false, 32 },
+  { AArch64::ST4Fourv4h_POST,   "st4",  ".4h",    1, false, 32 },
+  { AArch64::ST4Fourv2s_POST,   "st4",  ".2s",    1, false, 32 },
+};
+
+const LdStNInstrDesc *getLdStNInstrDesc(unsigned Opcode) {
+  for (const auto &Info : LdStNInstInfo)
+    if (Info.Opcode == Opcode)
+      return &Info;
+
+  return nullptr;
+}
+
 }
diff --git a/llvm/lib/Target/AArch64/Utils/AArch64BaseInfo.h b/llvm/lib/Target/AArch64/Utils/AArch64BaseInfo.h
index a3dea22335d21..ffbc69af0dffc 100644
--- a/llvm/lib/Target/AArch64/Utils/AArch64BaseInfo.h
+++ b/llvm/lib/Target/AArch64/Utils/AArch64BaseInfo.h
@@ -1057,6 +1057,16 @@ static inline std::optional<MemInstInfo> getStoreInfo(const MCInst &Inst) {
   return MemInstInfo { DestRegIdx, BaseRegIdx, OffsetIdx, IsPrePost, IsPair };
 }
 
+struct LdStNInstrDesc {
+  unsigned Opcode;
+  const char *Mnemonic;
+  const char *Layout;
+  int ListOperand;
+  bool HasLane;
+  int NaturalOffset;
+};
+
+const LdStNInstrDesc *getLdStNInstrDesc(unsigned Opcode);
 
 namespace AArch64CC {
 
diff --git a/llvm/test/MC/AArch64/LFI/mem.s b/llvm/test/MC/AArch64/LFI/mem.s
index 03dfd42b30454..469b068a786f2 100644
--- a/llvm/test/MC/AArch64/LFI/mem.s
+++ b/llvm/test/MC/AArch64/LFI/mem.s
@@ -132,3 +132,17 @@ stp x0, x1, [x2, #-8]!
 // CHECK:      add x28, x27, w2, uxtw
 // CHECK-NEXT: stp x0, x1, [x28, #-8]
 // CHECK-NEXT: sub x2, x2, #8
+
+ld3 { v0.4s, v1.4s, v2.4s }, [x0], #48
+// CHECK:      add x28, x27, w0, uxtw
+// CHECK-NEXT: ld3 { v0.4s, v1.4s, v2.4s }, [x28]
+// CHECK-NEXT: add x0, x0, #48
+
+st2 { v1.8b, v2.8b }, [x14], #16
+// CHECK:      add x28, x27, w14, uxtw
+// CHECK-NEXT: st2 { v1.8b, v2.8b }, [x28]
+// CHECK-NEXT: add x14, x14, #16
+
+st2 { v1.8b, v2.8b }, [x14]
+// CHECK:      add x28, x27, w14, uxtw
+// CHECK-NEXT: st2 { v1.8b, v2.8b }, [x28]

From fac255b128f3f9f0aa81fa75e8a7fef1261b5834 Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Sat, 23 Aug 2025 15:10:38 -0700
Subject: [PATCH 21/23] [LFI] Remove unnecessary auto-sandboxing flag

---
 llvm/lib/MC/MCLFI.cpp | 17 +++++------------
 1 file changed, 5 insertions(+), 12 deletions(-)

diff --git a/llvm/lib/MC/MCLFI.cpp b/llvm/lib/MC/MCLFI.cpp
index 3cad2e4eff452..615bdb2b8d776 100644
--- a/llvm/lib/MC/MCLFI.cpp
+++ b/llvm/lib/MC/MCLFI.cpp
@@ -23,11 +23,6 @@ static const char NoteNamespace[] = "LFI";
 
 namespace llvm {
 
-cl::opt<bool> FlagEnableAutoSandboxing("nacl-enable-autosandboxing",
-                                cl::desc("Don't use the autosandboxing"
-                                         " assembler for the NaCl SFI."),
-                                cl::init(true));
-
 void initializeLFIMCStreamer(MCStreamer &Streamer, MCContext &Ctx,
                               const Triple &TheTriple) {
   assert(TheTriple.isLFI());
@@ -48,13 +43,11 @@ void initializeLFIMCStreamer(MCStreamer &Streamer, MCContext &Ctx,
 
   // Create the Target specific MCLFIExpander
   assert(TheTarget != nullptr);
-  if (FlagEnableAutoSandboxing) {
-    TheTarget->createMCLFIExpander(
-      Streamer,
-      std::unique_ptr<MCRegisterInfo>(
-          TheTarget->createMCRegInfo(TheTriple.getTriple())),
-      std::unique_ptr<MCInstrInfo>(TheTarget->createMCInstrInfo()));
-  }
+  TheTarget->createMCLFIExpander(
+    Streamer,
+    std::unique_ptr<MCRegisterInfo>(
+        TheTarget->createMCRegInfo(TheTriple.getTriple())),
+    std::unique_ptr<MCInstrInfo>(TheTarget->createMCInstrInfo()));
 
   // Emit an ELF Note section in its own COMDAT group which identifies LFI
   // object files.

From 5e3ae1da139f495b39e7df033806f9de31f87039 Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Sat, 23 Aug 2025 15:16:40 -0700
Subject: [PATCH 22/23] [LFI] Apply clang-format to new files

---
 clang/lib/Driver/ToolChains/LFILinux.cpp      |   4 +-
 clang/lib/Driver/ToolChains/LFILinux.h        |  11 +-
 llvm/include/llvm/MC/MCLFI.h                  |   2 +-
 llvm/include/llvm/MC/MCLFIExpander.h          |   4 +-
 llvm/lib/MC/MCLFI.cpp                         |  32 +--
 llvm/lib/MC/MCLFIExpander.cpp                 |  25 +--
 llvm/lib/MC/MCParser/LFIAsmParser.cpp         |  17 +-
 .../MCTargetDesc/AArch64MCLFIExpander.cpp     | 189 ++++++++++--------
 .../MCTargetDesc/AArch64MCLFIExpander.h       |   8 +-
 9 files changed, 157 insertions(+), 135 deletions(-)

diff --git a/clang/lib/Driver/ToolChains/LFILinux.cpp b/clang/lib/Driver/ToolChains/LFILinux.cpp
index 37510c596b303..7ab2372756af9 100644
--- a/clang/lib/Driver/ToolChains/LFILinux.cpp
+++ b/clang/lib/Driver/ToolChains/LFILinux.cpp
@@ -13,8 +13,8 @@
 using namespace clang::driver::toolchains;
 using namespace llvm::opt;
 
-void LFILinuxToolChain::AddCXXStdlibLibArgs(
-    const ArgList &Args, ArgStringList &CmdArgs) const {
+void LFILinuxToolChain::AddCXXStdlibLibArgs(const ArgList &Args,
+                                            ArgStringList &CmdArgs) const {
   switch (GetCXXStdlibType(Args)) {
   case ToolChain::CST_Libcxx:
     CmdArgs.push_back("-lc++");
diff --git a/clang/lib/Driver/ToolChains/LFILinux.h b/clang/lib/Driver/ToolChains/LFILinux.h
index 4ef7b7334e8b0..e85f74803c1ce 100644
--- a/clang/lib/Driver/ToolChains/LFILinux.h
+++ b/clang/lib/Driver/ToolChains/LFILinux.h
@@ -20,13 +20,12 @@ class LLVM_LIBRARY_VISIBILITY LFILinuxToolChain : public Linux {
   LFILinuxToolChain(const Driver &D, const llvm::Triple &Triple,
                     const llvm::opt::ArgList &Args)
       : Linux(D, Triple, Args) {
-        ExtraOpts.push_back("-z");
-        ExtraOpts.push_back("separate-code");
-      }
+    ExtraOpts.push_back("-z");
+    ExtraOpts.push_back("separate-code");
+  }
 
-  void
-  AddCXXStdlibLibArgs(const llvm::opt::ArgList &Args,
-                            llvm::opt::ArgStringList &CmdArgs) const override;
+  void AddCXXStdlibLibArgs(const llvm::opt::ArgList &Args,
+                           llvm::opt::ArgStringList &CmdArgs) const override;
 };
 
 } // end namespace toolchains
diff --git a/llvm/include/llvm/MC/MCLFI.h b/llvm/include/llvm/MC/MCLFI.h
index a50a228d13643..544bda4c7c8d6 100644
--- a/llvm/include/llvm/MC/MCLFI.h
+++ b/llvm/include/llvm/MC/MCLFI.h
@@ -19,4 +19,4 @@ class Triple;
 void initializeLFIMCStreamer(MCStreamer &Streamer, MCContext &Ctx,
                              const Triple &TheTriple);
 
-}
+} // namespace llvm
diff --git a/llvm/include/llvm/MC/MCLFIExpander.h b/llvm/include/llvm/MC/MCLFIExpander.h
index 7ccb5963e8439..db0b254785ac2 100644
--- a/llvm/include/llvm/MC/MCLFIExpander.h
+++ b/llvm/include/llvm/MC/MCLFIExpander.h
@@ -43,7 +43,7 @@ class MCLFIExpander {
 
 public:
   MCLFIExpander(MCContext &Ctx, std::unique_ptr<MCRegisterInfo> &&RI,
-                 std::unique_ptr<MCInstrInfo> &&II)
+                std::unique_ptr<MCInstrInfo> &&II)
       : Ctx(Ctx), InstInfo(std::move(II)), RegInfo(std::move(RI)) {}
 
   void Error(const MCInst &Inst, const char msg[]);
@@ -75,5 +75,5 @@ class MCLFIExpander {
                           const MCSubtargetInfo &STI) = 0;
 };
 
-}
+} // namespace llvm
 #endif
diff --git a/llvm/lib/MC/MCLFI.cpp b/llvm/lib/MC/MCLFI.cpp
index 615bdb2b8d776..34c187c67c3c7 100644
--- a/llvm/lib/MC/MCLFI.cpp
+++ b/llvm/lib/MC/MCLFI.cpp
@@ -24,36 +24,36 @@ static const char NoteNamespace[] = "LFI";
 namespace llvm {
 
 void initializeLFIMCStreamer(MCStreamer &Streamer, MCContext &Ctx,
-                              const Triple &TheTriple) {
+                             const Triple &TheTriple) {
   assert(TheTriple.isLFI());
   const char *NoteName;
   const char *NoteArch;
   switch (TheTriple.getArch()) {
-    case Triple::aarch64:
-      NoteName = ".note.LFI.ABI.aarch64";
-      NoteArch = "aarch64";
-      break;
-    default:
-      report_fatal_error("Unsupported architecture for LFI");
+  case Triple::aarch64:
+    NoteName = ".note.LFI.ABI.aarch64";
+    NoteArch = "aarch64";
+    break;
+  default:
+    report_fatal_error("Unsupported architecture for LFI");
   }
 
-  std::string Error; //empty
+  std::string Error; // empty
   const Target *TheTarget =
-    TargetRegistry::lookupTarget(TheTriple.getTriple(), Error);
+      TargetRegistry::lookupTarget(TheTriple.getTriple(), Error);
 
   // Create the Target specific MCLFIExpander
   assert(TheTarget != nullptr);
   TheTarget->createMCLFIExpander(
-    Streamer,
-    std::unique_ptr<MCRegisterInfo>(
-        TheTarget->createMCRegInfo(TheTriple.getTriple())),
-    std::unique_ptr<MCInstrInfo>(TheTarget->createMCInstrInfo()));
+      Streamer,
+      std::unique_ptr<MCRegisterInfo>(
+          TheTarget->createMCRegInfo(TheTriple.getTriple())),
+      std::unique_ptr<MCInstrInfo>(TheTarget->createMCInstrInfo()));
 
   // Emit an ELF Note section in its own COMDAT group which identifies LFI
   // object files.
-  MCSectionELF* Note = Ctx.getELFSection(NoteName, ELF::SHT_NOTE,
-                                         ELF::SHF_ALLOC | ELF::SHF_GROUP,
-                                         0, NoteName, /*IsComdat=*/true);
+  MCSectionELF *Note = Ctx.getELFSection(NoteName, ELF::SHT_NOTE,
+                                         ELF::SHF_ALLOC | ELF::SHF_GROUP, 0,
+                                         NoteName, /*IsComdat=*/true);
 
   Streamer.pushSection();
   Streamer.switchSection(Note);
diff --git a/llvm/lib/MC/MCLFIExpander.cpp b/llvm/lib/MC/MCLFIExpander.cpp
index 9ed2286d61ee9..539203c0b1476 100644
--- a/llvm/lib/MC/MCLFIExpander.cpp
+++ b/llvm/lib/MC/MCLFIExpander.cpp
@@ -43,25 +43,17 @@ void MCLFIExpander::invalidateScratchRegs(const MCInst &Inst) {
   }
 }
 
-void MCLFIExpander::clearScratchRegs() {
-  ScratchRegs.clear();
-}
+void MCLFIExpander::clearScratchRegs() { ScratchRegs.clear(); }
 
-void MCLFIExpander::disable() {
-  Enabled = false;
-}
+void MCLFIExpander::disable() { Enabled = false; }
 
-void MCLFIExpander::enable() {
-  Enabled = true;
-}
+void MCLFIExpander::enable() { Enabled = true; }
 
-bool MCLFIExpander::isEnabled() {
-  return Enabled;
-}
+bool MCLFIExpander::isEnabled() { return Enabled; }
 
 MCRegister MCLFIExpander::getScratchReg(int index) {
   assert(index >= 0 && static_cast<unsigned>(index) < numScratchRegs());
-  return ScratchRegs[numScratchRegs()  - index - 1];
+  return ScratchRegs[numScratchRegs() - index - 1];
 }
 
 unsigned MCLFIExpander::numScratchRegs() const { return ScratchRegs.size(); }
@@ -102,12 +94,13 @@ bool MCLFIExpander::mayStore(const MCInst &Inst) const {
   return InstInfo->get(Inst.getOpcode()).mayStore();
 }
 
-bool MCLFIExpander::mayModifyRegister(const MCInst &Inst, MCRegister Reg) const {
+bool MCLFIExpander::mayModifyRegister(const MCInst &Inst,
+                                      MCRegister Reg) const {
   return InstInfo->get(Inst.getOpcode()).hasDefOfPhysReg(Inst, Reg, *RegInfo);
 }
 
 bool MCLFIExpander::explicitlyModifiesRegister(const MCInst &Inst,
-                                                MCRegister Reg) const {
+                                               MCRegister Reg) const {
   const MCInstrDesc &Desc = InstInfo->get(Inst.getOpcode());
   for (int i = 0; i < Desc.NumDefs; ++i) {
     if (Desc.operands()[i].OperandType == MCOI::OPERAND_REGISTER &&
@@ -116,4 +109,4 @@ bool MCLFIExpander::explicitlyModifiesRegister(const MCInst &Inst,
   }
   return false;
 }
-}
+} // namespace llvm
diff --git a/llvm/lib/MC/MCParser/LFIAsmParser.cpp b/llvm/lib/MC/MCParser/LFIAsmParser.cpp
index f0932a0df6b6b..bf91e3f6ae162 100644
--- a/llvm/lib/MC/MCParser/LFIAsmParser.cpp
+++ b/llvm/lib/MC/MCParser/LFIAsmParser.cpp
@@ -10,9 +10,9 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "llvm/MC/MCParser/MCAsmParserExtension.h"
 #include "llvm/MC/MCLFIExpander.h"
 #include "llvm/MC/MCParser/MCAsmLexer.h"
+#include "llvm/MC/MCParser/MCAsmParserExtension.h"
 #include "llvm/MC/MCParser/MCTargetAsmParser.h"
 #include "llvm/MC/MCRegister.h"
 #include "llvm/MC/MCStreamer.h"
@@ -23,15 +23,15 @@ using namespace llvm;
 
 class LFIAsmParser : public MCAsmParserExtension {
   MCLFIExpander *Expander;
-  template<bool (LFIAsmParser::*HandlerMethod)(StringRef, SMLoc)>
+  template <bool (LFIAsmParser::*HandlerMethod)(StringRef, SMLoc)>
   void addDirectiveHandler(StringRef Directive) {
-    MCAsmParser::ExtensionDirectiveHandler Handler = std::make_pair(
-        this, HandleDirective<LFIAsmParser, HandlerMethod>);
+    MCAsmParser::ExtensionDirectiveHandler Handler =
+        std::make_pair(this, HandleDirective<LFIAsmParser, HandlerMethod>);
 
     getParser().addDirectiveHandler(Directive, Handler);
   }
 
- public:
+public:
   LFIAsmParser(MCLFIExpander *Exp) : Expander(Exp) {}
   void Initialize(MCAsmParser &Parser) override {
     // Call the base implementation.
@@ -55,13 +55,12 @@ class LFIAsmParser : public MCAsmParserExtension {
 
       else if (getLexer().isNot(AsmToken::EndOfStatement))
         return Error(Loc, kInvalidOptionError);
-    }
-    else {
+    } else {
       return Error(Loc, kInvalidOptionError);
     }
     Lex();
 
-    if(Expander->addScratchReg(RegNo))
+    if (Expander->addScratchReg(RegNo))
       return Error(Loc, "Register can't be used as a scratch register");
     return false;
   }
@@ -107,4 +106,4 @@ namespace llvm {
 MCAsmParserExtension *createLFIAsmParser(MCLFIExpander *Exp) {
   return new LFIAsmParser(Exp);
 }
-}
+} // namespace llvm
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
index 58597f506e365..7de5d80408e16 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
@@ -30,8 +30,8 @@ using namespace llvm;
 
 #define DEBUG_TYPE "lfi"
 
-static cl::opt<bool> AArch64LFIErrorReserved("aarch64-lfi-error-reserved",
-    cl::Hidden, cl::init(false),
+static cl::opt<bool> AArch64LFIErrorReserved(
+    "aarch64-lfi-error-reserved", cl::Hidden, cl::init(false),
     cl::desc("Produce errors for uses of LFI reserved registers"));
 
 static MCRegister LFIAddrReg = AArch64::X28;
@@ -39,11 +39,13 @@ static MCRegister LFIBaseReg = AArch64::X27;
 static MCRegister LFIScratchReg = AArch64::X26;
 static MCRegister LFITLSReg = AArch64::X25;
 
-static bool hasFeature(const FeatureBitset Feature, const MCSubtargetInfo &STI) {
+static bool hasFeature(const FeatureBitset Feature,
+                       const MCSubtargetInfo &STI) {
   return (STI.getFeatureBits() & Feature) == Feature;
 }
 
-bool AArch64::AArch64MCLFIExpander::isValidScratchRegister(MCRegister Reg) const {
+bool AArch64::AArch64MCLFIExpander::isValidScratchRegister(
+    MCRegister Reg) const {
   return Reg != AArch64::SP;
 }
 
@@ -54,8 +56,8 @@ MCRegister AArch64::AArch64MCLFIExpander::getScratch() {
   return getScratchReg(0);
 }
 
-static void emit(unsigned int Op, MCRegister Rd, MCRegister Rs,
-    int64_t Imm, MCStreamer &Out, const MCSubtargetInfo &STI) {
+static void emit(unsigned int Op, MCRegister Rd, MCRegister Rs, int64_t Imm,
+                 MCStreamer &Out, const MCSubtargetInfo &STI) {
   MCInst Inst;
   Inst.setOpcode(Op);
   Inst.addOperand(MCOperand::createReg(Rd));
@@ -64,8 +66,8 @@ static void emit(unsigned int Op, MCRegister Rd, MCRegister Rs,
   Out.emitInstruction(Inst, STI);
 }
 
-static void emit(unsigned int Op, MCRegister Rd, MCRegister Rt1,
-    MCRegister Rt2, int64_t Imm, MCStreamer &Out, const MCSubtargetInfo &STI) {
+static void emit(unsigned int Op, MCRegister Rd, MCRegister Rt1, MCRegister Rt2,
+                 int64_t Imm, MCStreamer &Out, const MCSubtargetInfo &STI) {
   MCInst Inst;
   Inst.setOpcode(Op);
   Inst.addOperand(MCOperand::createReg(Rd));
@@ -75,9 +77,9 @@ static void emit(unsigned int Op, MCRegister Rd, MCRegister Rt1,
   Out.emitInstruction(Inst, STI);
 }
 
-static void emit(unsigned int Op, MCRegister Rd, MCRegister Rt1,
-    MCRegister Rt2, int64_t Imm1, int64_t Imm2,
-    MCStreamer &Out, const MCSubtargetInfo &STI) {
+static void emit(unsigned int Op, MCRegister Rd, MCRegister Rt1, MCRegister Rt2,
+                 int64_t Imm1, int64_t Imm2, MCStreamer &Out,
+                 const MCSubtargetInfo &STI) {
   MCInst Inst;
   Inst.setOpcode(Op);
   Inst.addOperand(MCOperand::createReg(Rd));
@@ -88,9 +90,8 @@ static void emit(unsigned int Op, MCRegister Rd, MCRegister Rt1,
   Out.emitInstruction(Inst, STI);
 }
 
-static void emit(unsigned int Op, MCRegister Rd, MCRegister Rt,
-    int64_t Imm1, int64_t Imm2,
-    MCStreamer &Out, const MCSubtargetInfo &STI) {
+static void emit(unsigned int Op, MCRegister Rd, MCRegister Rt, int64_t Imm1,
+                 int64_t Imm2, MCStreamer &Out, const MCSubtargetInfo &STI) {
   MCInst Inst;
   Inst.setOpcode(Op);
   Inst.addOperand(MCOperand::createReg(Rd));
@@ -101,30 +102,28 @@ static void emit(unsigned int Op, MCRegister Rd, MCRegister Rt,
 }
 
 static void emit(unsigned int Op, MCRegister Reg, MCStreamer &Out,
-    const MCSubtargetInfo &STI) {
+                 const MCSubtargetInfo &STI) {
   MCInst Inst;
   Inst.setOpcode(Op);
   Inst.addOperand(MCOperand::createReg(Reg));
   Out.emitInstruction(Inst, STI);
 }
 
-static void emitMov(MCRegister Dest, MCRegister Src, MCStreamer &Out, const MCSubtargetInfo &STI) {
+static void emitMov(MCRegister Dest, MCRegister Src, MCStreamer &Out,
+                    const MCSubtargetInfo &STI) {
   emit(AArch64::ORRXrs, Dest, AArch64::XZR, Src, 0, Out, STI);
 }
 
 // Emit 'add Dest, LFIBaseReg, W(Src), uxtw'
 static void emitAddMask(MCRegister Dest, MCRegister Src, MCStreamer &Out,
                         const MCSubtargetInfo &STI) {
-  emit(AArch64::ADDXrx,
-      Dest,
-      LFIBaseReg,
-      getWRegFromXReg(Src),
-      AArch64_AM::getArithExtendImm(AArch64_AM::UXTW, 0),
-      Out, STI);
+  emit(AArch64::ADDXrx, Dest, LFIBaseReg, getWRegFromXReg(Src),
+       AArch64_AM::getArithExtendImm(AArch64_AM::UXTW, 0), Out, STI);
 }
 
 // Emit 'Op(ld/st) Dest, [LFIBaseReg, W(Target), uxtw]'
-static void emitMemMask(unsigned Op, MCRegister Dest, MCRegister Target, MCStreamer &Out, const MCSubtargetInfo &STI) {
+static void emitMemMask(unsigned Op, MCRegister Dest, MCRegister Target,
+                        MCStreamer &Out, const MCSubtargetInfo &STI) {
   emit(Op, Dest, LFIBaseReg, getWRegFromXReg(Target), 0, 0, Out, STI);
 }
 
@@ -136,11 +135,10 @@ static void emitBranch(unsigned int Opcode, MCRegister Target, MCStreamer &Out,
   Out.emitInstruction(Branch, STI);
 }
 
-void AArch64::AArch64MCLFIExpander::expandIndirectBranch(const MCInst &Inst,
-                                                         MCStreamer &Out,
-                                                         const MCSubtargetInfo &STI,
-                                                         bool IsCall) {
-  (void) IsCall;
+void AArch64::AArch64MCLFIExpander::expandIndirectBranch(
+    const MCInst &Inst, MCStreamer &Out, const MCSubtargetInfo &STI,
+    bool IsCall) {
+  (void)IsCall;
 
   assert(Inst.getOperand(0).isReg());
   MCRegister BranchReg = Inst.getOperand(0).getReg();
@@ -149,7 +147,8 @@ void AArch64::AArch64MCLFIExpander::expandIndirectBranch(const MCInst &Inst,
   emitBranch(Inst.getOpcode(), LFIAddrReg, Out, STI);
 }
 
-void AArch64::AArch64MCLFIExpander::expandCall(const MCInst &Inst, MCStreamer &Out,
+void AArch64::AArch64MCLFIExpander::expandCall(const MCInst &Inst,
+                                               MCStreamer &Out,
                                                const MCSubtargetInfo &STI) {
   if (Inst.getOperand(0).isReg())
     expandIndirectBranch(Inst, Out, STI, true);
@@ -157,7 +156,8 @@ void AArch64::AArch64MCLFIExpander::expandCall(const MCInst &Inst, MCStreamer &O
     Out.emitInstruction(Inst, STI);
 }
 
-void AArch64::AArch64MCLFIExpander::expandReturn(const MCInst &Inst, MCStreamer &Out,
+void AArch64::AArch64MCLFIExpander::expandReturn(const MCInst &Inst,
+                                                 MCStreamer &Out,
                                                  const MCSubtargetInfo &STI) {
   assert(Inst.getOperand(0).isReg());
   if (Inst.getOperand(0).getReg() != AArch64::LR)
@@ -171,15 +171,15 @@ bool AArch64::AArch64MCLFIExpander::mayModifyStack(const MCInst &Inst) {
 }
 
 bool AArch64::AArch64MCLFIExpander::mayModifyReserved(const MCInst &Inst) {
-  return mayModifyRegister(Inst, LFIAddrReg) || mayModifyRegister(Inst, LFIBaseReg);
+  return mayModifyRegister(Inst, LFIAddrReg) ||
+         mayModifyRegister(Inst, LFIBaseReg);
 }
 
 bool AArch64::AArch64MCLFIExpander::mayModifyLR(const MCInst &Inst) {
   return mayModifyRegister(Inst, AArch64::LR);
 }
 
-static MCInst replaceReg(const MCInst &Inst,
-                         MCRegister Dest, MCRegister Src) {
+static MCInst replaceReg(const MCInst &Inst, MCRegister Dest, MCRegister Src) {
   MCInst New;
   New.setOpcode(Inst.getOpcode());
   New.setLoc(Inst.getLoc());
@@ -215,7 +215,8 @@ void AArch64::AArch64MCLFIExpander::expandStackModification(
 
   MCInst ModInst;
   MCRegister Scratch = getScratch();
-  assert(Inst.getOperand(0).isReg() && Inst.getOperand(0).getReg() == AArch64::SP);
+  assert(Inst.getOperand(0).isReg() &&
+         Inst.getOperand(0).getReg() == AArch64::SP);
   ModInst.setOpcode(Inst.getOpcode());
   ModInst.setLoc(Inst.getLoc());
   ModInst.addOperand(MCOperand::createReg(Scratch));
@@ -233,13 +234,17 @@ static unsigned convertUiToRoW(unsigned Op);
 static unsigned convertPreToRoW(unsigned Op);
 static unsigned convertPostToRoW(unsigned Op);
 
-static unsigned convertPrePostToBase(unsigned Op, bool &IsPre, bool &IsBaseNoOffset);
+static unsigned convertPrePostToBase(unsigned Op, bool &IsPre,
+                                     bool &IsBaseNoOffset);
 static unsigned getPrePostScale(unsigned Op);
 
-static void emitSafeLoadStoreDemoted(const MCInst &Inst, unsigned N, MCStreamer &Out, const MCSubtargetInfo &STI) {
+static void emitSafeLoadStoreDemoted(const MCInst &Inst, unsigned N,
+                                     MCStreamer &Out,
+                                     const MCSubtargetInfo &STI) {
   MCInst LoadStore;
   bool IsPre, IsBaseNoOffset;
-  auto NewOpCode = convertPrePostToBase(Inst.getOpcode(), IsPre, IsBaseNoOffset);
+  auto NewOpCode =
+      convertPrePostToBase(Inst.getOpcode(), IsPre, IsBaseNoOffset);
   LoadStore.setOpcode(NewOpCode);
   for (unsigned I = 1; I < N; I++)
     LoadStore.addOperand(Inst.getOperand(I));
@@ -251,7 +256,8 @@ static void emitSafeLoadStoreDemoted(const MCInst &Inst, unsigned N, MCStreamer
   Out.emitInstruction(LoadStore, STI);
 }
 
-static void emitSafeLoadStore(const MCInst &Inst, unsigned N, MCStreamer &Out, const MCSubtargetInfo &STI) {
+static void emitSafeLoadStore(const MCInst &Inst, unsigned N, MCStreamer &Out,
+                              const MCSubtargetInfo &STI) {
   MCInst LoadStore;
   LoadStore.setOpcode(Inst.getOpcode());
   for (unsigned I = 0; I < N; ++I)
@@ -262,8 +268,9 @@ static void emitSafeLoadStore(const MCInst &Inst, unsigned N, MCStreamer &Out, c
   Out.emitInstruction(LoadStore, STI);
 }
 
-void AArch64::AArch64MCLFIExpander::expandLoadStoreBasic(const MCInst &Inst, MemInstInfo &MII,
-    MCStreamer &Out, const MCSubtargetInfo &STI) {
+void AArch64::AArch64MCLFIExpander::expandLoadStoreBasic(
+    const MCInst &Inst, MemInstInfo &MII, MCStreamer &Out,
+    const MCSubtargetInfo &STI) {
   emitAddMask(LFIAddrReg, Inst.getOperand(MII.BaseRegIdx).getReg(), Out, STI);
 
   if (MII.IsPrePost) {
@@ -273,20 +280,23 @@ void AArch64::AArch64MCLFIExpander::expandLoadStoreBasic(const MCInst &Inst, Mem
     MCRegister Base = Inst.getOperand(MII.BaseRegIdx).getReg();
     MCOperand OffsetMO = Inst.getOperand(MII.OffsetIdx);
     if (OffsetMO.isReg()) {
-      // The immediate offset of post-indexed addressing NEON Instrs has a fixed value, and
-      // it is encoded as a post-index addressing with XZR register operand.
-      // e.g., LD3Threev3d_POST can only have #48 as its operand and its offset
-      // MachineOperand holds XZR, which is a *Register* kind, not Imm.
+      // The immediate offset of post-indexed addressing NEON Instrs has a fixed
+      // value, and it is encoded as a post-index addressing with XZR register
+      // operand. e.g., LD3Threev3d_POST can only have #48 as its operand and
+      // its offset MachineOperand holds XZR, which is a *Register* kind, not
+      // Imm.
       MCRegister OffReg = OffsetMO.getReg();
       if (OffReg == AArch64::XZR) {
         const LdStNInstrDesc *Info = getLdStNInstrDesc(Inst.getOpcode());
         assert(Info && Info->NaturalOffset >= 0);
-        return emit(AArch64::ADDXri, Base, Base, Info->NaturalOffset, 0, Out, STI);
+        return emit(AArch64::ADDXri, Base, Base, Info->NaturalOffset, 0, Out,
+                    STI);
       }
       assert(OffReg != AArch64::WZR);
       return emit(AArch64::ADDXrs, Base, Base, OffsetMO.getReg(), 0, Out, STI);
     } else {
-      auto Offset = Inst.getOperand(MII.OffsetIdx).getImm() * getPrePostScale(Inst.getOpcode());
+      auto Offset = Inst.getOperand(MII.OffsetIdx).getImm() *
+                    getPrePostScale(Inst.getOpcode());
       if (Offset >= 0)
         return emit(AArch64::ADDXri, Base, Base, Offset, 0, Out, STI);
       return emit(AArch64::SUBXri, Base, Base, -Offset, 0, Out, STI);
@@ -296,14 +306,16 @@ void AArch64::AArch64MCLFIExpander::expandLoadStoreBasic(const MCInst &Inst, Mem
   return emitSafeLoadStore(Inst, MII.BaseRegIdx, Out, STI);
 }
 
-void AArch64::AArch64MCLFIExpander::expandLoadStoreRoW(const MCInst &Inst, MemInstInfo &MII,
-    MCStreamer &Out, const MCSubtargetInfo &STI) {
+void AArch64::AArch64MCLFIExpander::expandLoadStoreRoW(
+    const MCInst &Inst, MemInstInfo &MII, MCStreamer &Out,
+    const MCSubtargetInfo &STI) {
   unsigned MemOp;
   unsigned Op = Inst.getOpcode();
   if ((MemOp = convertUiToRoW(Op)) != AArch64::INSTRUCTION_LIST_END) {
     auto OffsetMCO = Inst.getOperand(2);
     if (OffsetMCO.isImm() && OffsetMCO.getImm() == 0)
-      return emitMemMask(MemOp, Inst.getOperand(0).getReg(), Inst.getOperand(1).getReg(), Out, STI);
+      return emitMemMask(MemOp, Inst.getOperand(0).getReg(),
+                         Inst.getOperand(1).getReg(), Out, STI);
     return expandLoadStoreBasic(Inst, MII, Out, STI);
   }
 
@@ -338,9 +350,11 @@ void AArch64::AArch64MCLFIExpander::expandLoadStoreRoW(const MCInst &Inst, MemIn
     if (!IsShift)
       Shift = 0;
     if (Extend)
-      emit(AArch64::ADDXrx, Scratch, Reg1, Reg2, AArch64_AM::getArithExtendImm(AArch64_AM::SXTX, Shift), Out, STI);
+      emit(AArch64::ADDXrx, Scratch, Reg1, Reg2,
+           AArch64_AM::getArithExtendImm(AArch64_AM::SXTX, Shift), Out, STI);
     else
-      emit(AArch64::ADDXrs, Scratch, Reg1, Reg2, AArch64_AM::getShifterImm(AArch64_AM::LSL, Shift), Out, STI);
+      emit(AArch64::ADDXrs, Scratch, Reg1, Reg2,
+           AArch64_AM::getShifterImm(AArch64_AM::LSL, Shift), Out, STI);
     return emitMemMask(MemOp, Inst.getOperand(0).getReg(), Scratch, Out, STI);
   }
 
@@ -353,19 +367,21 @@ void AArch64::AArch64MCLFIExpander::expandLoadStoreRoW(const MCInst &Inst, MemIn
     if (!IsShift)
       Shift = 0;
     if (S)
-      emit(AArch64::ADDXrx, Scratch, Reg1, Reg2, AArch64_AM::getArithExtendImm(AArch64_AM::SXTW, Shift), Out, STI);
+      emit(AArch64::ADDXrx, Scratch, Reg1, Reg2,
+           AArch64_AM::getArithExtendImm(AArch64_AM::SXTW, Shift), Out, STI);
     else
-      emit(AArch64::ADDXrx, Scratch, Reg1, Reg2, AArch64_AM::getArithExtendImm(AArch64_AM::UXTW, Shift), Out, STI);
+      emit(AArch64::ADDXrx, Scratch, Reg1, Reg2,
+           AArch64_AM::getArithExtendImm(AArch64_AM::UXTW, Shift), Out, STI);
     return emitMemMask(MemOp, Inst.getOperand(0).getReg(), Scratch, Out, STI);
   }
 }
 
-void AArch64::AArch64MCLFIExpander::expandLoadStore(const MCInst &Inst,
-                                                    MCStreamer &Out,
-                                                    const MCSubtargetInfo &STI) {
+void AArch64::AArch64MCLFIExpander::expandLoadStore(
+    const MCInst &Inst, MCStreamer &Out, const MCSubtargetInfo &STI) {
   if (hasFeature(FeatureBitset({AArch64::FeatureLFIJumps}), STI))
     return Out.emitInstruction(Inst, STI);
-  if (hasFeature(FeatureBitset({AArch64::FeatureLFIStores}), STI) && !mayStore(Inst))
+  if (hasFeature(FeatureBitset({AArch64::FeatureLFIStores}), STI) &&
+      !mayStore(Inst))
     return Out.emitInstruction(Inst, STI);
 
   auto MII = getLoadInfo(Inst);
@@ -377,7 +393,7 @@ void AArch64::AArch64MCLFIExpander::expandLoadStore(const MCInst &Inst,
 
   // Stack accesses without a register offset don't need rewriting.
   if (Inst.getOperand(MII->BaseRegIdx).getReg() == AArch64::SP) {
-    if (MII->BaseRegIdx == (int) Inst.getNumOperands() - 1 ||
+    if (MII->BaseRegIdx == (int)Inst.getNumOperands() - 1 ||
         !Inst.getOperand(MII->BaseRegIdx + 1).isReg())
       return Out.emitInstruction(Inst, STI);
   }
@@ -389,33 +405,44 @@ void AArch64::AArch64MCLFIExpander::expandLoadStore(const MCInst &Inst,
     expandLoadStoreBasic(Inst, MII.value(), Out, STI);
 }
 
-void AArch64::AArch64MCLFIExpander::emitLFICall(LFICallType CallType, MCStreamer &Out, const MCSubtargetInfo &STI) {
+void AArch64::AArch64MCLFIExpander::emitLFICall(LFICallType CallType,
+                                                MCStreamer &Out,
+                                                const MCSubtargetInfo &STI) {
   MCRegister Scratch = getScratch();
   emitMov(Scratch, AArch64::LR, Out, STI);
   unsigned Offset;
   switch (CallType) {
-    case LFISyscall: Offset = 0; break;
-    case LFITLSRead: Offset = 1; break;
-    case LFITLSWrite: Offset = 2; break;
+  case LFISyscall:
+    Offset = 0;
+    break;
+  case LFITLSRead:
+    Offset = 1;
+    break;
+  case LFITLSWrite:
+    Offset = 2;
+    break;
   }
   emit(AArch64::LDRXui, AArch64::LR, LFIBaseReg, Offset, Out, STI);
   emit(AArch64::BLR, AArch64::LR, Out, STI);
   emitAddMask(AArch64::LR, Scratch, Out, STI);
 }
 
-void AArch64::AArch64MCLFIExpander::expandSyscall(const MCInst &Inst, MCStreamer &Out,
-                   const MCSubtargetInfo &STI) {
+void AArch64::AArch64MCLFIExpander::expandSyscall(const MCInst &Inst,
+                                                  MCStreamer &Out,
+                                                  const MCSubtargetInfo &STI) {
   emitLFICall(LFISyscall, Out, STI);
 }
 
-static void emitSwap(MCRegister Reg1, MCRegister Reg2, MCStreamer &Out, const MCSubtargetInfo &STI) {
+static void emitSwap(MCRegister Reg1, MCRegister Reg2, MCStreamer &Out,
+                     const MCSubtargetInfo &STI) {
   emit(AArch64::EORXrs, Reg1, Reg1, Reg2, 0, Out, STI);
   emit(AArch64::EORXrs, Reg2, Reg1, Reg2, 0, Out, STI);
   emit(AArch64::EORXrs, Reg1, Reg1, Reg2, 0, Out, STI);
 }
 
-void AArch64::AArch64MCLFIExpander::expandTLSRead(const MCInst &Inst, MCStreamer &Out,
-                   const MCSubtargetInfo &STI) {
+void AArch64::AArch64MCLFIExpander::expandTLSRead(const MCInst &Inst,
+                                                  MCStreamer &Out,
+                                                  const MCSubtargetInfo &STI) {
   MCRegister Reg = Inst.getOperand(0).getReg();
 
   if (hasFeature(FeatureBitset({AArch64::FeatureLFITLSReg}), STI))
@@ -430,8 +457,9 @@ void AArch64::AArch64MCLFIExpander::expandTLSRead(const MCInst &Inst, MCStreamer
   }
 }
 
-void AArch64::AArch64MCLFIExpander::expandTLSWrite(const MCInst &Inst, MCStreamer &Out,
-                    const MCSubtargetInfo &STI) {
+void AArch64::AArch64MCLFIExpander::expandTLSWrite(const MCInst &Inst,
+                                                   MCStreamer &Out,
+                                                   const MCSubtargetInfo &STI) {
   MCRegister Reg = Inst.getOperand(1).getReg();
   if (Reg == AArch64::X0) {
     emitLFICall(LFITLSWrite, Out, STI);
@@ -448,15 +476,16 @@ static bool isSyscall(const MCInst &Inst) {
 
 static bool isTLSRead(const MCInst &Inst) {
   return Inst.getOpcode() == AArch64::MRS &&
-    Inst.getOperand(1).getImm() == AArch64SysReg::TPIDR_EL0;
+         Inst.getOperand(1).getImm() == AArch64SysReg::TPIDR_EL0;
 }
 
 static bool isTLSWrite(const MCInst &Inst) {
   return Inst.getOpcode() == AArch64::MSR &&
-    Inst.getOperand(0).getImm() == AArch64SysReg::TPIDR_EL0;
+         Inst.getOperand(0).getImm() == AArch64SysReg::TPIDR_EL0;
 }
 
-void AArch64::AArch64MCLFIExpander::doExpandInst(const MCInst &Inst, MCStreamer &Out,
+void AArch64::AArch64MCLFIExpander::doExpandInst(const MCInst &Inst,
+                                                 MCStreamer &Out,
                                                  const MCSubtargetInfo &STI) {
   if (isSyscall(Inst))
     return expandSyscall(Inst, Out, STI);
@@ -508,7 +537,8 @@ void AArch64::AArch64MCLFIExpander::doExpandInst(const MCInst &Inst, MCStreamer
   return Out.emitInstruction(Inst, STI);
 }
 
-bool AArch64::AArch64MCLFIExpander::expandInst(const MCInst &Inst, MCStreamer &Out,
+bool AArch64::AArch64MCLFIExpander::expandInst(const MCInst &Inst,
+                                               MCStreamer &Out,
                                                const MCSubtargetInfo &STI) {
   if (Guard)
     return false;
@@ -823,13 +853,14 @@ static unsigned convertPostToRoW(unsigned Op) {
 static bool canConvertToRoW(unsigned Op) {
   unsigned Shift;
   return convertUiToRoW(Op) != AArch64::INSTRUCTION_LIST_END ||
-    convertPreToRoW(Op) != AArch64::INSTRUCTION_LIST_END ||
-    convertPostToRoW(Op) != AArch64::INSTRUCTION_LIST_END ||
-    convertRoXToRoW(Op, Shift) != AArch64::INSTRUCTION_LIST_END ||
-    convertRoWToRoW(Op, Shift) != AArch64::INSTRUCTION_LIST_END;
+         convertPreToRoW(Op) != AArch64::INSTRUCTION_LIST_END ||
+         convertPostToRoW(Op) != AArch64::INSTRUCTION_LIST_END ||
+         convertRoXToRoW(Op, Shift) != AArch64::INSTRUCTION_LIST_END ||
+         convertRoWToRoW(Op, Shift) != AArch64::INSTRUCTION_LIST_END;
 }
 
-static unsigned convertPrePostToBase(unsigned Op, bool &IsPre, bool &IsBaseNoOffset) {
+static unsigned convertPrePostToBase(unsigned Op, bool &IsPre,
+                                     bool &IsBaseNoOffset) {
   IsPre = false;
   IsBaseNoOffset = false;
   switch (Op) {
@@ -895,7 +926,7 @@ static unsigned convertPrePostToBase(unsigned Op, bool &IsPre, bool &IsBaseNoOff
   // case AArch64::STLRXpre:
   //   IsPre = true;
   //   IsBaseNoOffset = true;
-    // return AArch64::STLRX;
+  // return AArch64::STLRX;
   case AArch64::LD1i64_POST:
     IsBaseNoOffset = true;
     return AArch64::LD1i64;
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.h b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.h
index 2bfb66955f47c..d664ef9e37757 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.h
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.h
@@ -30,7 +30,7 @@ namespace AArch64 {
 class AArch64MCLFIExpander : public MCLFIExpander {
 public:
   AArch64MCLFIExpander(MCContext &Ctx, std::unique_ptr<MCRegisterInfo> &&RI,
-                    std::unique_ptr<MCInstrInfo> &&II)
+                       std::unique_ptr<MCInstrInfo> &&II)
       : MCLFIExpander(Ctx, std::move(RI), std::move(II)) {}
 
   bool expandInst(const MCInst &Inst, MCStreamer &Out,
@@ -66,7 +66,7 @@ class AArch64MCLFIExpander : public MCLFIExpander {
                                const MCSubtargetInfo &STI);
 
   void expandLRModification(const MCInst &Inst, MCStreamer &Out,
-                                 const MCSubtargetInfo &STI);
+                            const MCSubtargetInfo &STI);
 
   void expandPrefetch(const MCInst &Inst, MCStreamer &Out,
                       const MCSubtargetInfo &STI);
@@ -75,10 +75,10 @@ class AArch64MCLFIExpander : public MCLFIExpander {
                        const MCSubtargetInfo &STI);
 
   void expandLoadStoreBasic(const MCInst &Inst, MemInstInfo &InstInfo,
-      MCStreamer &Out, const MCSubtargetInfo &STI);
+                            MCStreamer &Out, const MCSubtargetInfo &STI);
 
   void expandLoadStoreRoW(const MCInst &Inst, MemInstInfo &InstInfo,
-      MCStreamer &Out, const MCSubtargetInfo &STI);
+                          MCStreamer &Out, const MCSubtargetInfo &STI);
 
   void expandSyscall(const MCInst &Inst, MCStreamer &Out,
                      const MCSubtargetInfo &STI);

From 7a00571ffdd91e657fdf7b39218b0f560adf6ff3 Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Sun, 24 Aug 2025 14:32:28 -0700
Subject: [PATCH 23/23] [LFI] [AArch64] Add rewrites for LSE atomics

---
 .../MCTargetDesc/AArch64MCLFIExpander.cpp     | 176 +++++++++++++++++-
 llvm/test/MC/AArch64/LFI/lse.s                |  15 ++
 llvm/test/MC/AArch64/LFI/mem.s                |  33 ++++
 3 files changed, 218 insertions(+), 6 deletions(-)
 create mode 100644 llvm/test/MC/AArch64/LFI/lse.s

diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
index 7de5d80408e16..ebf11f7f3e8e8 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
@@ -376,6 +376,21 @@ void AArch64::AArch64MCLFIExpander::expandLoadStoreRoW(
   }
 }
 
+static std::optional<MemInstInfo> getAtomicLoadStoreInfo(const MCInst &Inst);
+
+static std::optional<MemInstInfo> getMemInstInfo(const MCInst &Inst) {
+  auto MII = getLoadInfo(Inst);
+  if (MII.has_value())
+    return MII;
+  MII = getStoreInfo(Inst);
+  if (MII.has_value())
+    return MII;
+  MII = getAtomicLoadStoreInfo(Inst);
+  if (MII.has_value())
+    return MII;
+  return std::nullopt;
+}
+
 void AArch64::AArch64MCLFIExpander::expandLoadStore(
     const MCInst &Inst, MCStreamer &Out, const MCSubtargetInfo &STI) {
   if (hasFeature(FeatureBitset({AArch64::FeatureLFIJumps}), STI))
@@ -384,12 +399,9 @@ void AArch64::AArch64MCLFIExpander::expandLoadStore(
       !mayStore(Inst))
     return Out.emitInstruction(Inst, STI);
 
-  auto MII = getLoadInfo(Inst);
-  if (!MII.has_value()) {
-    MII = getStoreInfo(Inst);
-    if (!MII.has_value())
-      return Out.emitInstruction(Inst, STI);
-  }
+  auto MII = getMemInstInfo(Inst);
+  if (!MII.has_value())
+    return Out.emitInstruction(Inst, STI);
 
   // Stack accesses without a register offset don't need rewriting.
   if (Inst.getOperand(MII->BaseRegIdx).getReg() == AArch64::SP) {
@@ -1479,3 +1491,155 @@ static unsigned getPrePostScale(unsigned Op) {
   }
   return 0;
 }
+
+static std::optional<MemInstInfo> getAtomicLoadStoreInfo(const MCInst &Inst) {
+  int DestRegIdx;
+  int BaseRegIdx;
+  const int OffsetIdx = -1;
+  const bool IsPrePost = false;
+  bool IsPair = false;
+  // (ST|LD)OPRegister: ld*, st*
+  // CompareAndSwap(Pair)?(Unprvileged)?
+  // Swap(LSUI)?
+  // StoreRelease
+  // LoadAcquire
+  switch (Inst.getOpcode()) {
+  // Compare-and-swap CAS(A)?(L)?(B|H|W|X)
+  // MIR: Rs = op Rs, Rt, [Xn]
+  case AArch64::CASB:
+  case AArch64::CASH:
+  case AArch64::CASW:
+  case AArch64::CASX:
+  case AArch64::CASAB:
+  case AArch64::CASAH:
+  case AArch64::CASAW:
+  case AArch64::CASAX:
+  case AArch64::CASALB:
+  case AArch64::CASALH:
+  case AArch64::CASALW:
+  case AArch64::CASALX:
+  case AArch64::CASLB:
+  case AArch64::CASLH:
+  case AArch64::CASLW:
+  case AArch64::CASLX:
+    DestRegIdx = 2;
+    BaseRegIdx = 3;
+    break;
+  // rs1_rs2 = op rs1_rs2, rt1_rt2, [Xn]
+  case AArch64::CASPW:
+  case AArch64::CASPX:
+  case AArch64::CASPAW:
+  case AArch64::CASPAX:
+  case AArch64::CASPALW:
+  case AArch64::CASPALX:
+  case AArch64::CASPLW:
+  case AArch64::CASPLX:
+    DestRegIdx = 2;
+    BaseRegIdx = 3;
+    IsPair = true;
+    break;
+  // swap SWP(A)?(L)?(B|H|W|X)
+  // MIR: op Rs, Rt, [Xn]
+  case AArch64::SWPB:
+  case AArch64::SWPH:
+  case AArch64::SWPW:
+  case AArch64::SWPX:
+  case AArch64::SWPAB:
+  case AArch64::SWPAH:
+  case AArch64::SWPAW:
+  case AArch64::SWPAX:
+  case AArch64::SWPALB:
+  case AArch64::SWPALH:
+  case AArch64::SWPALW:
+  case AArch64::SWPALX:
+  case AArch64::SWPLB:
+  case AArch64::SWPLH:
+  case AArch64::SWPLW:
+  case AArch64::SWPLX:
+    DestRegIdx = 1;
+    BaseRegIdx = 2;
+    break;
+  // LD(ADD|CLR|EOR|SET)(A)?(L)?(B|H|W|X)
+  // op Rs, Rt, [Xn]
+  // if Rt == xzr, LD* aliases to ST*
+  case AArch64::LDADDB:
+  case AArch64::LDADDH:
+  case AArch64::LDADDX:
+  case AArch64::LDADDW:
+  case AArch64::LDADDAB:
+  case AArch64::LDADDAH:
+  case AArch64::LDADDAW:
+  case AArch64::LDADDAX:
+  case AArch64::LDADDALB:
+  case AArch64::LDADDALH:
+  case AArch64::LDADDALW:
+  case AArch64::LDADDALX:
+  case AArch64::LDADDLB:
+  case AArch64::LDADDLH:
+  case AArch64::LDADDLX:
+  case AArch64::LDADDLW:
+    DestRegIdx = 1;
+    BaseRegIdx = 2;
+    break;
+  case AArch64::LDCLRB:
+  case AArch64::LDCLRH:
+  case AArch64::LDCLRX:
+  case AArch64::LDCLRW:
+  case AArch64::LDCLRAB:
+  case AArch64::LDCLRAH:
+  case AArch64::LDCLRAW:
+  case AArch64::LDCLRAX:
+  case AArch64::LDCLRALB:
+  case AArch64::LDCLRALH:
+  case AArch64::LDCLRALW:
+  case AArch64::LDCLRALX:
+  case AArch64::LDCLRLB:
+  case AArch64::LDCLRLH:
+  case AArch64::LDCLRLX:
+  case AArch64::LDCLRLW:
+    DestRegIdx = 1;
+    BaseRegIdx = 2;
+    break;
+  case AArch64::LDEORB:
+  case AArch64::LDEORH:
+  case AArch64::LDEORX:
+  case AArch64::LDEORW:
+  case AArch64::LDEORAB:
+  case AArch64::LDEORAH:
+  case AArch64::LDEORAW:
+  case AArch64::LDEORAX:
+  case AArch64::LDEORALB:
+  case AArch64::LDEORALH:
+  case AArch64::LDEORALW:
+  case AArch64::LDEORALX:
+  case AArch64::LDEORLB:
+  case AArch64::LDEORLH:
+  case AArch64::LDEORLX:
+  case AArch64::LDEORLW:
+    DestRegIdx = 1;
+    BaseRegIdx = 2;
+    break;
+  case AArch64::LDSETB:
+  case AArch64::LDSETH:
+  case AArch64::LDSETX:
+  case AArch64::LDSETW:
+  case AArch64::LDSETAB:
+  case AArch64::LDSETAH:
+  case AArch64::LDSETAW:
+  case AArch64::LDSETAX:
+  case AArch64::LDSETALB:
+  case AArch64::LDSETALH:
+  case AArch64::LDSETALW:
+  case AArch64::LDSETALX:
+  case AArch64::LDSETLB:
+  case AArch64::LDSETLH:
+  case AArch64::LDSETLX:
+  case AArch64::LDSETLW:
+    DestRegIdx = 1;
+    BaseRegIdx = 2;
+    break;
+  default:
+    return std::nullopt;
+  }
+  return MemInstInfo{DestRegIdx, BaseRegIdx, OffsetIdx, IsPrePost, IsPair};
+}
diff --git a/llvm/test/MC/AArch64/LFI/lse.s b/llvm/test/MC/AArch64/LFI/lse.s
new file mode 100644
index 0000000000000..bda6c45c7a850
--- /dev/null
+++ b/llvm/test/MC/AArch64/LFI/lse.s
@@ -0,0 +1,15 @@
+// RUN: llvm-mc -filetype asm -triple aarch64_lfi %s | FileCheck %s
+
+.arch_extension lse
+
+ldadd x0, x1, [x2]
+// CHECK:      add x28, x27, w2, uxtw
+// CHECK-NEXT: ldadd x0, x1, [x28]
+
+swpal w0, w0, [x1]
+// CHECK:      add x28, x27, w1, uxtw
+// CHECK-NEXT: swpal w0, w0, [x28]
+
+caspal w0, w1, w2, w3, [x4]
+// CHECK:      add x28, x27, w4, uxtw
+// CHECK-NEXT: caspal w0, w1, w2, w3, [x28]
diff --git a/llvm/test/MC/AArch64/LFI/mem.s b/llvm/test/MC/AArch64/LFI/mem.s
index 469b068a786f2..6d87d3b29ba8f 100644
--- a/llvm/test/MC/AArch64/LFI/mem.s
+++ b/llvm/test/MC/AArch64/LFI/mem.s
@@ -146,3 +146,36 @@ st2 { v1.8b, v2.8b }, [x14], #16
 st2 { v1.8b, v2.8b }, [x14]
 // CHECK:      add x28, x27, w14, uxtw
 // CHECK-NEXT: st2 { v1.8b, v2.8b }, [x28]
+
+ld1 { v0.s }[1], [x8]
+// CHECK:      add x28, x27, w8, uxtw
+// CHECK-NEXT: ld1 { v0.s }[1], [x28]
+
+ld1r { v3.2d }, [x9]
+// CHECK:      add x28, x27, w9, uxtw
+// CHECK-NEXT: ld1r { v3.2d }, [x28]
+
+ld1 { v0.s }[1], [x8], x10
+// CHECK:      add x28, x27, w8, uxtw
+// CHECK-NEXT: ld1 { v0.s }[1], [x28]
+// CHECK-NEXT: add x8, x8, x10
+
+ldaxr x0, [x2]
+// CHECK:      add x28, x27, w2, uxtw
+// CHECK-NEXT: ldaxr x0, [x28]
+
+stlxr w15, w17, [x1]
+// CHECK:      add x28, x27, w1, uxtw
+// CHECK-NEXT: stlxr w15, w17, [x28]
+
+ldr w4, [sp, w3, uxtw #2]
+// CHECK:      add x26, sp, w3, uxtw #2
+// CHECK-NEXT: ldr w4, [x27, w26, uxtw]
+
+stxrb w11, w10, [x8]
+// CHECK:      add x28, x27, w8, uxtw
+// CHECK-NEXT: stxrb w11, w10, [x28]
+
+ldr x0, [x0, :got_lo12:x]
+// CHECK:      add x28, x27, w0, uxtw
+// CHECK-NEXT: ldr x0, [x28, :got_lo12:x]