diff --git a/clang/lib/Basic/Targets/AArch64.cpp b/clang/lib/Basic/Targets/AArch64.cpp index cabda0a1323a3..a3464a9b71d9d 100644 --- a/clang/lib/Basic/Targets/AArch64.cpp +++ b/clang/lib/Basic/Targets/AArch64.cpp @@ -413,6 +413,10 @@ void AArch64TargetInfo::getTargetDefines(const LangOptions &Opts, Builder.defineMacro("__aarch64__"); } + if (getTriple().isAArch64LFI()) { + Builder.defineMacro("__LFI__"); + } + // Inline assembly supports AArch64 flag outputs. Builder.defineMacro("__GCC_ASM_FLAG_OUTPUTS__"); diff --git a/clang/lib/Basic/Targets/X86.cpp b/clang/lib/Basic/Targets/X86.cpp index 40ad8fd9a0967..10c5b9cd6543a 100644 --- a/clang/lib/Basic/Targets/X86.cpp +++ b/clang/lib/Basic/Targets/X86.cpp @@ -557,6 +557,10 @@ void X86TargetInfo::getTargetDefines(const LangOptions &Opts, DefineStd(Builder, "i386", Opts); } + if (getTriple().isX8664LFI()) { + Builder.defineMacro("__LFI__"); + } + Builder.defineMacro("__SEG_GS"); Builder.defineMacro("__SEG_FS"); Builder.defineMacro("__seg_gs", "__attribute__((address_space(256)))"); diff --git a/clang/lib/Driver/CMakeLists.txt b/clang/lib/Driver/CMakeLists.txt index 5bdb6614389cf..9fd7beb4b29a7 100644 --- a/clang/lib/Driver/CMakeLists.txt +++ b/clang/lib/Driver/CMakeLists.txt @@ -64,6 +64,7 @@ add_clang_library(clangDriver ToolChains/Hexagon.cpp ToolChains/HLSL.cpp ToolChains/Hurd.cpp + ToolChains/LFILinux.cpp ToolChains/Linux.cpp ToolChains/MipsLinux.cpp ToolChains/MinGW.cpp diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp index 87855fdb79971..6183798031d96 100644 --- a/clang/lib/Driver/Driver.cpp +++ b/clang/lib/Driver/Driver.cpp @@ -28,6 +28,7 @@ #include "ToolChains/Haiku.h" #include "ToolChains/Hexagon.h" #include "ToolChains/Hurd.h" +#include "ToolChains/LFILinux.h" #include "ToolChains/Lanai.h" #include "ToolChains/Linux.h" #include "ToolChains/MSP430.h" @@ -6654,6 +6655,8 @@ const ToolChain &Driver::getToolChain(const ArgList &Args, TC = std::make_unique(*this, Target, Args); else if (Target.isOHOSFamily()) TC = std::make_unique(*this, Target, Args); + else if (Target.isLFI()) + TC = std::make_unique(*this, Target, Args); else TC = std::make_unique(*this, Target, Args); break; diff --git a/clang/lib/Driver/ToolChains/LFILinux.cpp b/clang/lib/Driver/ToolChains/LFILinux.cpp new file mode 100644 index 0000000000000..7ab2372756af9 --- /dev/null +++ b/clang/lib/Driver/ToolChains/LFILinux.cpp @@ -0,0 +1,29 @@ +//===-- LFILinux.cpp - LFI ToolChain Implementations ------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "LFILinux.h" +#include "clang/Driver/Driver.h" +#include "clang/Driver/Options.h" + +using namespace clang::driver::toolchains; +using namespace llvm::opt; + +void LFILinuxToolChain::AddCXXStdlibLibArgs(const ArgList &Args, + ArgStringList &CmdArgs) const { + switch (GetCXXStdlibType(Args)) { + case ToolChain::CST_Libcxx: + CmdArgs.push_back("-lc++"); + if (Args.hasArg(options::OPT_fexperimental_library)) + CmdArgs.push_back("-lc++experimental"); + CmdArgs.push_back("-lc++abi"); + break; + case ToolChain::CST_Libstdcxx: + CmdArgs.push_back("-lstdc++"); + break; + } +} diff --git a/clang/lib/Driver/ToolChains/LFILinux.h b/clang/lib/Driver/ToolChains/LFILinux.h new file mode 100644 index 0000000000000..e85f74803c1ce --- /dev/null +++ b/clang/lib/Driver/ToolChains/LFILinux.h @@ -0,0 +1,35 @@ +//===--- LFILinux.h - LFI ToolChain Implementations -------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_LFI_LINUX_H +#define LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_LFI_LINUX_H + +#include "Linux.h" + +namespace clang { +namespace driver { +namespace toolchains { + +class LLVM_LIBRARY_VISIBILITY LFILinuxToolChain : public Linux { +public: + LFILinuxToolChain(const Driver &D, const llvm::Triple &Triple, + const llvm::opt::ArgList &Args) + : Linux(D, Triple, Args) { + ExtraOpts.push_back("-z"); + ExtraOpts.push_back("separate-code"); + } + + void AddCXXStdlibLibArgs(const llvm::opt::ArgList &Args, + llvm::opt::ArgStringList &CmdArgs) const override; +}; + +} // end namespace toolchains +} // end namespace driver +} // end namespace clang + +#endif // LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_LFI_LINUX_H diff --git a/clang/tools/driver/cc1as_main.cpp b/clang/tools/driver/cc1as_main.cpp index 7fe97cc6e6ace..cb2f6bdb9c096 100644 --- a/clang/tools/driver/cc1as_main.cpp +++ b/clang/tools/driver/cc1as_main.cpp @@ -27,6 +27,7 @@ #include "llvm/MC/MCCodeEmitter.h" #include "llvm/MC/MCContext.h" #include "llvm/MC/MCInstrInfo.h" +#include "llvm/MC/MCLFI.h" #include "llvm/MC/MCObjectFileInfo.h" #include "llvm/MC/MCObjectWriter.h" #include "llvm/MC/MCParser/MCAsmParser.h" @@ -577,6 +578,8 @@ static bool ExecuteAssemblerImpl(AssemblerInvocation &Opts, Str.reset(TheTarget->createMCObjectStreamer( T, Ctx, std::move(MAB), std::move(OW), std::move(CE), *STI)); Str.get()->initSections(Opts.NoExecStack, *STI); + if (T.isLFI()) + initializeLFIMCStreamer(*Str.get(), Ctx, T); if (T.isOSBinFormatMachO() && T.isOSDarwin()) { Triple *TVT = Opts.DarwinTargetVariantTriple ? &*Opts.DarwinTargetVariantTriple diff --git a/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake b/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake index ee5be276f3df7..2e97293156dbe 100644 --- a/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake +++ b/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake @@ -1,8 +1,8 @@ -set(ARM64 aarch64) +set(ARM64 aarch64 aarch64_lfi) set(ARM32 arm armhf) set(HEXAGON hexagon) set(X86 i386) -set(X86_64 x86_64) +set(X86_64 x86_64 x86_64_lfi) set(LOONGARCH64 loongarch64) set(MIPS32 mips mipsel) set(MIPS64 mips64 mips64el) diff --git a/compiler-rt/cmake/builtin-config-ix.cmake b/compiler-rt/cmake/builtin-config-ix.cmake index b1bde47ec8555..d3b9ed487db61 100644 --- a/compiler-rt/cmake/builtin-config-ix.cmake +++ b/compiler-rt/cmake/builtin-config-ix.cmake @@ -58,12 +58,12 @@ else() endif() set(AMDGPU amdgcn) -set(ARM64 aarch64) +set(ARM64 aarch64 aarch64_lfi) set(ARM32 arm armhf armv4t armv5te armv6 armv6m armv7m armv7em armv7 armv7s armv7k armv8m.base armv8m.main armv8.1m.main) set(AVR avr) set(HEXAGON hexagon) set(X86 i386) -set(X86_64 x86_64) +set(X86_64 x86_64 x86_64_lfi) set(LOONGARCH64 loongarch64) set(MIPS32 mips mipsel) set(MIPS64 mips64 mips64el) diff --git a/compiler-rt/lib/builtins/CMakeLists.txt b/compiler-rt/lib/builtins/CMakeLists.txt index 19316c52d12ce..a69e7e35848ca 100644 --- a/compiler-rt/lib/builtins/CMakeLists.txt +++ b/compiler-rt/lib/builtins/CMakeLists.txt @@ -639,6 +639,8 @@ endif() set(amdgcn_SOURCES ${GENERIC_SOURCES}) +set(x86_64_lfi_SOURCES ${x86_64_SOURCES}) + set(armv4t_SOURCES ${arm_min_SOURCES}) set(armv5te_SOURCES ${arm_min_SOURCES}) set(armv6_SOURCES ${arm_min_SOURCES}) @@ -649,6 +651,7 @@ set(armv7k_SOURCES ${arm_SOURCES}) set(arm64_SOURCES ${aarch64_SOURCES}) set(arm64e_SOURCES ${aarch64_SOURCES}) set(arm64_32_SOURCES ${aarch64_SOURCES}) +set(aarch64_lfi_SOURCES ${aarch64_SOURCES}) # macho_embedded archs set(armv6m_SOURCES ${thumb1_SOURCES}) diff --git a/compiler-rt/lib/hwasan/hwasan_interceptors.cpp b/compiler-rt/lib/hwasan/hwasan_interceptors.cpp index c10b5c158548e..6b10b9ed246ae 100644 --- a/compiler-rt/lib/hwasan/hwasan_interceptors.cpp +++ b/compiler-rt/lib/hwasan/hwasan_interceptors.cpp @@ -382,7 +382,9 @@ InternalLongjmp(__hw_register_buf env, int retval) { "ldp x21, x22, [%0, #2<<3];" "ldp x23, x24, [%0, #4<<3];" "ldp x25, x26, [%0, #6<<3];" +#ifndef __LFI__ "ldp x27, x28, [%0, #8<<3];" +#endif "ldp x29, x30, [%0, #10<<3];" "ldp d8, d9, [%0, #14<<3];" "ldp d10, d11, [%0, #16<<3];" @@ -407,7 +409,9 @@ InternalLongjmp(__hw_register_buf env, int retval) { "mov (1*8)(%0),%%rbp;" "mov (2*8)(%0),%%r12;" "mov (3*8)(%0),%%r13;" +#ifndef __LFI__ "mov (4*8)(%0),%%r14;" +#endif "mov (5*8)(%0),%%r15;" "mov (6*8)(%0),%%rsp;" "mov (7*8)(%0),%%rdx;" diff --git a/compiler-rt/lib/orc/sysv_reenter.x86-64.S b/compiler-rt/lib/orc/sysv_reenter.x86-64.S index 0a36280f1d1f8..a59ee0fd701e6 100644 --- a/compiler-rt/lib/orc/sysv_reenter.x86-64.S +++ b/compiler-rt/lib/orc/sysv_reenter.x86-64.S @@ -61,7 +61,9 @@ __orc_rt_sysv_reenter: movq %rax, 8(%rbp) fxrstor64 (%rsp) movq -112(%rbp), %r15 +#ifndef __LFI__ movq -104(%rbp), %r14 +#endif movq -96(%rbp), %r13 movq -88(%rbp), %r12 movq -80(%rbp), %r11 diff --git a/compiler-rt/lib/xray/xray_trampoline_x86_64.S b/compiler-rt/lib/xray/xray_trampoline_x86_64.S index 0f480547b52cc..93a12e1340fa2 100644 --- a/compiler-rt/lib/xray/xray_trampoline_x86_64.S +++ b/compiler-rt/lib/xray/xray_trampoline_x86_64.S @@ -91,7 +91,9 @@ movq 32(%rsp), %r11 movq 24(%rsp), %r12 movq 16(%rsp), %r13 +#ifndef __LFI__ movq 8(%rsp), %r14 +#endif movq 0(%rsp), %r15 addq $240, %rsp CFI_ADJUST_CFA_OFFSET(-240) diff --git a/libunwind/src/UnwindRegistersRestore.S b/libunwind/src/UnwindRegistersRestore.S index 1702d016c368b..12507c792998a 100644 --- a/libunwind/src/UnwindRegistersRestore.S +++ b/libunwind/src/UnwindRegistersRestore.S @@ -106,7 +106,9 @@ DEFINE_LIBUNWIND_FUNCTION(__libunwind_Registers_x86_64_jumpto) movq 88(%rdi), %r11 movq 96(%rdi), %r12 movq 104(%rdi), %r13 +#ifndef __LFI__ movq 112(%rdi), %r14 +#endif movq 120(%rdi), %r15 # skip rflags # skip cs @@ -655,8 +657,13 @@ DEFINE_LIBUNWIND_FUNCTION(__libunwind_Registers_arm64_jumpto) ldp x20,x21, [x0, #0x0A0] ldp x22,x23, [x0, #0x0B0] ldp x24,x25, [x0, #0x0C0] +#ifdef __LFI__ + ldp x26,xzr, [x0, #0x0D0] + ldp xzr,x29, [x0, #0x0E0] +#else ldp x26,x27, [x0, #0x0D0] ldp x28,x29, [x0, #0x0E0] +#endif ldr x30, [x0, #0x100] // restore pc into lr #if defined(__ARM_FP) && __ARM_FP != 0 ldp d0, d1, [x0, #0x110] diff --git a/lld/ELF/Writer.cpp b/lld/ELF/Writer.cpp index 0db922b07aabf..39848de8755a4 100644 --- a/lld/ELF/Writer.cpp +++ b/lld/ELF/Writer.cpp @@ -2901,7 +2901,7 @@ template void Writer::writeTrapInstr() { if (p->p_type == PT_LOAD && (p->p_flags & PF_X)) fillTrap( ctx.target->trapInstr, - ctx.bufferStart + alignDown(p->firstSec->offset + p->p_filesz, 4), + ctx.bufferStart + p->firstSec->offset, ctx.bufferStart + alignToPowerOf2(p->firstSec->offset + p->p_filesz, ctx.arg.maxPageSize)); diff --git a/llvm/include/llvm/MC/MCLFI.h b/llvm/include/llvm/MC/MCLFI.h new file mode 100644 index 0000000000000..544bda4c7c8d6 --- /dev/null +++ b/llvm/include/llvm/MC/MCLFI.h @@ -0,0 +1,22 @@ +//===- MCLFI.h - LFI-specific code for MC -----------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +// This file was written by the Native Client authors, modified for LFI. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Support/CommandLine.h" + +namespace llvm { + +class MCContext; +class MCStreamer; +class Triple; + +void initializeLFIMCStreamer(MCStreamer &Streamer, MCContext &Ctx, + const Triple &TheTriple); + +} // namespace llvm diff --git a/llvm/include/llvm/MC/MCLFIExpander.h b/llvm/include/llvm/MC/MCLFIExpander.h new file mode 100644 index 0000000000000..d3861c6455793 --- /dev/null +++ b/llvm/include/llvm/MC/MCLFIExpander.h @@ -0,0 +1,79 @@ +//===- llvm/MC/MCLFIExpander.h ----------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +// This file was written by the Native Client authors, modified for LFI. +// +//===----------------------------------------------------------------------===// +// +// This file declares the MCLFIExpander class. This is an abstract +// class that encapsulates the expansion logic for MCInsts, and holds +// state such as available scratch registers. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_MC_MCLFIEXPANDER_H +#define LLVM_MC_MCLFIEXPANDER_H + +#include "llvm/ADT/SmallVector.h" +#include "llvm/MC/MCContext.h" +#include "llvm/MC/MCInstrInfo.h" +#include "llvm/MC/MCRegisterInfo.h" + +namespace llvm { +class MCInst; +class MCSubtargetInfo; +class MCStreamer; + +class MCLFIExpander { +private: + SmallVector ScratchRegs; + bool Enabled = true; + +protected: + MCContext &Ctx; + std::unique_ptr InstInfo; + std::unique_ptr RegInfo; + void invalidateScratchRegs(const MCInst &Inst); + MCRegister getScratchReg(int index); + unsigned numScratchRegs() const; + virtual bool isValidScratchRegister(MCRegister Reg) const = 0; + +public: + MCLFIExpander(MCContext &Ctx, std::unique_ptr &&RI, + std::unique_ptr &&II) + : Ctx(Ctx), InstInfo(std::move(II)), RegInfo(std::move(RI)) {} + + void Error(const MCInst &Inst, const char msg[]); + + bool addScratchReg(MCRegister Reg); + void clearScratchRegs(); + + void disable(); + void enable(); + bool isEnabled(); + + bool isPseudo(const MCInst &Inst) const; + + bool mayAffectControlFlow(const MCInst &Inst) const; + bool isCall(const MCInst &Inst) const; + bool isBranch(const MCInst &Inst) const; + bool isIndirectBranch(const MCInst &Inst) const; + bool isReturn(const MCInst &Inst) const; + bool isVariadic(const MCInst &Inst) const; + + bool mayLoad(const MCInst &Inst) const; + bool mayStore(const MCInst &Inst) const; + + bool mayModifyRegister(const MCInst &Inst, MCRegister Reg) const; + bool explicitlyModifiesRegister(const MCInst &Inst, MCRegister Reg) const; + + virtual ~MCLFIExpander() = default; + virtual bool expandInst(const MCInst &Inst, MCStreamer &Out, + const MCSubtargetInfo &STI) = 0; +}; + +} // namespace llvm +#endif diff --git a/llvm/include/llvm/MC/MCStreamer.h b/llvm/include/llvm/MC/MCStreamer.h index 558b14cebfd3d..96b6aff836932 100644 --- a/llvm/include/llvm/MC/MCStreamer.h +++ b/llvm/include/llvm/MC/MCStreamer.h @@ -21,6 +21,7 @@ #include "llvm/MC/MCDwarf.h" #include "llvm/MC/MCFragment.h" #include "llvm/MC/MCLinkerOptimizationHint.h" +#include "llvm/MC/MCLFIExpander.h" #include "llvm/MC/MCPseudoProbe.h" #include "llvm/MC/MCWinEH.h" #include "llvm/Support/Error.h" @@ -276,6 +277,8 @@ class MCStreamer { /// Returns true if the .cv_loc directive is in the right section. bool checkCVLocSection(unsigned FuncId, unsigned FileNo, SMLoc Loc); + std::unique_ptr LFIExpander; + public: MCStreamer(const MCStreamer &) = delete; MCStreamer &operator=(const MCStreamer &) = delete; @@ -293,6 +296,10 @@ class MCStreamer { return StartTokLocPtr ? *StartTokLocPtr : SMLoc(); } + void setLFIExpander(MCLFIExpander *Exp) { LFIExpander.reset(Exp); } + + MCLFIExpander *getLFIExpander() { return LFIExpander.get(); } + /// State management /// virtual void reset(); diff --git a/llvm/include/llvm/MC/TargetRegistry.h b/llvm/include/llvm/MC/TargetRegistry.h index 42d510c17bce3..abeb1d4405be7 100644 --- a/llvm/include/llvm/MC/TargetRegistry.h +++ b/llvm/include/llvm/MC/TargetRegistry.h @@ -46,6 +46,7 @@ class MCDisassembler; class MCInstPrinter; class MCInstrAnalysis; class MCInstrInfo; +class MCLFIExpander; class MCObjectWriter; class MCRegisterInfo; class MCRelocationInfo; @@ -230,6 +231,10 @@ class Target { mca::InstrumentManager *(*)(const MCSubtargetInfo &STI, const MCInstrInfo &MCII); + using MCLFIExpanderCtorTy = MCLFIExpander *(*)( + MCStreamer &S, std::unique_ptr &&RegInfo, + std::unique_ptr &&InstInfo); + private: /// Next - The next registered target in the linked list, maintained by the /// TargetRegistry. @@ -340,6 +345,10 @@ class Target { /// InstrumentManager, if registered (default = nullptr). InstrumentManagerCtorTy InstrumentManagerCtorFn = nullptr; + // MCLFIExpanderCtorFn - Construction function for this target's + // MCLFIExpander, if registered. + MCLFIExpanderCtorTy MCLFIExpanderCtorFn; + public: Target() = default; @@ -538,6 +547,13 @@ class Target { const MCSubtargetInfo &STI, bool, bool, bool) const; + void createMCLFIExpander(MCStreamer &S, + std::unique_ptr &&RegInfo, + std::unique_ptr &&InstInfo) const { + if (MCLFIExpanderCtorFn) + MCLFIExpanderCtorFn(S, std::move(RegInfo), std::move(InstInfo)); + } + MCStreamer *createAsmStreamer(MCContext &Ctx, std::unique_ptr OS, MCInstPrinter *IP, @@ -1008,6 +1024,11 @@ struct TargetRegistry { T.InstrumentManagerCtorFn = Fn; } + static void RegisterMCLFIExpander(Target &T, + Target::MCLFIExpanderCtorTy Fn) { + T.MCLFIExpanderCtorFn = Fn; + } + /// @} }; diff --git a/llvm/include/llvm/TargetParser/Triple.h b/llvm/include/llvm/TargetParser/Triple.h index 7d67966d17256..fa04dbbf3eec3 100644 --- a/llvm/include/llvm/TargetParser/Triple.h +++ b/llvm/include/llvm/TargetParser/Triple.h @@ -148,6 +148,8 @@ class Triple { AArch64SubArch_arm64e, AArch64SubArch_arm64ec, + AArch64SubArch_lfi, + KalimbaSubArch_v3, KalimbaSubArch_v4, KalimbaSubArch_v5, @@ -165,6 +167,8 @@ class Triple { SPIRVSubArch_v15, SPIRVSubArch_v16, + X8664SubArch_lfi, + // DXIL sub-arch corresponds to its version. DXILSubArch_v1_0, DXILSubArch_v1_1, @@ -676,6 +680,21 @@ class Triple { getSubArch() == Triple::AArch64SubArch_arm64ec; } + bool isLFI() const { + return isAArch64LFI() || isX8664LFI(); + } + + /// Checks if we're targeting the AArch64 LFI subarch. + bool isAArch64LFI() const { + return getArch() == Triple::aarch64 && + getSubArch() == Triple::AArch64SubArch_lfi; + } + + bool isX8664LFI() const { + return getArch() == Triple::x86_64 && + getSubArch() == Triple::X8664SubArch_lfi; + } + bool isWindowsCoreCLREnvironment() const { return isOSWindows() && getEnvironment() == Triple::CoreCLR; } diff --git a/llvm/lib/MC/CMakeLists.txt b/llvm/lib/MC/CMakeLists.txt index e1d19196c8766..0456768b9cfaf 100644 --- a/llvm/lib/MC/CMakeLists.txt +++ b/llvm/lib/MC/CMakeLists.txt @@ -30,6 +30,8 @@ add_llvm_component_library(LLVMMC MCInstrAnalysis.cpp MCInstrDesc.cpp MCInstrInfo.cpp + MCLFI.cpp + MCLFIExpander.cpp MCLabel.cpp MCLinkerOptimizationHint.cpp MCMachOStreamer.cpp diff --git a/llvm/lib/MC/MCAsmStreamer.cpp b/llvm/lib/MC/MCAsmStreamer.cpp index dd8058c6d5cd8..00f49fd90ca3e 100644 --- a/llvm/lib/MC/MCAsmStreamer.cpp +++ b/llvm/lib/MC/MCAsmStreamer.cpp @@ -20,6 +20,7 @@ #include "llvm/MC/MCFixupKindInfo.h" #include "llvm/MC/MCInst.h" #include "llvm/MC/MCInstPrinter.h" +#include "llvm/MC/MCLFIExpander.h" #include "llvm/MC/MCObjectFileInfo.h" #include "llvm/MC/MCObjectWriter.h" #include "llvm/MC/MCPseudoProbe.h" @@ -2453,6 +2454,9 @@ void MCAsmStreamer::AddEncodingComment(const MCInst &Inst, void MCAsmStreamer::emitInstruction(const MCInst &Inst, const MCSubtargetInfo &STI) { + if (LFIExpander && LFIExpander->isEnabled() && LFIExpander->expandInst(Inst, *this, STI)) + return; + if (MAI->isAIX() && CurFrag) // Now that a machine instruction has been assembled into this section, make // a line entry for any .loc directive that has been seen. diff --git a/llvm/lib/MC/MCLFI.cpp b/llvm/lib/MC/MCLFI.cpp new file mode 100644 index 0000000000000..c3cfdc3e21b14 --- /dev/null +++ b/llvm/lib/MC/MCLFI.cpp @@ -0,0 +1,81 @@ +//===- lib/MC/MCLFI.cpp - LFI-specific MC implementation ------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +// This file was written by the Native Client authors, modified for LFI. +// +//===----------------------------------------------------------------------===// + +#include "llvm/MC/MCLFI.h" +#include "llvm/BinaryFormat/ELF.h" +#include "llvm/MC/MCContext.h" +#include "llvm/MC/MCLFIExpander.h" +#include "llvm/MC/MCSectionELF.h" +#include "llvm/MC/MCStreamer.h" +#include "llvm/MC/TargetRegistry.h" +#include "llvm/Support/Alignment.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/TargetParser/Triple.h" + +static const char NoteNamespace[] = "LFI"; + +namespace llvm { + +void initializeLFIMCStreamer(MCStreamer &Streamer, MCContext &Ctx, + const Triple &TheTriple) { + assert(TheTriple.isLFI()); + const char *NoteName; + const char *NoteArch; + Align BundleAlign = Align(1); + switch (TheTriple.getArch()) { + case Triple::aarch64: + NoteName = ".note.LFI.ABI.aarch64"; + NoteArch = "aarch64"; + break; + case Triple::x86_64: + NoteName = ".note.LFI.ABI.x86_64"; + NoteArch = "x86_64"; + BundleAlign = Align(32); + break; + default: + report_fatal_error("Unsupported architecture for LFI"); + } + + std::string Error; // empty + const Target *TheTarget = + TargetRegistry::lookupTarget(TheTriple.getTriple(), Error); + + // Create the Target specific MCLFIExpander + assert(TheTarget != nullptr); + TheTarget->createMCLFIExpander( + Streamer, + std::unique_ptr( + TheTarget->createMCRegInfo(TheTriple.getTriple())), + std::unique_ptr(TheTarget->createMCInstrInfo())); + + if (BundleAlign != Align(1)) + Streamer.emitBundleAlignMode(BundleAlign); + + // Emit an ELF Note section in its own COMDAT group which identifies LFI + // object files. + MCSectionELF *Note = Ctx.getELFSection(NoteName, ELF::SHT_NOTE, + ELF::SHF_ALLOC | ELF::SHF_GROUP, 0, + NoteName, /*IsComdat=*/true); + + Streamer.pushSection(); + Streamer.switchSection(Note); + Streamer.emitIntValue(strlen(NoteNamespace) + 1, 4); + Streamer.emitIntValue(strlen(NoteArch) + 1, 4); + Streamer.emitIntValue(ELF::NT_VERSION, 4); + Streamer.emitBytes(NoteNamespace); + Streamer.emitIntValue(0, 1); // NUL terminator + Streamer.emitValueToAlignment(Align(4)); + Streamer.emitBytes(NoteArch); + Streamer.emitIntValue(0, 1); // NUL terminator + Streamer.emitValueToAlignment(Align(4)); + Streamer.popSection(); +} + +} // namespace llvm diff --git a/llvm/lib/MC/MCLFIExpander.cpp b/llvm/lib/MC/MCLFIExpander.cpp new file mode 100644 index 0000000000000..539203c0b1476 --- /dev/null +++ b/llvm/lib/MC/MCLFIExpander.cpp @@ -0,0 +1,112 @@ +//===- MCLFIExpander.cpp ----------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +// This file was written by the Native Client authors, modified for LFI. +// +//===----------------------------------------------------------------------===// +// +// This file implements the MCLFIExpander class. This is a base +// class that encapsulates the expansion logic for MCInsts, and holds +// state such as available scratch registers. +// +//===----------------------------------------------------------------------===// + +#include "llvm/MC/MCLFIExpander.h" +#include "llvm/MC/MCInst.h" +#include "llvm/MC/MCInstrInfo.h" +#include "llvm/MC/MCRegisterInfo.h" + +namespace llvm { + +void MCLFIExpander::Error(const MCInst &Inst, const char msg[]) { + Ctx.reportError(Inst.getLoc(), msg); +} + +bool MCLFIExpander::addScratchReg(MCRegister Reg) { + if (!isValidScratchRegister(Reg)) + return true; + ScratchRegs.push_back(Reg); + return false; +} + +void MCLFIExpander::invalidateScratchRegs(const MCInst &Inst) { + // TODO: There are arch-specific special cases where this fails, e.g. + // xchg/cmpxchg + // TODO(zyedidia): investigate whether there are cases for AArch64. + const MCInstrDesc &Desc = InstInfo->get(Inst.getOpcode()); + for (auto I = ScratchRegs.begin(), E = ScratchRegs.end(); I != E; ++I) { + if (Desc.hasDefOfPhysReg(Inst, *I, *RegInfo)) + I = ScratchRegs.erase(I); + } +} + +void MCLFIExpander::clearScratchRegs() { ScratchRegs.clear(); } + +void MCLFIExpander::disable() { Enabled = false; } + +void MCLFIExpander::enable() { Enabled = true; } + +bool MCLFIExpander::isEnabled() { return Enabled; } + +MCRegister MCLFIExpander::getScratchReg(int index) { + assert(index >= 0 && static_cast(index) < numScratchRegs()); + return ScratchRegs[numScratchRegs() - index - 1]; +} + +unsigned MCLFIExpander::numScratchRegs() const { return ScratchRegs.size(); } + +bool MCLFIExpander::isPseudo(const MCInst &Inst) const { + return InstInfo->get(Inst.getOpcode()).isPseudo(); +} + +bool MCLFIExpander::mayAffectControlFlow(const MCInst &Inst) const { + return InstInfo->get(Inst.getOpcode()).mayAffectControlFlow(Inst, *RegInfo); +} + +bool MCLFIExpander::isCall(const MCInst &Inst) const { + return InstInfo->get(Inst.getOpcode()).isCall(); +} + +bool MCLFIExpander::isBranch(const MCInst &Inst) const { + return InstInfo->get(Inst.getOpcode()).isBranch(); +} + +bool MCLFIExpander::isIndirectBranch(const MCInst &Inst) const { + return InstInfo->get(Inst.getOpcode()).isIndirectBranch(); +} + +bool MCLFIExpander::isReturn(const MCInst &Inst) const { + return InstInfo->get(Inst.getOpcode()).isReturn(); +} + +bool MCLFIExpander::isVariadic(const MCInst &Inst) const { + return InstInfo->get(Inst.getOpcode()).isVariadic(); +} + +bool MCLFIExpander::mayLoad(const MCInst &Inst) const { + return InstInfo->get(Inst.getOpcode()).mayLoad(); +} + +bool MCLFIExpander::mayStore(const MCInst &Inst) const { + return InstInfo->get(Inst.getOpcode()).mayStore(); +} + +bool MCLFIExpander::mayModifyRegister(const MCInst &Inst, + MCRegister Reg) const { + return InstInfo->get(Inst.getOpcode()).hasDefOfPhysReg(Inst, Reg, *RegInfo); +} + +bool MCLFIExpander::explicitlyModifiesRegister(const MCInst &Inst, + MCRegister Reg) const { + const MCInstrDesc &Desc = InstInfo->get(Inst.getOpcode()); + for (int i = 0; i < Desc.NumDefs; ++i) { + if (Desc.operands()[i].OperandType == MCOI::OPERAND_REGISTER && + RegInfo->isSubRegisterEq(Reg, Inst.getOperand(i).getReg())) + return true; + } + return false; +} +} // namespace llvm diff --git a/llvm/lib/MC/MCObjectStreamer.cpp b/llvm/lib/MC/MCObjectStreamer.cpp index fff30955b2576..cc18cccf779fd 100644 --- a/llvm/lib/MC/MCObjectStreamer.cpp +++ b/llvm/lib/MC/MCObjectStreamer.cpp @@ -15,6 +15,7 @@ #include "llvm/MC/MCContext.h" #include "llvm/MC/MCDwarf.h" #include "llvm/MC/MCExpr.h" +#include "llvm/MC/MCLFIExpander.h" #include "llvm/MC/MCObjectFileInfo.h" #include "llvm/MC/MCObjectWriter.h" #include "llvm/MC/MCSection.h" @@ -349,9 +350,12 @@ void MCObjectStreamer::emitInstruction(const MCInst &Inst, void MCObjectStreamer::emitInstructionImpl(const MCInst &Inst, const MCSubtargetInfo &STI) { + MCSection *Sec = getCurrentSectionOnly(); + if (LFIExpander && LFIExpander->isEnabled() && LFIExpander->expandInst(Inst, *this, STI)) + return; + MCStreamer::emitInstruction(Inst, STI); - MCSection *Sec = getCurrentSectionOnly(); Sec->setHasInstructions(true); // Now that a machine instruction has been assembled into this section, make diff --git a/llvm/lib/MC/MCParser/AsmParser.cpp b/llvm/lib/MC/MCParser/AsmParser.cpp index bf952df1b2418..4f4d6fb9a7065 100644 --- a/llvm/lib/MC/MCParser/AsmParser.cpp +++ b/llvm/lib/MC/MCParser/AsmParser.cpp @@ -125,6 +125,7 @@ class AsmParser : public MCAsmParser { SourceMgr::DiagHandlerTy SavedDiagHandler; void *SavedDiagContext; std::unique_ptr PlatformParser; + std::unique_ptr LFIParser; SMLoc StartTokLoc; std::optional CFIStartProcLoc; @@ -769,6 +770,7 @@ extern MCAsmParserExtension *createCOFFAsmParser(); extern MCAsmParserExtension *createGOFFAsmParser(); extern MCAsmParserExtension *createXCOFFAsmParser(); extern MCAsmParserExtension *createWasmAsmParser(); +extern MCAsmParserExtension *createLFIAsmParser(MCLFIExpander *Exp); } // end namespace llvm @@ -776,7 +778,7 @@ enum { DEFAULT_ADDRSPACE = 0 }; AsmParser::AsmParser(SourceMgr &SM, MCContext &Ctx, MCStreamer &Out, const MCAsmInfo &MAI, unsigned CB = 0) - : Lexer(MAI), Ctx(Ctx), Out(Out), MAI(MAI), SrcMgr(SM), + : Lexer(MAI), Ctx(Ctx), Out(Out), MAI(MAI), SrcMgr(SM), LFIParser(nullptr), CurBuffer(CB ? CB : SM.getMainFileID()), MacrosEnabledFlag(true) { HadError = false; // Save the old handler. @@ -819,6 +821,10 @@ AsmParser::AsmParser(SourceMgr &SM, MCContext &Ctx, MCStreamer &Out, } PlatformParser->Initialize(*this); + if (Out.getLFIExpander()) { + LFIParser.reset(createLFIAsmParser(Out.getLFIExpander())); + LFIParser->Initialize(*this); + } initializeDirectiveKindMap(); initializeCVDefRangeTypeMap(); diff --git a/llvm/lib/MC/MCParser/CMakeLists.txt b/llvm/lib/MC/MCParser/CMakeLists.txt index d3fa2675a255e..93f0ff4ca0140 100644 --- a/llvm/lib/MC/MCParser/CMakeLists.txt +++ b/llvm/lib/MC/MCParser/CMakeLists.txt @@ -6,6 +6,7 @@ add_llvm_component_library(LLVMMCParser GOFFAsmParser.cpp DarwinAsmParser.cpp ELFAsmParser.cpp + LFIAsmParser.cpp MCAsmLexer.cpp MCAsmParser.cpp MCAsmParserExtension.cpp diff --git a/llvm/lib/MC/MCParser/LFIAsmParser.cpp b/llvm/lib/MC/MCParser/LFIAsmParser.cpp new file mode 100644 index 0000000000000..bf91e3f6ae162 --- /dev/null +++ b/llvm/lib/MC/MCParser/LFIAsmParser.cpp @@ -0,0 +1,109 @@ +//===- LFIAsmParser.cpp - LFI Assembly Parser -----------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file was written by the Native Client authors, modified for LFI. +// +//===----------------------------------------------------------------------===// + +#include "llvm/MC/MCLFIExpander.h" +#include "llvm/MC/MCParser/MCAsmLexer.h" +#include "llvm/MC/MCParser/MCAsmParserExtension.h" +#include "llvm/MC/MCParser/MCTargetAsmParser.h" +#include "llvm/MC/MCRegister.h" +#include "llvm/MC/MCStreamer.h" +#include "llvm/Support/SourceMgr.h" +#include "llvm/Support/raw_ostream.h" + +using namespace llvm; + +class LFIAsmParser : public MCAsmParserExtension { + MCLFIExpander *Expander; + template + void addDirectiveHandler(StringRef Directive) { + MCAsmParser::ExtensionDirectiveHandler Handler = + std::make_pair(this, HandleDirective); + + getParser().addDirectiveHandler(Directive, Handler); + } + +public: + LFIAsmParser(MCLFIExpander *Exp) : Expander(Exp) {} + void Initialize(MCAsmParser &Parser) override { + // Call the base implementation. + MCAsmParserExtension::Initialize(Parser); + addDirectiveHandler<&LFIAsmParser::ParseScratch>(".scratch"); + addDirectiveHandler<&LFIAsmParser::ParseUnscratch>(".scratch_clear"); + addDirectiveHandler<&LFIAsmParser::ParseExpandDisable>(".no_expand"); + addDirectiveHandler<&LFIAsmParser::ParseExpandEnable>(".expand"); + } + + /// ::= {.scratch} reg + bool ParseScratch(StringRef Directive, SMLoc Loc) { + getParser().checkForValidSection(); + MCRegister RegNo; + const char *kInvalidOptionError = + "expected register name after '.scratch' directive"; + + if (getLexer().isNot(AsmToken::EndOfStatement)) { + if (getParser().getTargetParser().parseRegister(RegNo, Loc, Loc)) + return Error(Loc, kInvalidOptionError); + + else if (getLexer().isNot(AsmToken::EndOfStatement)) + return Error(Loc, kInvalidOptionError); + } else { + return Error(Loc, kInvalidOptionError); + } + Lex(); + + if (Expander->addScratchReg(RegNo)) + return Error(Loc, "Register can't be used as a scratch register"); + return false; + } + + /// ::= {.scratch_clear} + bool ParseUnscratch(StringRef Directive, SMLoc Loc) { + getParser().checkForValidSection(); + if (getLexer().isNot(AsmToken::EndOfStatement)) + return TokError("unexpected token in '.scratch_clear' directive"); + Lex(); + + Expander->clearScratchRegs(); + + return false; + } + + /// ::= {.no_expand} + bool ParseExpandDisable(StringRef Directive, SMLoc Loc) { + getParser().checkForValidSection(); + if (getLexer().isNot(AsmToken::EndOfStatement)) + return TokError("unexpected token in '.no_expand' directive"); + Lex(); + + Expander->disable(); + + return false; + } + + /// ::= {.expand} + bool ParseExpandEnable(StringRef Directive, SMLoc Loc) { + getParser().checkForValidSection(); + if (getLexer().isNot(AsmToken::EndOfStatement)) + return TokError("unexpected token in '.expand' directive"); + Lex(); + + Expander->enable(); + + return false; + } +}; + +namespace llvm { +MCAsmParserExtension *createLFIAsmParser(MCLFIExpander *Exp) { + return new LFIAsmParser(Exp); +} +} // namespace llvm diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp index c6b4a219d201f..b899c73d529ad 100644 --- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp +++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp @@ -50,6 +50,7 @@ #include "llvm/MC/MCContext.h" #include "llvm/MC/MCInst.h" #include "llvm/MC/MCInstBuilder.h" +#include "llvm/MC/MCLFI.h" #include "llvm/MC/MCSectionELF.h" #include "llvm/MC/MCSectionMachO.h" #include "llvm/MC/MCStreamer.h" @@ -394,6 +395,10 @@ void AArch64AsmPrinter::emitStartOfAsmFile(Module &M) { // Emit AArch64 Build Attributes emitAttributes(BAFlags, PAuthABIPlatform, PAuthABIVersion, TS); + + if (TT.isLFI()) + initializeLFIMCStreamer(*OutStreamer.get(), OutContext, TT); + // Emit a .note.gnu.property section with the flags. TS->emitNoteSection(GNUFlags, PAuthABIPlatform, PAuthABIVersion); } diff --git a/llvm/lib/Target/AArch64/AArch64Features.td b/llvm/lib/Target/AArch64/AArch64Features.td index d47dcfe92ff19..e909152b20035 100644 --- a/llvm/lib/Target/AArch64/AArch64Features.td +++ b/llvm/lib/Target/AArch64/AArch64Features.td @@ -948,3 +948,16 @@ def FeatureHardenSlsNoComdat : SubtargetFeature<"harden-sls-nocomdat", // Only intended to be used by disassemblers. def FeatureAll : SubtargetFeature<"all", "IsAll", "true", "Enable all instructions">; + +//===----------------------------------------------------------------------===// +// LFI +//===----------------------------------------------------------------------===// + +def FeatureLFIStores : SubtargetFeature<"lfi-stores", "LFIStores", "true", + "Isolate only store+jump instructions">; + +def FeatureLFIJumps : SubtargetFeature<"lfi-jumps", "LFIJumps", "true", + "Isolate only jump instructions">; + +def FeatureLFITLSReg : SubtargetFeature<"lfi-tls-reg", "LFITLSReg", "true", + "Reserve a register for thread-local LFI data">; diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp index 0f2b969fba35c..b6f5e410e32d8 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp @@ -34,6 +34,7 @@ #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/RegisterScavenging.h" #include "llvm/CodeGen/StackMaps.h" +#include "llvm/CodeGen/TargetOpcodes.h" #include "llvm/CodeGen/TargetRegisterInfo.h" #include "llvm/CodeGen/TargetSubtargetInfo.h" #include "llvm/IR/DebugInfoMetadata.h" @@ -106,6 +107,7 @@ unsigned AArch64InstrInfo::getInstSizeInBytes(const MachineInstr &MI) const { unsigned NumBytes = 0; const MCInstrDesc &Desc = MI.getDesc(); + const auto &STI = MF->getSubtarget(); if (!MI.isBundle() && isTailCallReturnInst(MI)) { NumBytes = Desc.getSize() ? Desc.getSize() : 4; @@ -113,7 +115,6 @@ unsigned AArch64InstrInfo::getInstSizeInBytes(const MachineInstr &MI) const { if (!MFI->shouldSignReturnAddress(MF)) return NumBytes; - const auto &STI = MF->getSubtarget(); auto Method = STI.getAuthenticatedLRCheckMethod(*MF); NumBytes += AArch64PAuth::getCheckerSizeInBytes(Method); return NumBytes; @@ -124,8 +125,10 @@ unsigned AArch64InstrInfo::getInstSizeInBytes(const MachineInstr &MI) const { // Specific cases handle instructions of variable sizes switch (Desc.getOpcode()) { default: - if (Desc.getSize()) - return Desc.getSize(); + if (Desc.getSize()) { + NumBytes = Desc.getSize(); + break; + } // Anything not explicitly designated otherwise (i.e. pseudo-instructions // with fixed constant size but not specified in .td file) is a normal @@ -174,6 +177,14 @@ unsigned AArch64InstrInfo::getInstSizeInBytes(const MachineInstr &MI) const { break; } + if (STI.getTargetTriple().isAArch64LFI()) { + // Loads and stores are frequent and may be expanded to include an + // additional guard instruction, so we overestimate the size here to allow + // things like branch relaxation to be more accurate. + if (Desc.mayLoad() || Desc.mayStore()) + NumBytes += 4; + } + return NumBytes; } diff --git a/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp b/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp index 367f6b626b420..a52465c07840a 100644 --- a/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp @@ -403,6 +403,18 @@ AArch64RegisterInfo::explainReservedReg(const MachineFunction &MF, if (hasBasePointer(MF) && MCRegisterInfo::regsOverlap(PhysReg, AArch64::X19)) return std::string("X19 is used as the frame base pointer register."); + if (MF.getSubtarget().isAArch64LFI()) { + bool warn = false; + if (MCRegisterInfo::regsOverlap(PhysReg, AArch64::X28) || + MCRegisterInfo::regsOverlap(PhysReg, AArch64::X27) || + MCRegisterInfo::regsOverlap(PhysReg, AArch64::X26)) + warn = true; + + if (warn) + return std::string(AArch64InstPrinter::getRegisterName(PhysReg)) + + " is used as a reserved register by LFI."; + } + if (MF.getSubtarget().isWindowsArm64EC()) { bool warn = false; if (MCRegisterInfo::regsOverlap(PhysReg, AArch64::X13) || @@ -448,6 +460,19 @@ AArch64RegisterInfo::getStrictlyReservedRegs(const MachineFunction &MF) const { markSuperRegs(Reserved, i); } + if (MF.getSubtarget().isAArch64LFI()) { + markSuperRegs(Reserved, AArch64::W28); + markSuperRegs(Reserved, AArch64::W27); + markSuperRegs(Reserved, AArch64::W26); + auto TLSReg = FeatureBitset({AArch64::FeatureLFITLSReg}); + if ((MF.getSubtarget().getFeatureBits() & TLSReg) == TLSReg) + markSuperRegs(Reserved, AArch64::W25); + if (!MF.getProperties().hasProperty(MachineFunctionProperties::Property::NoVRegs)) { + markSuperRegs(Reserved, AArch64::LR); + markSuperRegs(Reserved, AArch64::W30); + } + } + for (size_t i = 0; i < AArch64::GPR32commonRegClass.getNumRegs(); ++i) { if (MF.getSubtarget().isXRegisterReserved(i)) markSuperRegs(Reserved, AArch64::GPR32commonRegClass.getRegister(i)); diff --git a/llvm/lib/Target/AArch64/AArch64Subtarget.h b/llvm/lib/Target/AArch64/AArch64Subtarget.h index c6eb77e3bc3ba..f7875ebdfb6a4 100644 --- a/llvm/lib/Target/AArch64/AArch64Subtarget.h +++ b/llvm/lib/Target/AArch64/AArch64Subtarget.h @@ -288,6 +288,9 @@ class AArch64Subtarget final : public AArch64GenSubtargetInfo { bool isTargetAndroid() const { return TargetTriple.isAndroid(); } bool isTargetFuchsia() const { return TargetTriple.isOSFuchsia(); } bool isWindowsArm64EC() const { return TargetTriple.isWindowsArm64EC(); } + bool isAArch64LFI() const { return TargetTriple.isAArch64LFI(); } + bool isLFIStores() const { return LFIStores; } + bool isLFIJumps() const { return LFIJumps; } bool isTargetCOFF() const { return TargetTriple.isOSBinFormatCOFF(); } bool isTargetELF() const { return TargetTriple.isOSBinFormatELF(); } diff --git a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp index 07f072446081a..846892b79cd64 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp @@ -369,6 +369,11 @@ AArch64TargetMachine::AArch64TargetMachine(const Target &T, const Triple &TT, this->Options.NoTrapAfterNoreturn = true; } + // Disable jump table compression since it may cause assembler errors after + // LFI instrumentation is applied. + if (TT.isAArch64LFI()) + EnableCompressJumpTables = false; + if (getMCAsmInfo()->usesWindowsCFI()) { // Unwinding can get confused if the last instruction in an // exception-handling region (function, funclet, try block, etc.) diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.cpp index 875b505549f0a..e6dd82fcea387 100644 --- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.cpp +++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.cpp @@ -420,366 +420,6 @@ static bool isTblTbxInstruction(unsigned Opcode, StringRef &Layout, } } -struct LdStNInstrDesc { - unsigned Opcode; - const char *Mnemonic; - const char *Layout; - int ListOperand; - bool HasLane; - int NaturalOffset; -}; - -static const LdStNInstrDesc LdStNInstInfo[] = { - { AArch64::LD1i8, "ld1", ".b", 1, true, 0 }, - { AArch64::LD1i16, "ld1", ".h", 1, true, 0 }, - { AArch64::LD1i32, "ld1", ".s", 1, true, 0 }, - { AArch64::LD1i64, "ld1", ".d", 1, true, 0 }, - { AArch64::LD1i8_POST, "ld1", ".b", 2, true, 1 }, - { AArch64::LD1i16_POST, "ld1", ".h", 2, true, 2 }, - { AArch64::LD1i32_POST, "ld1", ".s", 2, true, 4 }, - { AArch64::LD1i64_POST, "ld1", ".d", 2, true, 8 }, - { AArch64::LD1Rv16b, "ld1r", ".16b", 0, false, 0 }, - { AArch64::LD1Rv8h, "ld1r", ".8h", 0, false, 0 }, - { AArch64::LD1Rv4s, "ld1r", ".4s", 0, false, 0 }, - { AArch64::LD1Rv2d, "ld1r", ".2d", 0, false, 0 }, - { AArch64::LD1Rv8b, "ld1r", ".8b", 0, false, 0 }, - { AArch64::LD1Rv4h, "ld1r", ".4h", 0, false, 0 }, - { AArch64::LD1Rv2s, "ld1r", ".2s", 0, false, 0 }, - { AArch64::LD1Rv1d, "ld1r", ".1d", 0, false, 0 }, - { AArch64::LD1Rv16b_POST, "ld1r", ".16b", 1, false, 1 }, - { AArch64::LD1Rv8h_POST, "ld1r", ".8h", 1, false, 2 }, - { AArch64::LD1Rv4s_POST, "ld1r", ".4s", 1, false, 4 }, - { AArch64::LD1Rv2d_POST, "ld1r", ".2d", 1, false, 8 }, - { AArch64::LD1Rv8b_POST, "ld1r", ".8b", 1, false, 1 }, - { AArch64::LD1Rv4h_POST, "ld1r", ".4h", 1, false, 2 }, - { AArch64::LD1Rv2s_POST, "ld1r", ".2s", 1, false, 4 }, - { AArch64::LD1Rv1d_POST, "ld1r", ".1d", 1, false, 8 }, - { AArch64::LD1Onev16b, "ld1", ".16b", 0, false, 0 }, - { AArch64::LD1Onev8h, "ld1", ".8h", 0, false, 0 }, - { AArch64::LD1Onev4s, "ld1", ".4s", 0, false, 0 }, - { AArch64::LD1Onev2d, "ld1", ".2d", 0, false, 0 }, - { AArch64::LD1Onev8b, "ld1", ".8b", 0, false, 0 }, - { AArch64::LD1Onev4h, "ld1", ".4h", 0, false, 0 }, - { AArch64::LD1Onev2s, "ld1", ".2s", 0, false, 0 }, - { AArch64::LD1Onev1d, "ld1", ".1d", 0, false, 0 }, - { AArch64::LD1Onev16b_POST, "ld1", ".16b", 1, false, 16 }, - { AArch64::LD1Onev8h_POST, "ld1", ".8h", 1, false, 16 }, - { AArch64::LD1Onev4s_POST, "ld1", ".4s", 1, false, 16 }, - { AArch64::LD1Onev2d_POST, "ld1", ".2d", 1, false, 16 }, - { AArch64::LD1Onev8b_POST, "ld1", ".8b", 1, false, 8 }, - { AArch64::LD1Onev4h_POST, "ld1", ".4h", 1, false, 8 }, - { AArch64::LD1Onev2s_POST, "ld1", ".2s", 1, false, 8 }, - { AArch64::LD1Onev1d_POST, "ld1", ".1d", 1, false, 8 }, - { AArch64::LD1Twov16b, "ld1", ".16b", 0, false, 0 }, - { AArch64::LD1Twov8h, "ld1", ".8h", 0, false, 0 }, - { AArch64::LD1Twov4s, "ld1", ".4s", 0, false, 0 }, - { AArch64::LD1Twov2d, "ld1", ".2d", 0, false, 0 }, - { AArch64::LD1Twov8b, "ld1", ".8b", 0, false, 0 }, - { AArch64::LD1Twov4h, "ld1", ".4h", 0, false, 0 }, - { AArch64::LD1Twov2s, "ld1", ".2s", 0, false, 0 }, - { AArch64::LD1Twov1d, "ld1", ".1d", 0, false, 0 }, - { AArch64::LD1Twov16b_POST, "ld1", ".16b", 1, false, 32 }, - { AArch64::LD1Twov8h_POST, "ld1", ".8h", 1, false, 32 }, - { AArch64::LD1Twov4s_POST, "ld1", ".4s", 1, false, 32 }, - { AArch64::LD1Twov2d_POST, "ld1", ".2d", 1, false, 32 }, - { AArch64::LD1Twov8b_POST, "ld1", ".8b", 1, false, 16 }, - { AArch64::LD1Twov4h_POST, "ld1", ".4h", 1, false, 16 }, - { AArch64::LD1Twov2s_POST, "ld1", ".2s", 1, false, 16 }, - { AArch64::LD1Twov1d_POST, "ld1", ".1d", 1, false, 16 }, - { AArch64::LD1Threev16b, "ld1", ".16b", 0, false, 0 }, - { AArch64::LD1Threev8h, "ld1", ".8h", 0, false, 0 }, - { AArch64::LD1Threev4s, "ld1", ".4s", 0, false, 0 }, - { AArch64::LD1Threev2d, "ld1", ".2d", 0, false, 0 }, - { AArch64::LD1Threev8b, "ld1", ".8b", 0, false, 0 }, - { AArch64::LD1Threev4h, "ld1", ".4h", 0, false, 0 }, - { AArch64::LD1Threev2s, "ld1", ".2s", 0, false, 0 }, - { AArch64::LD1Threev1d, "ld1", ".1d", 0, false, 0 }, - { AArch64::LD1Threev16b_POST, "ld1", ".16b", 1, false, 48 }, - { AArch64::LD1Threev8h_POST, "ld1", ".8h", 1, false, 48 }, - { AArch64::LD1Threev4s_POST, "ld1", ".4s", 1, false, 48 }, - { AArch64::LD1Threev2d_POST, "ld1", ".2d", 1, false, 48 }, - { AArch64::LD1Threev8b_POST, "ld1", ".8b", 1, false, 24 }, - { AArch64::LD1Threev4h_POST, "ld1", ".4h", 1, false, 24 }, - { AArch64::LD1Threev2s_POST, "ld1", ".2s", 1, false, 24 }, - { AArch64::LD1Threev1d_POST, "ld1", ".1d", 1, false, 24 }, - { AArch64::LD1Fourv16b, "ld1", ".16b", 0, false, 0 }, - { AArch64::LD1Fourv8h, "ld1", ".8h", 0, false, 0 }, - { AArch64::LD1Fourv4s, "ld1", ".4s", 0, false, 0 }, - { AArch64::LD1Fourv2d, "ld1", ".2d", 0, false, 0 }, - { AArch64::LD1Fourv8b, "ld1", ".8b", 0, false, 0 }, - { AArch64::LD1Fourv4h, "ld1", ".4h", 0, false, 0 }, - { AArch64::LD1Fourv2s, "ld1", ".2s", 0, false, 0 }, - { AArch64::LD1Fourv1d, "ld1", ".1d", 0, false, 0 }, - { AArch64::LD1Fourv16b_POST, "ld1", ".16b", 1, false, 64 }, - { AArch64::LD1Fourv8h_POST, "ld1", ".8h", 1, false, 64 }, - { AArch64::LD1Fourv4s_POST, "ld1", ".4s", 1, false, 64 }, - { AArch64::LD1Fourv2d_POST, "ld1", ".2d", 1, false, 64 }, - { AArch64::LD1Fourv8b_POST, "ld1", ".8b", 1, false, 32 }, - { AArch64::LD1Fourv4h_POST, "ld1", ".4h", 1, false, 32 }, - { AArch64::LD1Fourv2s_POST, "ld1", ".2s", 1, false, 32 }, - { AArch64::LD1Fourv1d_POST, "ld1", ".1d", 1, false, 32 }, - { AArch64::LD2i8, "ld2", ".b", 1, true, 0 }, - { AArch64::LD2i16, "ld2", ".h", 1, true, 0 }, - { AArch64::LD2i32, "ld2", ".s", 1, true, 0 }, - { AArch64::LD2i64, "ld2", ".d", 1, true, 0 }, - { AArch64::LD2i8_POST, "ld2", ".b", 2, true, 2 }, - { AArch64::LD2i16_POST, "ld2", ".h", 2, true, 4 }, - { AArch64::LD2i32_POST, "ld2", ".s", 2, true, 8 }, - { AArch64::LD2i64_POST, "ld2", ".d", 2, true, 16 }, - { AArch64::LD2Rv16b, "ld2r", ".16b", 0, false, 0 }, - { AArch64::LD2Rv8h, "ld2r", ".8h", 0, false, 0 }, - { AArch64::LD2Rv4s, "ld2r", ".4s", 0, false, 0 }, - { AArch64::LD2Rv2d, "ld2r", ".2d", 0, false, 0 }, - { AArch64::LD2Rv8b, "ld2r", ".8b", 0, false, 0 }, - { AArch64::LD2Rv4h, "ld2r", ".4h", 0, false, 0 }, - { AArch64::LD2Rv2s, "ld2r", ".2s", 0, false, 0 }, - { AArch64::LD2Rv1d, "ld2r", ".1d", 0, false, 0 }, - { AArch64::LD2Rv16b_POST, "ld2r", ".16b", 1, false, 2 }, - { AArch64::LD2Rv8h_POST, "ld2r", ".8h", 1, false, 4 }, - { AArch64::LD2Rv4s_POST, "ld2r", ".4s", 1, false, 8 }, - { AArch64::LD2Rv2d_POST, "ld2r", ".2d", 1, false, 16 }, - { AArch64::LD2Rv8b_POST, "ld2r", ".8b", 1, false, 2 }, - { AArch64::LD2Rv4h_POST, "ld2r", ".4h", 1, false, 4 }, - { AArch64::LD2Rv2s_POST, "ld2r", ".2s", 1, false, 8 }, - { AArch64::LD2Rv1d_POST, "ld2r", ".1d", 1, false, 16 }, - { AArch64::LD2Twov16b, "ld2", ".16b", 0, false, 0 }, - { AArch64::LD2Twov8h, "ld2", ".8h", 0, false, 0 }, - { AArch64::LD2Twov4s, "ld2", ".4s", 0, false, 0 }, - { AArch64::LD2Twov2d, "ld2", ".2d", 0, false, 0 }, - { AArch64::LD2Twov8b, "ld2", ".8b", 0, false, 0 }, - { AArch64::LD2Twov4h, "ld2", ".4h", 0, false, 0 }, - { AArch64::LD2Twov2s, "ld2", ".2s", 0, false, 0 }, - { AArch64::LD2Twov16b_POST, "ld2", ".16b", 1, false, 32 }, - { AArch64::LD2Twov8h_POST, "ld2", ".8h", 1, false, 32 }, - { AArch64::LD2Twov4s_POST, "ld2", ".4s", 1, false, 32 }, - { AArch64::LD2Twov2d_POST, "ld2", ".2d", 1, false, 32 }, - { AArch64::LD2Twov8b_POST, "ld2", ".8b", 1, false, 16 }, - { AArch64::LD2Twov4h_POST, "ld2", ".4h", 1, false, 16 }, - { AArch64::LD2Twov2s_POST, "ld2", ".2s", 1, false, 16 }, - { AArch64::LD3i8, "ld3", ".b", 1, true, 0 }, - { AArch64::LD3i16, "ld3", ".h", 1, true, 0 }, - { AArch64::LD3i32, "ld3", ".s", 1, true, 0 }, - { AArch64::LD3i64, "ld3", ".d", 1, true, 0 }, - { AArch64::LD3i8_POST, "ld3", ".b", 2, true, 3 }, - { AArch64::LD3i16_POST, "ld3", ".h", 2, true, 6 }, - { AArch64::LD3i32_POST, "ld3", ".s", 2, true, 12 }, - { AArch64::LD3i64_POST, "ld3", ".d", 2, true, 24 }, - { AArch64::LD3Rv16b, "ld3r", ".16b", 0, false, 0 }, - { AArch64::LD3Rv8h, "ld3r", ".8h", 0, false, 0 }, - { AArch64::LD3Rv4s, "ld3r", ".4s", 0, false, 0 }, - { AArch64::LD3Rv2d, "ld3r", ".2d", 0, false, 0 }, - { AArch64::LD3Rv8b, "ld3r", ".8b", 0, false, 0 }, - { AArch64::LD3Rv4h, "ld3r", ".4h", 0, false, 0 }, - { AArch64::LD3Rv2s, "ld3r", ".2s", 0, false, 0 }, - { AArch64::LD3Rv1d, "ld3r", ".1d", 0, false, 0 }, - { AArch64::LD3Rv16b_POST, "ld3r", ".16b", 1, false, 3 }, - { AArch64::LD3Rv8h_POST, "ld3r", ".8h", 1, false, 6 }, - { AArch64::LD3Rv4s_POST, "ld3r", ".4s", 1, false, 12 }, - { AArch64::LD3Rv2d_POST, "ld3r", ".2d", 1, false, 24 }, - { AArch64::LD3Rv8b_POST, "ld3r", ".8b", 1, false, 3 }, - { AArch64::LD3Rv4h_POST, "ld3r", ".4h", 1, false, 6 }, - { AArch64::LD3Rv2s_POST, "ld3r", ".2s", 1, false, 12 }, - { AArch64::LD3Rv1d_POST, "ld3r", ".1d", 1, false, 24 }, - { AArch64::LD3Threev16b, "ld3", ".16b", 0, false, 0 }, - { AArch64::LD3Threev8h, "ld3", ".8h", 0, false, 0 }, - { AArch64::LD3Threev4s, "ld3", ".4s", 0, false, 0 }, - { AArch64::LD3Threev2d, "ld3", ".2d", 0, false, 0 }, - { AArch64::LD3Threev8b, "ld3", ".8b", 0, false, 0 }, - { AArch64::LD3Threev4h, "ld3", ".4h", 0, false, 0 }, - { AArch64::LD3Threev2s, "ld3", ".2s", 0, false, 0 }, - { AArch64::LD3Threev16b_POST, "ld3", ".16b", 1, false, 48 }, - { AArch64::LD3Threev8h_POST, "ld3", ".8h", 1, false, 48 }, - { AArch64::LD3Threev4s_POST, "ld3", ".4s", 1, false, 48 }, - { AArch64::LD3Threev2d_POST, "ld3", ".2d", 1, false, 48 }, - { AArch64::LD3Threev8b_POST, "ld3", ".8b", 1, false, 24 }, - { AArch64::LD3Threev4h_POST, "ld3", ".4h", 1, false, 24 }, - { AArch64::LD3Threev2s_POST, "ld3", ".2s", 1, false, 24 }, - { AArch64::LD4i8, "ld4", ".b", 1, true, 0 }, - { AArch64::LD4i16, "ld4", ".h", 1, true, 0 }, - { AArch64::LD4i32, "ld4", ".s", 1, true, 0 }, - { AArch64::LD4i64, "ld4", ".d", 1, true, 0 }, - { AArch64::LD4i8_POST, "ld4", ".b", 2, true, 4 }, - { AArch64::LD4i16_POST, "ld4", ".h", 2, true, 8 }, - { AArch64::LD4i32_POST, "ld4", ".s", 2, true, 16 }, - { AArch64::LD4i64_POST, "ld4", ".d", 2, true, 32 }, - { AArch64::LD4Rv16b, "ld4r", ".16b", 0, false, 0 }, - { AArch64::LD4Rv8h, "ld4r", ".8h", 0, false, 0 }, - { AArch64::LD4Rv4s, "ld4r", ".4s", 0, false, 0 }, - { AArch64::LD4Rv2d, "ld4r", ".2d", 0, false, 0 }, - { AArch64::LD4Rv8b, "ld4r", ".8b", 0, false, 0 }, - { AArch64::LD4Rv4h, "ld4r", ".4h", 0, false, 0 }, - { AArch64::LD4Rv2s, "ld4r", ".2s", 0, false, 0 }, - { AArch64::LD4Rv1d, "ld4r", ".1d", 0, false, 0 }, - { AArch64::LD4Rv16b_POST, "ld4r", ".16b", 1, false, 4 }, - { AArch64::LD4Rv8h_POST, "ld4r", ".8h", 1, false, 8 }, - { AArch64::LD4Rv4s_POST, "ld4r", ".4s", 1, false, 16 }, - { AArch64::LD4Rv2d_POST, "ld4r", ".2d", 1, false, 32 }, - { AArch64::LD4Rv8b_POST, "ld4r", ".8b", 1, false, 4 }, - { AArch64::LD4Rv4h_POST, "ld4r", ".4h", 1, false, 8 }, - { AArch64::LD4Rv2s_POST, "ld4r", ".2s", 1, false, 16 }, - { AArch64::LD4Rv1d_POST, "ld4r", ".1d", 1, false, 32 }, - { AArch64::LD4Fourv16b, "ld4", ".16b", 0, false, 0 }, - { AArch64::LD4Fourv8h, "ld4", ".8h", 0, false, 0 }, - { AArch64::LD4Fourv4s, "ld4", ".4s", 0, false, 0 }, - { AArch64::LD4Fourv2d, "ld4", ".2d", 0, false, 0 }, - { AArch64::LD4Fourv8b, "ld4", ".8b", 0, false, 0 }, - { AArch64::LD4Fourv4h, "ld4", ".4h", 0, false, 0 }, - { AArch64::LD4Fourv2s, "ld4", ".2s", 0, false, 0 }, - { AArch64::LD4Fourv16b_POST, "ld4", ".16b", 1, false, 64 }, - { AArch64::LD4Fourv8h_POST, "ld4", ".8h", 1, false, 64 }, - { AArch64::LD4Fourv4s_POST, "ld4", ".4s", 1, false, 64 }, - { AArch64::LD4Fourv2d_POST, "ld4", ".2d", 1, false, 64 }, - { AArch64::LD4Fourv8b_POST, "ld4", ".8b", 1, false, 32 }, - { AArch64::LD4Fourv4h_POST, "ld4", ".4h", 1, false, 32 }, - { AArch64::LD4Fourv2s_POST, "ld4", ".2s", 1, false, 32 }, - { AArch64::ST1i8, "st1", ".b", 0, true, 0 }, - { AArch64::ST1i16, "st1", ".h", 0, true, 0 }, - { AArch64::ST1i32, "st1", ".s", 0, true, 0 }, - { AArch64::ST1i64, "st1", ".d", 0, true, 0 }, - { AArch64::ST1i8_POST, "st1", ".b", 1, true, 1 }, - { AArch64::ST1i16_POST, "st1", ".h", 1, true, 2 }, - { AArch64::ST1i32_POST, "st1", ".s", 1, true, 4 }, - { AArch64::ST1i64_POST, "st1", ".d", 1, true, 8 }, - { AArch64::ST1Onev16b, "st1", ".16b", 0, false, 0 }, - { AArch64::ST1Onev8h, "st1", ".8h", 0, false, 0 }, - { AArch64::ST1Onev4s, "st1", ".4s", 0, false, 0 }, - { AArch64::ST1Onev2d, "st1", ".2d", 0, false, 0 }, - { AArch64::ST1Onev8b, "st1", ".8b", 0, false, 0 }, - { AArch64::ST1Onev4h, "st1", ".4h", 0, false, 0 }, - { AArch64::ST1Onev2s, "st1", ".2s", 0, false, 0 }, - { AArch64::ST1Onev1d, "st1", ".1d", 0, false, 0 }, - { AArch64::ST1Onev16b_POST, "st1", ".16b", 1, false, 16 }, - { AArch64::ST1Onev8h_POST, "st1", ".8h", 1, false, 16 }, - { AArch64::ST1Onev4s_POST, "st1", ".4s", 1, false, 16 }, - { AArch64::ST1Onev2d_POST, "st1", ".2d", 1, false, 16 }, - { AArch64::ST1Onev8b_POST, "st1", ".8b", 1, false, 8 }, - { AArch64::ST1Onev4h_POST, "st1", ".4h", 1, false, 8 }, - { AArch64::ST1Onev2s_POST, "st1", ".2s", 1, false, 8 }, - { AArch64::ST1Onev1d_POST, "st1", ".1d", 1, false, 8 }, - { AArch64::ST1Twov16b, "st1", ".16b", 0, false, 0 }, - { AArch64::ST1Twov8h, "st1", ".8h", 0, false, 0 }, - { AArch64::ST1Twov4s, "st1", ".4s", 0, false, 0 }, - { AArch64::ST1Twov2d, "st1", ".2d", 0, false, 0 }, - { AArch64::ST1Twov8b, "st1", ".8b", 0, false, 0 }, - { AArch64::ST1Twov4h, "st1", ".4h", 0, false, 0 }, - { AArch64::ST1Twov2s, "st1", ".2s", 0, false, 0 }, - { AArch64::ST1Twov1d, "st1", ".1d", 0, false, 0 }, - { AArch64::ST1Twov16b_POST, "st1", ".16b", 1, false, 32 }, - { AArch64::ST1Twov8h_POST, "st1", ".8h", 1, false, 32 }, - { AArch64::ST1Twov4s_POST, "st1", ".4s", 1, false, 32 }, - { AArch64::ST1Twov2d_POST, "st1", ".2d", 1, false, 32 }, - { AArch64::ST1Twov8b_POST, "st1", ".8b", 1, false, 16 }, - { AArch64::ST1Twov4h_POST, "st1", ".4h", 1, false, 16 }, - { AArch64::ST1Twov2s_POST, "st1", ".2s", 1, false, 16 }, - { AArch64::ST1Twov1d_POST, "st1", ".1d", 1, false, 16 }, - { AArch64::ST1Threev16b, "st1", ".16b", 0, false, 0 }, - { AArch64::ST1Threev8h, "st1", ".8h", 0, false, 0 }, - { AArch64::ST1Threev4s, "st1", ".4s", 0, false, 0 }, - { AArch64::ST1Threev2d, "st1", ".2d", 0, false, 0 }, - { AArch64::ST1Threev8b, "st1", ".8b", 0, false, 0 }, - { AArch64::ST1Threev4h, "st1", ".4h", 0, false, 0 }, - { AArch64::ST1Threev2s, "st1", ".2s", 0, false, 0 }, - { AArch64::ST1Threev1d, "st1", ".1d", 0, false, 0 }, - { AArch64::ST1Threev16b_POST, "st1", ".16b", 1, false, 48 }, - { AArch64::ST1Threev8h_POST, "st1", ".8h", 1, false, 48 }, - { AArch64::ST1Threev4s_POST, "st1", ".4s", 1, false, 48 }, - { AArch64::ST1Threev2d_POST, "st1", ".2d", 1, false, 48 }, - { AArch64::ST1Threev8b_POST, "st1", ".8b", 1, false, 24 }, - { AArch64::ST1Threev4h_POST, "st1", ".4h", 1, false, 24 }, - { AArch64::ST1Threev2s_POST, "st1", ".2s", 1, false, 24 }, - { AArch64::ST1Threev1d_POST, "st1", ".1d", 1, false, 24 }, - { AArch64::ST1Fourv16b, "st1", ".16b", 0, false, 0 }, - { AArch64::ST1Fourv8h, "st1", ".8h", 0, false, 0 }, - { AArch64::ST1Fourv4s, "st1", ".4s", 0, false, 0 }, - { AArch64::ST1Fourv2d, "st1", ".2d", 0, false, 0 }, - { AArch64::ST1Fourv8b, "st1", ".8b", 0, false, 0 }, - { AArch64::ST1Fourv4h, "st1", ".4h", 0, false, 0 }, - { AArch64::ST1Fourv2s, "st1", ".2s", 0, false, 0 }, - { AArch64::ST1Fourv1d, "st1", ".1d", 0, false, 0 }, - { AArch64::ST1Fourv16b_POST, "st1", ".16b", 1, false, 64 }, - { AArch64::ST1Fourv8h_POST, "st1", ".8h", 1, false, 64 }, - { AArch64::ST1Fourv4s_POST, "st1", ".4s", 1, false, 64 }, - { AArch64::ST1Fourv2d_POST, "st1", ".2d", 1, false, 64 }, - { AArch64::ST1Fourv8b_POST, "st1", ".8b", 1, false, 32 }, - { AArch64::ST1Fourv4h_POST, "st1", ".4h", 1, false, 32 }, - { AArch64::ST1Fourv2s_POST, "st1", ".2s", 1, false, 32 }, - { AArch64::ST1Fourv1d_POST, "st1", ".1d", 1, false, 32 }, - { AArch64::ST2i8, "st2", ".b", 0, true, 0 }, - { AArch64::ST2i16, "st2", ".h", 0, true, 0 }, - { AArch64::ST2i32, "st2", ".s", 0, true, 0 }, - { AArch64::ST2i64, "st2", ".d", 0, true, 0 }, - { AArch64::ST2i8_POST, "st2", ".b", 1, true, 2 }, - { AArch64::ST2i16_POST, "st2", ".h", 1, true, 4 }, - { AArch64::ST2i32_POST, "st2", ".s", 1, true, 8 }, - { AArch64::ST2i64_POST, "st2", ".d", 1, true, 16 }, - { AArch64::ST2Twov16b, "st2", ".16b", 0, false, 0 }, - { AArch64::ST2Twov8h, "st2", ".8h", 0, false, 0 }, - { AArch64::ST2Twov4s, "st2", ".4s", 0, false, 0 }, - { AArch64::ST2Twov2d, "st2", ".2d", 0, false, 0 }, - { AArch64::ST2Twov8b, "st2", ".8b", 0, false, 0 }, - { AArch64::ST2Twov4h, "st2", ".4h", 0, false, 0 }, - { AArch64::ST2Twov2s, "st2", ".2s", 0, false, 0 }, - { AArch64::ST2Twov16b_POST, "st2", ".16b", 1, false, 32 }, - { AArch64::ST2Twov8h_POST, "st2", ".8h", 1, false, 32 }, - { AArch64::ST2Twov4s_POST, "st2", ".4s", 1, false, 32 }, - { AArch64::ST2Twov2d_POST, "st2", ".2d", 1, false, 32 }, - { AArch64::ST2Twov8b_POST, "st2", ".8b", 1, false, 16 }, - { AArch64::ST2Twov4h_POST, "st2", ".4h", 1, false, 16 }, - { AArch64::ST2Twov2s_POST, "st2", ".2s", 1, false, 16 }, - { AArch64::ST3i8, "st3", ".b", 0, true, 0 }, - { AArch64::ST3i16, "st3", ".h", 0, true, 0 }, - { AArch64::ST3i32, "st3", ".s", 0, true, 0 }, - { AArch64::ST3i64, "st3", ".d", 0, true, 0 }, - { AArch64::ST3i8_POST, "st3", ".b", 1, true, 3 }, - { AArch64::ST3i16_POST, "st3", ".h", 1, true, 6 }, - { AArch64::ST3i32_POST, "st3", ".s", 1, true, 12 }, - { AArch64::ST3i64_POST, "st3", ".d", 1, true, 24 }, - { AArch64::ST3Threev16b, "st3", ".16b", 0, false, 0 }, - { AArch64::ST3Threev8h, "st3", ".8h", 0, false, 0 }, - { AArch64::ST3Threev4s, "st3", ".4s", 0, false, 0 }, - { AArch64::ST3Threev2d, "st3", ".2d", 0, false, 0 }, - { AArch64::ST3Threev8b, "st3", ".8b", 0, false, 0 }, - { AArch64::ST3Threev4h, "st3", ".4h", 0, false, 0 }, - { AArch64::ST3Threev2s, "st3", ".2s", 0, false, 0 }, - { AArch64::ST3Threev16b_POST, "st3", ".16b", 1, false, 48 }, - { AArch64::ST3Threev8h_POST, "st3", ".8h", 1, false, 48 }, - { AArch64::ST3Threev4s_POST, "st3", ".4s", 1, false, 48 }, - { AArch64::ST3Threev2d_POST, "st3", ".2d", 1, false, 48 }, - { AArch64::ST3Threev8b_POST, "st3", ".8b", 1, false, 24 }, - { AArch64::ST3Threev4h_POST, "st3", ".4h", 1, false, 24 }, - { AArch64::ST3Threev2s_POST, "st3", ".2s", 1, false, 24 }, - { AArch64::ST4i8, "st4", ".b", 0, true, 0 }, - { AArch64::ST4i16, "st4", ".h", 0, true, 0 }, - { AArch64::ST4i32, "st4", ".s", 0, true, 0 }, - { AArch64::ST4i64, "st4", ".d", 0, true, 0 }, - { AArch64::ST4i8_POST, "st4", ".b", 1, true, 4 }, - { AArch64::ST4i16_POST, "st4", ".h", 1, true, 8 }, - { AArch64::ST4i32_POST, "st4", ".s", 1, true, 16 }, - { AArch64::ST4i64_POST, "st4", ".d", 1, true, 32 }, - { AArch64::ST4Fourv16b, "st4", ".16b", 0, false, 0 }, - { AArch64::ST4Fourv8h, "st4", ".8h", 0, false, 0 }, - { AArch64::ST4Fourv4s, "st4", ".4s", 0, false, 0 }, - { AArch64::ST4Fourv2d, "st4", ".2d", 0, false, 0 }, - { AArch64::ST4Fourv8b, "st4", ".8b", 0, false, 0 }, - { AArch64::ST4Fourv4h, "st4", ".4h", 0, false, 0 }, - { AArch64::ST4Fourv2s, "st4", ".2s", 0, false, 0 }, - { AArch64::ST4Fourv16b_POST, "st4", ".16b", 1, false, 64 }, - { AArch64::ST4Fourv8h_POST, "st4", ".8h", 1, false, 64 }, - { AArch64::ST4Fourv4s_POST, "st4", ".4s", 1, false, 64 }, - { AArch64::ST4Fourv2d_POST, "st4", ".2d", 1, false, 64 }, - { AArch64::ST4Fourv8b_POST, "st4", ".8b", 1, false, 32 }, - { AArch64::ST4Fourv4h_POST, "st4", ".4h", 1, false, 32 }, - { AArch64::ST4Fourv2s_POST, "st4", ".2s", 1, false, 32 }, -}; - -static const LdStNInstrDesc *getLdStNInstrDesc(unsigned Opcode) { - for (const auto &Info : LdStNInstInfo) - if (Info.Opcode == Opcode) - return &Info; - - return nullptr; -} - void AArch64AppleInstPrinter::printInst(const MCInst *MI, uint64_t Address, StringRef Annot, const MCSubtargetInfo &STI, diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp new file mode 100644 index 0000000000000..1fa6590640c32 --- /dev/null +++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp @@ -0,0 +1,1648 @@ +//===- AArch64MCLFIExpander.cpp ---------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file implements the AArch64MCLFIExpander class, the AArch64 specific +// subclass of MCLFIExpander. +// +//===----------------------------------------------------------------------===// +#include "AArch64MCLFIExpander.h" +#include "AArch64AddressingModes.h" +#include "Utils/AArch64BaseInfo.h" + +#include "llvm/MC/MCInst.h" +#include "llvm/MC/MCInstrDesc.h" +#include "llvm/MC/MCInstrInfo.h" +#include "llvm/MC/MCLFIExpander.h" +#include "llvm/MC/MCObjectFileInfo.h" +#include "llvm/MC/MCRegisterInfo.h" +#include "llvm/MC/MCStreamer.h" +#include "llvm/MC/MCSubtargetInfo.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Debug.h" + +using namespace llvm; + +#define DEBUG_TYPE "lfi" + +static cl::opt AArch64LFIErrorReserved( + "aarch64-lfi-error-reserved", cl::Hidden, cl::init(false), + cl::desc("Produce errors for uses of LFI reserved registers")); + +static MCRegister LFIAddrReg = AArch64::X28; +static MCRegister LFIBaseReg = AArch64::X27; +static MCRegister LFIScratchReg = AArch64::X26; +static MCRegister LFITLSReg = AArch64::X25; + +static bool hasFeature(const FeatureBitset Feature, + const MCSubtargetInfo &STI) { + return (STI.getFeatureBits() & Feature) == Feature; +} + +bool AArch64::AArch64MCLFIExpander::isValidScratchRegister( + MCRegister Reg) const { + return Reg != AArch64::SP; +} + +MCRegister AArch64::AArch64MCLFIExpander::getScratch() { + if (numScratchRegs() == 0) { + return LFIScratchReg; + } + return getScratchReg(0); +} + +static void emit(unsigned int Op, MCRegister Rd, MCRegister Rs, int64_t Imm, + MCStreamer &Out, const MCSubtargetInfo &STI) { + MCInst Inst; + Inst.setOpcode(Op); + Inst.addOperand(MCOperand::createReg(Rd)); + Inst.addOperand(MCOperand::createReg(Rs)); + Inst.addOperand(MCOperand::createImm(Imm)); + Out.emitInstruction(Inst, STI); +} + +static void emit(unsigned int Op, MCRegister Rd, MCRegister Rt1, MCRegister Rt2, + int64_t Imm, MCStreamer &Out, const MCSubtargetInfo &STI) { + MCInst Inst; + Inst.setOpcode(Op); + Inst.addOperand(MCOperand::createReg(Rd)); + Inst.addOperand(MCOperand::createReg(Rt1)); + Inst.addOperand(MCOperand::createReg(Rt2)); + Inst.addOperand(MCOperand::createImm(Imm)); + Out.emitInstruction(Inst, STI); +} + +static void emit(unsigned int Op, MCRegister Rd, MCRegister Rt1, MCRegister Rt2, + int64_t Imm1, int64_t Imm2, MCStreamer &Out, + const MCSubtargetInfo &STI) { + MCInst Inst; + Inst.setOpcode(Op); + Inst.addOperand(MCOperand::createReg(Rd)); + Inst.addOperand(MCOperand::createReg(Rt1)); + Inst.addOperand(MCOperand::createReg(Rt2)); + Inst.addOperand(MCOperand::createImm(Imm1)); + Inst.addOperand(MCOperand::createImm(Imm2)); + Out.emitInstruction(Inst, STI); +} + +static void emit(unsigned int Op, MCRegister Rd, MCRegister Rt, int64_t Imm1, + int64_t Imm2, MCStreamer &Out, const MCSubtargetInfo &STI) { + MCInst Inst; + Inst.setOpcode(Op); + Inst.addOperand(MCOperand::createReg(Rd)); + Inst.addOperand(MCOperand::createReg(Rt)); + Inst.addOperand(MCOperand::createImm(Imm1)); + Inst.addOperand(MCOperand::createImm(Imm2)); + Out.emitInstruction(Inst, STI); +} + +static void emit(unsigned int Op, MCRegister Reg, MCStreamer &Out, + const MCSubtargetInfo &STI) { + MCInst Inst; + Inst.setOpcode(Op); + Inst.addOperand(MCOperand::createReg(Reg)); + Out.emitInstruction(Inst, STI); +} + +static void emitMov(MCRegister Dest, MCRegister Src, MCStreamer &Out, + const MCSubtargetInfo &STI) { + emit(AArch64::ORRXrs, Dest, AArch64::XZR, Src, 0, Out, STI); +} + +// Emit 'add Dest, LFIBaseReg, W(Src), uxtw' +static void emitAddMask(MCRegister Dest, MCRegister Src, MCStreamer &Out, + const MCSubtargetInfo &STI) { + emit(AArch64::ADDXrx, Dest, LFIBaseReg, getWRegFromXReg(Src), + AArch64_AM::getArithExtendImm(AArch64_AM::UXTW, 0), Out, STI); +} + +// Emit 'Op(ld/st) Dest, [LFIBaseReg, W(Target), uxtw]' +static void emitMemMask(unsigned Op, MCRegister Dest, MCRegister Target, + MCStreamer &Out, const MCSubtargetInfo &STI) { + emit(Op, Dest, LFIBaseReg, getWRegFromXReg(Target), 0, 0, Out, STI); +} + +static void emitBranch(unsigned int Opcode, MCRegister Target, MCStreamer &Out, + const MCSubtargetInfo &STI) { + MCInst Branch; + Branch.setOpcode(Opcode); + Branch.addOperand(MCOperand::createReg(Target)); + Out.emitInstruction(Branch, STI); +} + +void AArch64::AArch64MCLFIExpander::expandIndirectBranch( + const MCInst &Inst, MCStreamer &Out, const MCSubtargetInfo &STI, + bool IsCall) { + (void)IsCall; + + assert(Inst.getOperand(0).isReg()); + MCRegister BranchReg = Inst.getOperand(0).getReg(); + + emitAddMask(LFIAddrReg, BranchReg, Out, STI); + emitBranch(Inst.getOpcode(), LFIAddrReg, Out, STI); +} + +void AArch64::AArch64MCLFIExpander::expandCall(const MCInst &Inst, + MCStreamer &Out, + const MCSubtargetInfo &STI) { + if (Inst.getOperand(0).isReg()) + expandIndirectBranch(Inst, Out, STI, true); + else + Out.emitInstruction(Inst, STI); +} + +void AArch64::AArch64MCLFIExpander::expandReturn(const MCInst &Inst, + MCStreamer &Out, + const MCSubtargetInfo &STI) { + assert(Inst.getOperand(0).isReg()); + if (Inst.getOperand(0).getReg() != AArch64::LR) + expandIndirectBranch(Inst, Out, STI, false); + else + Out.emitInstruction(Inst, STI); +} + +bool AArch64::AArch64MCLFIExpander::mayModifyStack(const MCInst &Inst) { + return mayModifyRegister(Inst, AArch64::SP); +} + +bool AArch64::AArch64MCLFIExpander::mayModifyReserved(const MCInst &Inst) { + return mayModifyRegister(Inst, LFIAddrReg) || + mayModifyRegister(Inst, LFIBaseReg); +} + +bool AArch64::AArch64MCLFIExpander::mayModifyLR(const MCInst &Inst) { + return mayModifyRegister(Inst, AArch64::LR); +} + +static MCInst replaceReg(const MCInst &Inst, MCRegister Dest, MCRegister Src) { + MCInst New; + New.setOpcode(Inst.getOpcode()); + New.setLoc(Inst.getLoc()); + for (unsigned I = 0; I < Inst.getNumOperands(); ++I) { + const MCOperand &Op = Inst.getOperand(I); + if (Op.isReg() && Op.getReg() == Src) { + New.addOperand(MCOperand::createReg(Dest)); + } else { + New.addOperand(Op); + } + } + return New; +} + +void AArch64::AArch64MCLFIExpander::expandLRModification( + const MCInst &Inst, MCStreamer &Out, const MCSubtargetInfo &STI) { + MCRegister Scratch = getScratch(); + MCInst New = replaceReg(Inst, Scratch, AArch64::LR); + if (mayLoad(New) || mayStore(New)) + expandLoadStore(New, Out, STI); + else + Out.emitInstruction(New, STI); + emitAddMask(AArch64::LR, Scratch, Out, STI); +} + +void AArch64::AArch64MCLFIExpander::expandStackModification( + const MCInst &Inst, MCStreamer &Out, const MCSubtargetInfo &STI) { + if (mayLoad(Inst) || mayStore(Inst)) { + if (mayModifyLR(Inst)) + return expandLRModification(Inst, Out, STI); + return Out.emitInstruction(Inst, STI); + } + + if (hasFeature(FeatureBitset({AArch64::FeatureLFIJumps}), STI)) + return Out.emitInstruction(Inst, STI); + + MCInst ModInst; + MCRegister Scratch = getScratch(); + assert(Inst.getOperand(0).isReg() && + Inst.getOperand(0).getReg() == AArch64::SP); + ModInst.setOpcode(Inst.getOpcode()); + ModInst.setLoc(Inst.getLoc()); + ModInst.addOperand(MCOperand::createReg(Scratch)); + for (unsigned I = 1, E = Inst.getNumOperands(); I != E; ++I) { + ModInst.addOperand(Inst.getOperand(I)); + } + Out.emitInstruction(ModInst, STI); + emitAddMask(AArch64::SP, Scratch, Out, STI); +} + +static bool canConvertToRoW(unsigned Op); +static unsigned convertRoXToRoW(unsigned Op, unsigned &Shift); +static unsigned convertRoWToRoW(unsigned Op, unsigned &Shift); +static unsigned convertUiToRoW(unsigned Op); +static unsigned convertPreToRoW(unsigned Op); +static unsigned convertPostToRoW(unsigned Op); + +static unsigned convertPrePostToBase(unsigned Op, bool &IsPre, + bool &IsBaseNoOffset); +static unsigned getPrePostScale(unsigned Op); + +static void emitSafeLoadStoreDemoted(const MCInst &Inst, unsigned N, + MCStreamer &Out, + const MCSubtargetInfo &STI) { + MCInst LoadStore; + bool IsPre, IsBaseNoOffset; + auto NewOpCode = + convertPrePostToBase(Inst.getOpcode(), IsPre, IsBaseNoOffset); + LoadStore.setOpcode(NewOpCode); + for (unsigned I = 1; I < N; I++) + LoadStore.addOperand(Inst.getOperand(I)); + LoadStore.addOperand(MCOperand::createReg(LFIAddrReg)); + if (IsPre) + LoadStore.addOperand(Inst.getOperand(N + 1)); + else if (!IsBaseNoOffset) + LoadStore.addOperand(MCOperand::createImm(0)); + Out.emitInstruction(LoadStore, STI); +} + +static void emitSafeLoadStore(const MCInst &Inst, unsigned N, MCStreamer &Out, + const MCSubtargetInfo &STI) { + MCInst LoadStore; + LoadStore.setOpcode(Inst.getOpcode()); + for (unsigned I = 0; I < N; ++I) + LoadStore.addOperand(Inst.getOperand(I)); + LoadStore.addOperand(MCOperand::createReg(LFIAddrReg)); + for (unsigned I = N + 1; I < Inst.getNumOperands(); ++I) + LoadStore.addOperand(Inst.getOperand(I)); + Out.emitInstruction(LoadStore, STI); +} + +void AArch64::AArch64MCLFIExpander::expandLoadStoreBasic( + const MCInst &Inst, MemInstInfo &MII, MCStreamer &Out, + const MCSubtargetInfo &STI) { + emitAddMask(LFIAddrReg, Inst.getOperand(MII.BaseRegIdx).getReg(), Out, STI); + + if (MII.IsPrePost) { + assert(MII.OffsetIdx != -1 && "Pre/Post must have valid OffsetIdx"); + + emitSafeLoadStoreDemoted(Inst, MII.BaseRegIdx, Out, STI); + MCRegister Base = Inst.getOperand(MII.BaseRegIdx).getReg(); + MCOperand OffsetMO = Inst.getOperand(MII.OffsetIdx); + if (OffsetMO.isReg()) { + // The immediate offset of post-indexed addressing NEON Instrs has a fixed + // value, and it is encoded as a post-index addressing with XZR register + // operand. e.g., LD3Threev3d_POST can only have #48 as its operand and + // its offset MachineOperand holds XZR, which is a *Register* kind, not + // Imm. + MCRegister OffReg = OffsetMO.getReg(); + if (OffReg == AArch64::XZR) { + const LdStNInstrDesc *Info = getLdStNInstrDesc(Inst.getOpcode()); + assert(Info && Info->NaturalOffset >= 0); + return emit(AArch64::ADDXri, Base, Base, Info->NaturalOffset, 0, Out, + STI); + } + assert(OffReg != AArch64::WZR); + return emit(AArch64::ADDXrs, Base, Base, OffsetMO.getReg(), 0, Out, STI); + } else { + auto Offset = Inst.getOperand(MII.OffsetIdx).getImm() * + getPrePostScale(Inst.getOpcode()); + if (Offset >= 0) + return emit(AArch64::ADDXri, Base, Base, Offset, 0, Out, STI); + return emit(AArch64::SUBXri, Base, Base, -Offset, 0, Out, STI); + } + } + + return emitSafeLoadStore(Inst, MII.BaseRegIdx, Out, STI); +} + +void AArch64::AArch64MCLFIExpander::expandLoadStoreRoW( + const MCInst &Inst, MemInstInfo &MII, MCStreamer &Out, + const MCSubtargetInfo &STI) { + unsigned MemOp; + unsigned Op = Inst.getOpcode(); + if ((MemOp = convertUiToRoW(Op)) != AArch64::INSTRUCTION_LIST_END) { + auto OffsetMCO = Inst.getOperand(2); + if (OffsetMCO.isImm() && OffsetMCO.getImm() == 0) + return emitMemMask(MemOp, Inst.getOperand(0).getReg(), + Inst.getOperand(1).getReg(), Out, STI); + return expandLoadStoreBasic(Inst, MII, Out, STI); + } + + if ((MemOp = convertPreToRoW(Op)) != AArch64::INSTRUCTION_LIST_END) { + MCRegister Reg = Inst.getOperand(2).getReg(); + int64_t Imm = Inst.getOperand(3).getImm(); + if (Imm >= 0) + emit(AArch64::ADDXri, Reg, Reg, Imm, 0, Out, STI); + else + emit(AArch64::SUBXri, Reg, Reg, -Imm, 0, Out, STI); + return emitMemMask(MemOp, Inst.getOperand(1).getReg(), Reg, Out, STI); + } + + if ((MemOp = convertPostToRoW(Op)) != AArch64::INSTRUCTION_LIST_END) { + MCRegister Reg = Inst.getOperand(2).getReg(); + emitMemMask(MemOp, Inst.getOperand(1).getReg(), Reg, Out, STI); + int64_t Imm = Inst.getOperand(3).getImm(); + if (Imm >= 0) + emit(AArch64::ADDXri, Reg, Reg, Imm, 0, Out, STI); + else + emit(AArch64::SUBXri, Reg, Reg, -Imm, 0, Out, STI); + return; + } + + unsigned Shift; + if ((MemOp = convertRoXToRoW(Op, Shift)) != AArch64::INSTRUCTION_LIST_END) { + MCRegister Reg1 = Inst.getOperand(1).getReg(); + MCRegister Reg2 = Inst.getOperand(2).getReg(); + int64_t Extend = Inst.getOperand(3).getImm(); + int64_t IsShift = Inst.getOperand(4).getImm(); + MCRegister Scratch = getScratch(); + if (!IsShift) + Shift = 0; + if (Extend) + emit(AArch64::ADDXrx, Scratch, Reg1, Reg2, + AArch64_AM::getArithExtendImm(AArch64_AM::SXTX, Shift), Out, STI); + else + emit(AArch64::ADDXrs, Scratch, Reg1, Reg2, + AArch64_AM::getShifterImm(AArch64_AM::LSL, Shift), Out, STI); + return emitMemMask(MemOp, Inst.getOperand(0).getReg(), Scratch, Out, STI); + } + + if ((MemOp = convertRoWToRoW(Op, Shift)) != AArch64::INSTRUCTION_LIST_END) { + MCRegister Reg1 = Inst.getOperand(1).getReg(); + MCRegister Reg2 = Inst.getOperand(2).getReg(); + int64_t S = Inst.getOperand(3).getImm(); + int64_t IsShift = Inst.getOperand(4).getImm(); + MCRegister Scratch = getScratch(); + if (!IsShift) + Shift = 0; + if (S) + emit(AArch64::ADDXrx, Scratch, Reg1, Reg2, + AArch64_AM::getArithExtendImm(AArch64_AM::SXTW, Shift), Out, STI); + else + emit(AArch64::ADDXrx, Scratch, Reg1, Reg2, + AArch64_AM::getArithExtendImm(AArch64_AM::UXTW, Shift), Out, STI); + return emitMemMask(MemOp, Inst.getOperand(0).getReg(), Scratch, Out, STI); + } +} + +static std::optional getAtomicLoadStoreInfo(const MCInst &Inst); + +static std::optional getMemInstInfo(const MCInst &Inst) { + auto MII = getLoadInfo(Inst); + if (MII.has_value()) + return MII; + MII = getStoreInfo(Inst); + if (MII.has_value()) + return MII; + MII = getAtomicLoadStoreInfo(Inst); + if (MII.has_value()) + return MII; + return std::nullopt; +} + +void AArch64::AArch64MCLFIExpander::expandLoadStore( + const MCInst &Inst, MCStreamer &Out, const MCSubtargetInfo &STI) { + if (hasFeature(FeatureBitset({AArch64::FeatureLFIJumps}), STI)) + return Out.emitInstruction(Inst, STI); + if (hasFeature(FeatureBitset({AArch64::FeatureLFIStores}), STI) && + !mayStore(Inst)) + return Out.emitInstruction(Inst, STI); + + auto MII = getMemInstInfo(Inst); + if (!MII.has_value()) + return Out.emitInstruction(Inst, STI); + + // Stack accesses without a register offset don't need rewriting. + if (Inst.getOperand(MII->BaseRegIdx).getReg() == AArch64::SP) { + if (MII->BaseRegIdx == (int)Inst.getNumOperands() - 1 || + !Inst.getOperand(MII->BaseRegIdx + 1).isReg()) + return Out.emitInstruction(Inst, STI); + } + + // Try to convert to RoW if we can, otherwise use fallback. + if (canConvertToRoW(Inst.getOpcode())) + expandLoadStoreRoW(Inst, MII.value(), Out, STI); + else + expandLoadStoreBasic(Inst, MII.value(), Out, STI); +} + +void AArch64::AArch64MCLFIExpander::emitLFICall(LFICallType CallType, + MCStreamer &Out, + const MCSubtargetInfo &STI) { + MCRegister Scratch = getScratch(); + emitMov(Scratch, AArch64::LR, Out, STI); + unsigned Offset; + switch (CallType) { + case LFISyscall: + Offset = 0; + break; + case LFITLSRead: + Offset = 1; + break; + case LFITLSWrite: + Offset = 2; + break; + } + emit(AArch64::LDRXui, AArch64::LR, LFIBaseReg, Offset, Out, STI); + emit(AArch64::BLR, AArch64::LR, Out, STI); + emitAddMask(AArch64::LR, Scratch, Out, STI); +} + +void AArch64::AArch64MCLFIExpander::expandSyscall(const MCInst &Inst, + MCStreamer &Out, + const MCSubtargetInfo &STI) { + emitLFICall(LFISyscall, Out, STI); +} + +static void emitSwap(MCRegister Reg1, MCRegister Reg2, MCStreamer &Out, + const MCSubtargetInfo &STI) { + emit(AArch64::EORXrs, Reg1, Reg1, Reg2, 0, Out, STI); + emit(AArch64::EORXrs, Reg2, Reg1, Reg2, 0, Out, STI); + emit(AArch64::EORXrs, Reg1, Reg1, Reg2, 0, Out, STI); +} + +void AArch64::AArch64MCLFIExpander::expandTLSRead(const MCInst &Inst, + MCStreamer &Out, + const MCSubtargetInfo &STI) { + MCRegister Reg = Inst.getOperand(0).getReg(); + + if (hasFeature(FeatureBitset({AArch64::FeatureLFITLSReg}), STI)) + return emit(AArch64::LDRXui, Reg, LFITLSReg, 0, Out, STI); + + if (Reg == AArch64::X0) { + emitLFICall(LFITLSRead, Out, STI); + } else { + emitMov(Reg, AArch64::X0, Out, STI); + emitLFICall(LFITLSRead, Out, STI); + emitSwap(AArch64::X0, Reg, Out, STI); + } +} + +void AArch64::AArch64MCLFIExpander::expandTLSWrite(const MCInst &Inst, + MCStreamer &Out, + const MCSubtargetInfo &STI) { + MCRegister Reg = Inst.getOperand(1).getReg(); + if (Reg == AArch64::X0) { + emitLFICall(LFITLSWrite, Out, STI); + } else { + emitSwap(Reg, AArch64::X0, Out, STI); + emitLFICall(LFITLSWrite, Out, STI); + emitSwap(AArch64::X0, Reg, Out, STI); + } +} + +static bool isSyscall(const MCInst &Inst) { + return Inst.getOpcode() == AArch64::SVC; +} + +static bool isTLSRead(const MCInst &Inst) { + return Inst.getOpcode() == AArch64::MRS && + Inst.getOperand(1).getImm() == AArch64SysReg::TPIDR_EL0; +} + +static bool isTLSWrite(const MCInst &Inst) { + return Inst.getOpcode() == AArch64::MSR && + Inst.getOperand(0).getImm() == AArch64SysReg::TPIDR_EL0; +} + +void AArch64::AArch64MCLFIExpander::doExpandInst(const MCInst &Inst, + MCStreamer &Out, + const MCSubtargetInfo &STI) { + if (isSyscall(Inst)) + return expandSyscall(Inst, Out, STI); + + if (isTLSRead(Inst)) + return expandTLSRead(Inst, Out, STI); + + if (isTLSWrite(Inst)) + return expandTLSWrite(Inst, Out, STI); + + if (isReturn(Inst)) + return expandReturn(Inst, Out, STI); + + if (isIndirectBranch(Inst)) + return expandIndirectBranch(Inst, Out, STI, false); + + if (isCall(Inst)) + return expandCall(Inst, Out, STI); + + if (isBranch(Inst)) + return Out.emitInstruction(Inst, STI); + + // Bail out with an error. In the future, we could consider automatically + // rewriting uses of reserved LFI registers. + if (mayModifyReserved(Inst)) { + if (AArch64LFIErrorReserved) + return Out.getContext().reportError( + Inst.getLoc(), "illegal modification of reserved LFI register"); + Out.getContext().reportWarning( + Inst.getLoc(), "deleting modification of reserved LFI register"); + MCInst New = replaceReg(Inst, AArch64::XZR, LFIBaseReg); + if (mayModifyReserved(New)) { + MCRegister Scratch = getScratch(); + New = replaceReg(New, Scratch, LFIAddrReg); + } + assert(!mayModifyReserved(New)); + return doExpandInst(New, Out, STI); + } + + if (mayModifyStack(Inst)) + return expandStackModification(Inst, Out, STI); + + if (mayModifyLR(Inst)) + return expandLRModification(Inst, Out, STI); + + if (mayLoad(Inst) || mayStore(Inst)) + return expandLoadStore(Inst, Out, STI); + + return Out.emitInstruction(Inst, STI); +} + +bool AArch64::AArch64MCLFIExpander::expandInst(const MCInst &Inst, + MCStreamer &Out, + const MCSubtargetInfo &STI) { + if (Guard) + return false; + Guard = true; + + doExpandInst(Inst, Out, STI); + + Guard = false; + return true; +} + +static unsigned convertRoXToRoW(unsigned Op, unsigned &Shift) { + Shift = 0; + switch (Op) { + case AArch64::LDRBBroX: + return AArch64::LDRBBroW; + case AArch64::LDRBroX: + return AArch64::LDRBroW; + case AArch64::LDRDroX: + Shift = 3; + return AArch64::LDRDroW; + case AArch64::LDRHHroX: + Shift = 1; + return AArch64::LDRHHroW; + case AArch64::LDRHroX: + Shift = 1; + return AArch64::LDRHroW; + case AArch64::LDRQroX: + Shift = 4; + return AArch64::LDRQroW; + case AArch64::LDRSBWroX: + Shift = 1; + return AArch64::LDRSBWroW; + case AArch64::LDRSBXroX: + Shift = 1; + return AArch64::LDRSBXroW; + case AArch64::LDRSHWroX: + Shift = 1; + return AArch64::LDRSHWroW; + case AArch64::LDRSHXroX: + Shift = 1; + return AArch64::LDRSHXroW; + case AArch64::LDRSWroX: + Shift = 2; + return AArch64::LDRSWroW; + case AArch64::LDRSroX: + Shift = 2; + return AArch64::LDRSroW; + case AArch64::LDRWroX: + Shift = 2; + return AArch64::LDRWroW; + case AArch64::LDRXroX: + Shift = 3; + return AArch64::LDRXroW; + case AArch64::STRBBroX: + return AArch64::STRBBroW; + case AArch64::STRBroX: + return AArch64::STRBroW; + case AArch64::STRDroX: + Shift = 3; + return AArch64::STRDroW; + case AArch64::STRHHroX: + Shift = 1; + return AArch64::STRHHroW; + case AArch64::STRHroX: + Shift = 1; + return AArch64::STRHroW; + case AArch64::STRQroX: + Shift = 4; + return AArch64::STRQroW; + case AArch64::STRSroX: + Shift = 2; + return AArch64::STRSroW; + case AArch64::STRWroX: + Shift = 2; + return AArch64::STRWroW; + case AArch64::STRXroX: + Shift = 3; + return AArch64::STRXroW; + } + return AArch64::INSTRUCTION_LIST_END; +} + +static unsigned convertRoWToRoW(unsigned Op, unsigned &Shift) { + Shift = 0; + switch (Op) { + case AArch64::LDRBBroW: + return AArch64::LDRBBroW; + case AArch64::LDRBroW: + return AArch64::LDRBroW; + case AArch64::LDRDroW: + Shift = 3; + return AArch64::LDRDroW; + case AArch64::LDRHHroW: + Shift = 1; + return AArch64::LDRHHroW; + case AArch64::LDRHroW: + Shift = 1; + return AArch64::LDRHroW; + case AArch64::LDRQroW: + Shift = 4; + return AArch64::LDRQroW; + case AArch64::LDRSBWroW: + Shift = 1; + return AArch64::LDRSBWroW; + case AArch64::LDRSBXroW: + Shift = 1; + return AArch64::LDRSBXroW; + case AArch64::LDRSHWroW: + Shift = 1; + return AArch64::LDRSHWroW; + case AArch64::LDRSHXroW: + Shift = 1; + return AArch64::LDRSHXroW; + case AArch64::LDRSWroW: + Shift = 2; + return AArch64::LDRSWroW; + case AArch64::LDRSroW: + Shift = 2; + return AArch64::LDRSroW; + case AArch64::LDRWroW: + Shift = 2; + return AArch64::LDRWroW; + case AArch64::LDRXroW: + Shift = 3; + return AArch64::LDRXroW; + case AArch64::STRBBroW: + return AArch64::STRBBroW; + case AArch64::STRBroW: + return AArch64::STRBroW; + case AArch64::STRDroW: + Shift = 3; + return AArch64::STRDroW; + case AArch64::STRHHroW: + Shift = 1; + return AArch64::STRHHroW; + case AArch64::STRHroW: + Shift = 1; + return AArch64::STRHroW; + case AArch64::STRQroW: + Shift = 4; + return AArch64::STRQroW; + case AArch64::STRSroW: + Shift = 2; + return AArch64::STRSroW; + case AArch64::STRWroW: + Shift = 2; + return AArch64::STRWroW; + case AArch64::STRXroW: + Shift = 3; + return AArch64::STRXroW; + } + return AArch64::INSTRUCTION_LIST_END; +} + +static unsigned convertUiToRoW(unsigned Op) { + switch (Op) { + case AArch64::LDRBBui: + return AArch64::LDRBBroW; + case AArch64::LDRBui: + return AArch64::LDRBroW; + case AArch64::LDRDui: + return AArch64::LDRDroW; + case AArch64::LDRHHui: + return AArch64::LDRHHroW; + case AArch64::LDRHui: + return AArch64::LDRHroW; + case AArch64::LDRQui: + return AArch64::LDRQroW; + case AArch64::LDRSBWui: + return AArch64::LDRSBWroW; + case AArch64::LDRSBXui: + return AArch64::LDRSBXroW; + case AArch64::LDRSHWui: + return AArch64::LDRSHWroW; + case AArch64::LDRSHXui: + return AArch64::LDRSHXroW; + case AArch64::LDRSWui: + return AArch64::LDRSWroW; + case AArch64::LDRSui: + return AArch64::LDRSroW; + case AArch64::LDRWui: + return AArch64::LDRWroW; + case AArch64::LDRXui: + return AArch64::LDRXroW; + case AArch64::STRBBui: + return AArch64::STRBBroW; + case AArch64::STRBui: + return AArch64::STRBroW; + case AArch64::STRDui: + return AArch64::STRDroW; + case AArch64::STRHHui: + return AArch64::STRHHroW; + case AArch64::STRHui: + return AArch64::STRHroW; + case AArch64::STRQui: + return AArch64::STRQroW; + case AArch64::STRSui: + return AArch64::STRSroW; + case AArch64::STRWui: + return AArch64::STRWroW; + case AArch64::STRXui: + return AArch64::STRXroW; + } + return AArch64::INSTRUCTION_LIST_END; +} + +static unsigned convertPreToRoW(unsigned Op) { + switch (Op) { + case AArch64::LDRBBpre: + return AArch64::LDRBBroW; + case AArch64::LDRBpre: + return AArch64::LDRBroW; + case AArch64::LDRDpre: + return AArch64::LDRDroW; + case AArch64::LDRHHpre: + return AArch64::LDRHHroW; + case AArch64::LDRHpre: + return AArch64::LDRHroW; + case AArch64::LDRQpre: + return AArch64::LDRQroW; + case AArch64::LDRSBWpre: + return AArch64::LDRSBWroW; + case AArch64::LDRSBXpre: + return AArch64::LDRSBXroW; + case AArch64::LDRSHWpre: + return AArch64::LDRSHWroW; + case AArch64::LDRSHXpre: + return AArch64::LDRSHXroW; + case AArch64::LDRSWpre: + return AArch64::LDRSWroW; + case AArch64::LDRSpre: + return AArch64::LDRSroW; + case AArch64::LDRWpre: + return AArch64::LDRWroW; + case AArch64::LDRXpre: + return AArch64::LDRXroW; + case AArch64::STRBBpre: + return AArch64::STRBBroW; + case AArch64::STRBpre: + return AArch64::STRBroW; + case AArch64::STRDpre: + return AArch64::STRDroW; + case AArch64::STRHHpre: + return AArch64::STRHHroW; + case AArch64::STRHpre: + return AArch64::STRHroW; + case AArch64::STRQpre: + return AArch64::STRQroW; + case AArch64::STRSpre: + return AArch64::STRSroW; + case AArch64::STRWpre: + return AArch64::STRWroW; + case AArch64::STRXpre: + return AArch64::STRXroW; + } + return AArch64::INSTRUCTION_LIST_END; +} + +static unsigned convertPostToRoW(unsigned Op) { + switch (Op) { + case AArch64::LDRBBpost: + return AArch64::LDRBBroW; + case AArch64::LDRBpost: + return AArch64::LDRBroW; + case AArch64::LDRDpost: + return AArch64::LDRDroW; + case AArch64::LDRHHpost: + return AArch64::LDRHHroW; + case AArch64::LDRHpost: + return AArch64::LDRHroW; + case AArch64::LDRQpost: + return AArch64::LDRQroW; + case AArch64::LDRSBWpost: + return AArch64::LDRSBWroW; + case AArch64::LDRSBXpost: + return AArch64::LDRSBXroW; + case AArch64::LDRSHWpost: + return AArch64::LDRSHWroW; + case AArch64::LDRSHXpost: + return AArch64::LDRSHXroW; + case AArch64::LDRSWpost: + return AArch64::LDRSWroW; + case AArch64::LDRSpost: + return AArch64::LDRSroW; + case AArch64::LDRWpost: + return AArch64::LDRWroW; + case AArch64::LDRXpost: + return AArch64::LDRXroW; + case AArch64::STRBBpost: + return AArch64::STRBBroW; + case AArch64::STRBpost: + return AArch64::STRBroW; + case AArch64::STRDpost: + return AArch64::STRDroW; + case AArch64::STRHHpost: + return AArch64::STRHHroW; + case AArch64::STRHpost: + return AArch64::STRHroW; + case AArch64::STRQpost: + return AArch64::STRQroW; + case AArch64::STRSpost: + return AArch64::STRSroW; + case AArch64::STRWpost: + return AArch64::STRWroW; + case AArch64::STRXpost: + return AArch64::STRXroW; + } + return AArch64::INSTRUCTION_LIST_END; +} + +static bool canConvertToRoW(unsigned Op) { + unsigned Shift; + return convertUiToRoW(Op) != AArch64::INSTRUCTION_LIST_END || + convertPreToRoW(Op) != AArch64::INSTRUCTION_LIST_END || + convertPostToRoW(Op) != AArch64::INSTRUCTION_LIST_END || + convertRoXToRoW(Op, Shift) != AArch64::INSTRUCTION_LIST_END || + convertRoWToRoW(Op, Shift) != AArch64::INSTRUCTION_LIST_END; +} + +static unsigned convertPrePostToBase(unsigned Op, bool &IsPre, + bool &IsBaseNoOffset) { + IsPre = false; + IsBaseNoOffset = false; + switch (Op) { + case AArch64::LDPDpost: + return AArch64::LDPDi; + case AArch64::LDPDpre: + IsPre = true; + return AArch64::LDPDi; + case AArch64::LDPQpost: + return AArch64::LDPQi; + case AArch64::LDPQpre: + IsPre = true; + return AArch64::LDPQi; + case AArch64::LDPSWpost: + return AArch64::LDPSWi; + case AArch64::LDPSWpre: + IsPre = true; + return AArch64::LDPSWi; + case AArch64::LDPSpost: + return AArch64::LDPSi; + case AArch64::LDPSpre: + IsPre = true; + return AArch64::LDPSi; + case AArch64::LDPWpost: + return AArch64::LDPWi; + case AArch64::LDPWpre: + IsPre = true; + return AArch64::LDPWi; + case AArch64::LDPXpost: + return AArch64::LDPXi; + case AArch64::LDPXpre: + IsPre = true; + return AArch64::LDPXi; + case AArch64::STPDpost: + return AArch64::STPDi; + case AArch64::STPDpre: + IsPre = true; + return AArch64::STPDi; + case AArch64::STPQpost: + return AArch64::STPQi; + case AArch64::STPQpre: + IsPre = true; + return AArch64::STPQi; + case AArch64::STPSpost: + return AArch64::STPSi; + case AArch64::STPSpre: + IsPre = true; + return AArch64::STPSi; + case AArch64::STPWpost: + return AArch64::STPWi; + case AArch64::STPWpre: + IsPre = true; + return AArch64::STPWi; + case AArch64::STPXpost: + return AArch64::STPXi; + case AArch64::STPXpre: + IsPre = true; + return AArch64::STPXi; + // case AArch64::STLRWpre: + // IsPre = true; + // IsBaseNoOffset = true; + // return AArch64::STPWi; + // case AArch64::STLRXpre: + // IsPre = true; + // IsBaseNoOffset = true; + // return AArch64::STLRX; + case AArch64::LD1i64_POST: + IsBaseNoOffset = true; + return AArch64::LD1i64; + case AArch64::LD2i64_POST: + IsBaseNoOffset = true; + return AArch64::LD2i64; + case AArch64::LD1i8_POST: + IsBaseNoOffset = true; + return AArch64::LD1i8; + case AArch64::LD1i16_POST: + IsBaseNoOffset = true; + return AArch64::LD1i16; + case AArch64::LD1i32_POST: + IsBaseNoOffset = true; + return AArch64::LD1i32; + case AArch64::LD2i8_POST: + IsBaseNoOffset = true; + return AArch64::LD2i8; + case AArch64::LD2i16_POST: + IsBaseNoOffset = true; + return AArch64::LD2i16; + case AArch64::LD2i32_POST: + IsBaseNoOffset = true; + return AArch64::LD2i32; + case AArch64::LD3i8_POST: + IsBaseNoOffset = true; + return AArch64::LD3i8; + case AArch64::LD3i16_POST: + IsBaseNoOffset = true; + return AArch64::LD3i16; + case AArch64::LD3i32_POST: + IsBaseNoOffset = true; + return AArch64::LD3i32; + case AArch64::LD3i64_POST: + IsBaseNoOffset = true; + return AArch64::LD3i64; + case AArch64::LD4i8_POST: + IsBaseNoOffset = true; + return AArch64::LD4i8; + case AArch64::LD4i16_POST: + IsBaseNoOffset = true; + return AArch64::LD4i16; + case AArch64::LD4i32_POST: + IsBaseNoOffset = true; + return AArch64::LD4i32; + case AArch64::LD4i64_POST: + IsBaseNoOffset = true; + return AArch64::LD4i64; + case AArch64::LD1Onev1d_POST: + IsBaseNoOffset = true; + return AArch64::LD1Onev1d; + case AArch64::LD1Onev2s_POST: + IsBaseNoOffset = true; + return AArch64::LD1Onev2s; + case AArch64::LD1Onev4h_POST: + IsBaseNoOffset = true; + return AArch64::LD1Onev4h; + case AArch64::LD1Onev8b_POST: + IsBaseNoOffset = true; + return AArch64::LD1Onev8b; + case AArch64::LD1Onev2d_POST: + IsBaseNoOffset = true; + return AArch64::LD1Onev2d; + case AArch64::LD1Onev4s_POST: + IsBaseNoOffset = true; + return AArch64::LD1Onev4s; + case AArch64::LD1Onev8h_POST: + IsBaseNoOffset = true; + return AArch64::LD1Onev8h; + case AArch64::LD1Onev16b_POST: + IsBaseNoOffset = true; + return AArch64::LD1Onev16b; + case AArch64::LD1Rv1d_POST: + IsBaseNoOffset = true; + return AArch64::LD1Rv1d; + case AArch64::LD1Rv2s_POST: + IsBaseNoOffset = true; + return AArch64::LD1Rv2s; + case AArch64::LD1Rv4h_POST: + IsBaseNoOffset = true; + return AArch64::LD1Rv4h; + case AArch64::LD1Rv8b_POST: + IsBaseNoOffset = true; + return AArch64::LD1Rv8b; + case AArch64::LD1Rv2d_POST: + IsBaseNoOffset = true; + return AArch64::LD1Rv2d; + case AArch64::LD1Rv4s_POST: + IsBaseNoOffset = true; + return AArch64::LD1Rv4s; + case AArch64::LD1Rv8h_POST: + IsBaseNoOffset = true; + return AArch64::LD1Rv8h; + case AArch64::LD1Rv16b_POST: + IsBaseNoOffset = true; + return AArch64::LD1Rv16b; + case AArch64::LD1Twov1d_POST: + IsBaseNoOffset = true; + return AArch64::LD1Twov1d; + case AArch64::LD1Twov2s_POST: + IsBaseNoOffset = true; + return AArch64::LD1Twov2s; + case AArch64::LD1Twov4h_POST: + IsBaseNoOffset = true; + return AArch64::LD1Twov4h; + case AArch64::LD1Twov8b_POST: + IsBaseNoOffset = true; + return AArch64::LD1Twov8b; + case AArch64::LD1Twov2d_POST: + IsBaseNoOffset = true; + return AArch64::LD1Twov2d; + case AArch64::LD1Twov4s_POST: + IsBaseNoOffset = true; + return AArch64::LD1Twov4s; + case AArch64::LD1Twov8h_POST: + IsBaseNoOffset = true; + return AArch64::LD1Twov8h; + case AArch64::LD1Twov16b_POST: + IsBaseNoOffset = true; + return AArch64::LD1Twov16b; + case AArch64::LD1Threev1d_POST: + IsBaseNoOffset = true; + return AArch64::LD1Threev1d; + case AArch64::LD1Threev2s_POST: + IsBaseNoOffset = true; + return AArch64::LD1Threev2s; + case AArch64::LD1Threev4h_POST: + IsBaseNoOffset = true; + return AArch64::LD1Threev4h; + case AArch64::LD1Threev8b_POST: + IsBaseNoOffset = true; + return AArch64::LD1Threev8b; + case AArch64::LD1Threev2d_POST: + IsBaseNoOffset = true; + return AArch64::LD1Threev2d; + case AArch64::LD1Threev4s_POST: + IsBaseNoOffset = true; + return AArch64::LD1Threev4s; + case AArch64::LD1Threev8h_POST: + IsBaseNoOffset = true; + return AArch64::LD1Threev8h; + case AArch64::LD1Threev16b_POST: + IsBaseNoOffset = true; + return AArch64::LD1Threev16b; + case AArch64::LD1Fourv1d_POST: + IsBaseNoOffset = true; + return AArch64::LD1Fourv1d; + case AArch64::LD1Fourv2s_POST: + IsBaseNoOffset = true; + return AArch64::LD1Fourv2s; + case AArch64::LD1Fourv4h_POST: + IsBaseNoOffset = true; + return AArch64::LD1Fourv4h; + case AArch64::LD1Fourv8b_POST: + IsBaseNoOffset = true; + return AArch64::LD1Fourv8b; + case AArch64::LD1Fourv2d_POST: + IsBaseNoOffset = true; + return AArch64::LD1Fourv2d; + case AArch64::LD1Fourv4s_POST: + IsBaseNoOffset = true; + return AArch64::LD1Fourv4s; + case AArch64::LD1Fourv8h_POST: + IsBaseNoOffset = true; + return AArch64::LD1Fourv8h; + case AArch64::LD1Fourv16b_POST: + IsBaseNoOffset = true; + return AArch64::LD1Fourv16b; + case AArch64::LD2Twov2s_POST: + IsBaseNoOffset = true; + return AArch64::LD2Twov2s; + case AArch64::LD2Twov4s_POST: + IsBaseNoOffset = true; + return AArch64::LD2Twov4s; + case AArch64::LD2Twov8b_POST: + IsBaseNoOffset = true; + return AArch64::LD2Twov8b; + case AArch64::LD2Twov2d_POST: + IsBaseNoOffset = true; + return AArch64::LD2Twov2d; + case AArch64::LD2Twov4h_POST: + IsBaseNoOffset = true; + return AArch64::LD2Twov4h; + case AArch64::LD2Twov8h_POST: + IsBaseNoOffset = true; + return AArch64::LD2Twov8h; + case AArch64::LD2Twov16b_POST: + IsBaseNoOffset = true; + return AArch64::LD2Twov16b; + case AArch64::LD2Rv1d_POST: + IsBaseNoOffset = true; + return AArch64::LD2Rv1d; + case AArch64::LD2Rv2s_POST: + IsBaseNoOffset = true; + return AArch64::LD2Rv2s; + case AArch64::LD2Rv4s_POST: + IsBaseNoOffset = true; + return AArch64::LD2Rv4s; + case AArch64::LD2Rv8b_POST: + IsBaseNoOffset = true; + return AArch64::LD2Rv8b; + case AArch64::LD2Rv2d_POST: + IsBaseNoOffset = true; + return AArch64::LD2Rv2d; + case AArch64::LD2Rv4h_POST: + IsBaseNoOffset = true; + return AArch64::LD2Rv4h; + case AArch64::LD2Rv8h_POST: + IsBaseNoOffset = true; + return AArch64::LD2Rv8h; + case AArch64::LD2Rv16b_POST: + IsBaseNoOffset = true; + return AArch64::LD2Rv16b; + case AArch64::LD3Threev2s_POST: + IsBaseNoOffset = true; + return AArch64::LD3Threev2s; + case AArch64::LD3Threev4h_POST: + IsBaseNoOffset = true; + return AArch64::LD3Threev4h; + case AArch64::LD3Threev8b_POST: + IsBaseNoOffset = true; + return AArch64::LD3Threev8b; + case AArch64::LD3Threev2d_POST: + IsBaseNoOffset = true; + return AArch64::LD3Threev2d; + case AArch64::LD3Threev4s_POST: + IsBaseNoOffset = true; + return AArch64::LD3Threev4s; + case AArch64::LD3Threev8h_POST: + IsBaseNoOffset = true; + return AArch64::LD3Threev8h; + case AArch64::LD3Threev16b_POST: + IsBaseNoOffset = true; + return AArch64::LD3Threev16b; + case AArch64::LD3Rv1d_POST: + IsBaseNoOffset = true; + return AArch64::LD3Rv1d; + case AArch64::LD3Rv2s_POST: + IsBaseNoOffset = true; + return AArch64::LD3Rv2s; + case AArch64::LD3Rv4h_POST: + IsBaseNoOffset = true; + return AArch64::LD3Rv4h; + case AArch64::LD3Rv8b_POST: + IsBaseNoOffset = true; + return AArch64::LD3Rv8b; + case AArch64::LD3Rv2d_POST: + IsBaseNoOffset = true; + return AArch64::LD3Rv2d; + case AArch64::LD3Rv4s_POST: + IsBaseNoOffset = true; + return AArch64::LD3Rv4s; + case AArch64::LD3Rv8h_POST: + IsBaseNoOffset = true; + return AArch64::LD3Rv8h; + case AArch64::LD3Rv16b_POST: + IsBaseNoOffset = true; + return AArch64::LD3Rv16b; + case AArch64::LD4Fourv2s_POST: + IsBaseNoOffset = true; + return AArch64::LD4Fourv2s; + case AArch64::LD4Fourv4h_POST: + IsBaseNoOffset = true; + return AArch64::LD4Fourv4h; + case AArch64::LD4Fourv8b_POST: + IsBaseNoOffset = true; + return AArch64::LD4Fourv8b; + case AArch64::LD4Fourv2d_POST: + IsBaseNoOffset = true; + return AArch64::LD4Fourv2d; + case AArch64::LD4Fourv4s_POST: + IsBaseNoOffset = true; + return AArch64::LD4Fourv4s; + case AArch64::LD4Fourv8h_POST: + IsBaseNoOffset = true; + return AArch64::LD4Fourv8h; + case AArch64::LD4Fourv16b_POST: + IsBaseNoOffset = true; + return AArch64::LD4Fourv16b; + case AArch64::LD4Rv1d_POST: + IsBaseNoOffset = true; + return AArch64::LD4Rv1d; + case AArch64::LD4Rv2s_POST: + IsBaseNoOffset = true; + return AArch64::LD4Rv2s; + case AArch64::LD4Rv4h_POST: + IsBaseNoOffset = true; + return AArch64::LD4Rv4h; + case AArch64::LD4Rv8b_POST: + IsBaseNoOffset = true; + return AArch64::LD4Rv8b; + case AArch64::LD4Rv2d_POST: + IsBaseNoOffset = true; + return AArch64::LD4Rv2d; + case AArch64::LD4Rv4s_POST: + IsBaseNoOffset = true; + return AArch64::LD4Rv4s; + case AArch64::LD4Rv8h_POST: + IsBaseNoOffset = true; + return AArch64::LD4Rv8h; + case AArch64::LD4Rv16b_POST: + IsBaseNoOffset = true; + return AArch64::LD4Rv16b; + case AArch64::ST1i64_POST: + IsBaseNoOffset = true; + return AArch64::ST1i64; + case AArch64::ST2i64_POST: + IsBaseNoOffset = true; + return AArch64::ST2i64; + case AArch64::ST1i8_POST: + IsBaseNoOffset = true; + return AArch64::ST1i8; + case AArch64::ST1i16_POST: + IsBaseNoOffset = true; + return AArch64::ST1i16; + case AArch64::ST1i32_POST: + IsBaseNoOffset = true; + return AArch64::ST1i32; + case AArch64::ST2i8_POST: + IsBaseNoOffset = true; + return AArch64::ST2i8; + case AArch64::ST2i16_POST: + IsBaseNoOffset = true; + return AArch64::ST2i16; + case AArch64::ST2i32_POST: + IsBaseNoOffset = true; + return AArch64::ST2i32; + case AArch64::ST3i8_POST: + IsBaseNoOffset = true; + return AArch64::ST3i8; + case AArch64::ST3i16_POST: + IsBaseNoOffset = true; + return AArch64::ST3i16; + case AArch64::ST3i32_POST: + IsBaseNoOffset = true; + return AArch64::ST3i32; + case AArch64::ST3i64_POST: + IsBaseNoOffset = true; + return AArch64::ST3i64; + case AArch64::ST4i8_POST: + IsBaseNoOffset = true; + return AArch64::ST4i8; + case AArch64::ST4i16_POST: + IsBaseNoOffset = true; + return AArch64::ST4i16; + case AArch64::ST4i32_POST: + IsBaseNoOffset = true; + return AArch64::ST4i32; + case AArch64::ST4i64_POST: + IsBaseNoOffset = true; + return AArch64::ST4i64; + case AArch64::ST1Onev1d_POST: + IsBaseNoOffset = true; + return AArch64::ST1Onev1d; + case AArch64::ST1Onev2s_POST: + IsBaseNoOffset = true; + return AArch64::ST1Onev2s; + case AArch64::ST1Onev4h_POST: + IsBaseNoOffset = true; + return AArch64::ST1Onev4h; + case AArch64::ST1Onev8b_POST: + IsBaseNoOffset = true; + return AArch64::ST1Onev8b; + case AArch64::ST1Onev2d_POST: + IsBaseNoOffset = true; + return AArch64::ST1Onev2d; + case AArch64::ST1Onev4s_POST: + IsBaseNoOffset = true; + return AArch64::ST1Onev4s; + case AArch64::ST1Onev8h_POST: + IsBaseNoOffset = true; + return AArch64::ST1Onev8h; + case AArch64::ST1Onev16b_POST: + IsBaseNoOffset = true; + return AArch64::ST1Onev16b; + case AArch64::ST1Twov1d_POST: + IsBaseNoOffset = true; + return AArch64::ST1Twov1d; + case AArch64::ST1Twov2s_POST: + IsBaseNoOffset = true; + return AArch64::ST1Twov2s; + case AArch64::ST1Twov4h_POST: + IsBaseNoOffset = true; + return AArch64::ST1Twov4h; + case AArch64::ST1Twov8b_POST: + IsBaseNoOffset = true; + return AArch64::ST1Twov8b; + case AArch64::ST1Twov2d_POST: + IsBaseNoOffset = true; + return AArch64::ST1Twov2d; + case AArch64::ST1Twov4s_POST: + IsBaseNoOffset = true; + return AArch64::ST1Twov4s; + case AArch64::ST1Twov8h_POST: + IsBaseNoOffset = true; + return AArch64::ST1Twov8h; + case AArch64::ST1Twov16b_POST: + IsBaseNoOffset = true; + return AArch64::ST1Twov16b; + case AArch64::ST1Threev1d_POST: + IsBaseNoOffset = true; + return AArch64::ST1Threev1d; + case AArch64::ST1Threev2s_POST: + IsBaseNoOffset = true; + return AArch64::ST1Threev2s; + case AArch64::ST1Threev4h_POST: + IsBaseNoOffset = true; + return AArch64::ST1Threev4h; + case AArch64::ST1Threev8b_POST: + IsBaseNoOffset = true; + return AArch64::ST1Threev8b; + case AArch64::ST1Threev2d_POST: + IsBaseNoOffset = true; + return AArch64::ST1Threev2d; + case AArch64::ST1Threev4s_POST: + IsBaseNoOffset = true; + return AArch64::ST1Threev4s; + case AArch64::ST1Threev8h_POST: + IsBaseNoOffset = true; + return AArch64::ST1Threev8h; + case AArch64::ST1Threev16b_POST: + IsBaseNoOffset = true; + return AArch64::ST1Threev16b; + case AArch64::ST1Fourv1d_POST: + IsBaseNoOffset = true; + return AArch64::ST1Fourv1d; + case AArch64::ST1Fourv2s_POST: + IsBaseNoOffset = true; + return AArch64::ST1Fourv2s; + case AArch64::ST1Fourv4h_POST: + IsBaseNoOffset = true; + return AArch64::ST1Fourv4h; + case AArch64::ST1Fourv8b_POST: + IsBaseNoOffset = true; + return AArch64::ST1Fourv8b; + case AArch64::ST1Fourv2d_POST: + IsBaseNoOffset = true; + return AArch64::ST1Fourv2d; + case AArch64::ST1Fourv4s_POST: + IsBaseNoOffset = true; + return AArch64::ST1Fourv4s; + case AArch64::ST1Fourv8h_POST: + IsBaseNoOffset = true; + return AArch64::ST1Fourv8h; + case AArch64::ST1Fourv16b_POST: + IsBaseNoOffset = true; + return AArch64::ST1Fourv16b; + case AArch64::ST2Twov2s_POST: + IsBaseNoOffset = true; + return AArch64::ST2Twov2s; + case AArch64::ST2Twov4s_POST: + IsBaseNoOffset = true; + return AArch64::ST2Twov4s; + case AArch64::ST2Twov8b_POST: + IsBaseNoOffset = true; + return AArch64::ST2Twov8b; + case AArch64::ST2Twov2d_POST: + IsBaseNoOffset = true; + return AArch64::ST2Twov2d; + case AArch64::ST2Twov4h_POST: + IsBaseNoOffset = true; + return AArch64::ST2Twov4h; + case AArch64::ST2Twov8h_POST: + IsBaseNoOffset = true; + return AArch64::ST2Twov8h; + case AArch64::ST2Twov16b_POST: + IsBaseNoOffset = true; + return AArch64::ST2Twov16b; + case AArch64::ST3Threev2s_POST: + IsBaseNoOffset = true; + return AArch64::ST3Threev2s; + case AArch64::ST3Threev4h_POST: + IsBaseNoOffset = true; + return AArch64::ST3Threev4h; + case AArch64::ST3Threev8b_POST: + IsBaseNoOffset = true; + return AArch64::ST3Threev8b; + case AArch64::ST3Threev2d_POST: + IsBaseNoOffset = true; + return AArch64::ST3Threev2d; + case AArch64::ST3Threev4s_POST: + IsBaseNoOffset = true; + return AArch64::ST3Threev4s; + case AArch64::ST3Threev8h_POST: + IsBaseNoOffset = true; + return AArch64::ST3Threev8h; + case AArch64::ST3Threev16b_POST: + IsBaseNoOffset = true; + return AArch64::ST3Threev16b; + case AArch64::ST4Fourv2s_POST: + IsBaseNoOffset = true; + return AArch64::ST4Fourv2s; + case AArch64::ST4Fourv4h_POST: + IsBaseNoOffset = true; + return AArch64::ST4Fourv4h; + case AArch64::ST4Fourv8b_POST: + IsBaseNoOffset = true; + return AArch64::ST4Fourv8b; + case AArch64::ST4Fourv2d_POST: + IsBaseNoOffset = true; + return AArch64::ST4Fourv2d; + case AArch64::ST4Fourv4s_POST: + IsBaseNoOffset = true; + return AArch64::ST4Fourv4s; + case AArch64::ST4Fourv8h_POST: + IsBaseNoOffset = true; + return AArch64::ST4Fourv8h; + case AArch64::ST4Fourv16b_POST: + IsBaseNoOffset = true; + return AArch64::ST4Fourv16b; + } + return AArch64::INSTRUCTION_LIST_END; +} + +static unsigned getPrePostScale(unsigned Op) { + switch (Op) { + case AArch64::LDPDpost: + case AArch64::LDPDpre: + return 8; + case AArch64::LDPQpost: + case AArch64::LDPQpre: + return 16; + case AArch64::LDPSWpost: + case AArch64::LDPSWpre: + return 4; + case AArch64::LDPSpost: + case AArch64::LDPSpre: + return 4; + case AArch64::LDPWpost: + case AArch64::LDPWpre: + return 4; + case AArch64::LDPXpost: + case AArch64::LDPXpre: + return 8; + case AArch64::STPDpost: + case AArch64::STPDpre: + return 8; + case AArch64::STPQpost: + case AArch64::STPQpre: + return 16; + case AArch64::STPSpost: + case AArch64::STPSpre: + return 4; + case AArch64::STPWpost: + case AArch64::STPWpre: + return 4; + case AArch64::STPXpost: + case AArch64::STPXpre: + return 8; + } + return 0; +} + +static std::optional getAtomicLoadStoreInfo(const MCInst &Inst) { + int DestRegIdx; + int BaseRegIdx; + const int OffsetIdx = -1; + const bool IsPrePost = false; + bool IsPair = false; + // (ST|LD)OPRegister: ld*, st* + // CompareAndSwap(Pair)?(Unprvileged)? + // Swap(LSUI)? + // StoreRelease + // LoadAcquire + switch (Inst.getOpcode()) { + // Compare-and-swap CAS(A)?(L)?(B|H|W|X) + // MIR: Rs = op Rs, Rt, [Xn] + case AArch64::CASB: + case AArch64::CASH: + case AArch64::CASW: + case AArch64::CASX: + case AArch64::CASAB: + case AArch64::CASAH: + case AArch64::CASAW: + case AArch64::CASAX: + case AArch64::CASALB: + case AArch64::CASALH: + case AArch64::CASALW: + case AArch64::CASALX: + case AArch64::CASLB: + case AArch64::CASLH: + case AArch64::CASLW: + case AArch64::CASLX: + DestRegIdx = 2; + BaseRegIdx = 3; + break; + // rs1_rs2 = op rs1_rs2, rt1_rt2, [Xn] + case AArch64::CASPW: + case AArch64::CASPX: + case AArch64::CASPAW: + case AArch64::CASPAX: + case AArch64::CASPALW: + case AArch64::CASPALX: + case AArch64::CASPLW: + case AArch64::CASPLX: + DestRegIdx = 2; + BaseRegIdx = 3; + IsPair = true; + break; + // swap SWP(A)?(L)?(B|H|W|X) + // MIR: op Rs, Rt, [Xn] + case AArch64::SWPB: + case AArch64::SWPH: + case AArch64::SWPW: + case AArch64::SWPX: + case AArch64::SWPAB: + case AArch64::SWPAH: + case AArch64::SWPAW: + case AArch64::SWPAX: + case AArch64::SWPALB: + case AArch64::SWPALH: + case AArch64::SWPALW: + case AArch64::SWPALX: + case AArch64::SWPLB: + case AArch64::SWPLH: + case AArch64::SWPLW: + case AArch64::SWPLX: + DestRegIdx = 1; + BaseRegIdx = 2; + break; + // LD(ADD|CLR|EOR|SET)(A)?(L)?(B|H|W|X) + // op Rs, Rt, [Xn] + // if Rt == xzr, LD* aliases to ST* + case AArch64::LDADDB: + case AArch64::LDADDH: + case AArch64::LDADDX: + case AArch64::LDADDW: + case AArch64::LDADDAB: + case AArch64::LDADDAH: + case AArch64::LDADDAW: + case AArch64::LDADDAX: + case AArch64::LDADDALB: + case AArch64::LDADDALH: + case AArch64::LDADDALW: + case AArch64::LDADDALX: + case AArch64::LDADDLB: + case AArch64::LDADDLH: + case AArch64::LDADDLX: + case AArch64::LDADDLW: + DestRegIdx = 1; + BaseRegIdx = 2; + break; + case AArch64::LDCLRB: + case AArch64::LDCLRH: + case AArch64::LDCLRX: + case AArch64::LDCLRW: + case AArch64::LDCLRAB: + case AArch64::LDCLRAH: + case AArch64::LDCLRAW: + case AArch64::LDCLRAX: + case AArch64::LDCLRALB: + case AArch64::LDCLRALH: + case AArch64::LDCLRALW: + case AArch64::LDCLRALX: + case AArch64::LDCLRLB: + case AArch64::LDCLRLH: + case AArch64::LDCLRLX: + case AArch64::LDCLRLW: + DestRegIdx = 1; + BaseRegIdx = 2; + break; + case AArch64::LDEORB: + case AArch64::LDEORH: + case AArch64::LDEORX: + case AArch64::LDEORW: + case AArch64::LDEORAB: + case AArch64::LDEORAH: + case AArch64::LDEORAW: + case AArch64::LDEORAX: + case AArch64::LDEORALB: + case AArch64::LDEORALH: + case AArch64::LDEORALW: + case AArch64::LDEORALX: + case AArch64::LDEORLB: + case AArch64::LDEORLH: + case AArch64::LDEORLX: + case AArch64::LDEORLW: + DestRegIdx = 1; + BaseRegIdx = 2; + break; + case AArch64::LDSETB: + case AArch64::LDSETH: + case AArch64::LDSETX: + case AArch64::LDSETW: + case AArch64::LDSETAB: + case AArch64::LDSETAH: + case AArch64::LDSETAW: + case AArch64::LDSETAX: + case AArch64::LDSETALB: + case AArch64::LDSETALH: + case AArch64::LDSETALW: + case AArch64::LDSETALX: + case AArch64::LDSETLB: + case AArch64::LDSETLH: + case AArch64::LDSETLX: + case AArch64::LDSETLW: + DestRegIdx = 1; + BaseRegIdx = 2; + break; + default: + return std::nullopt; + } + return MemInstInfo{DestRegIdx, BaseRegIdx, OffsetIdx, IsPrePost, IsPair}; +} diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.h b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.h new file mode 100644 index 0000000000000..d664ef9e37757 --- /dev/null +++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.h @@ -0,0 +1,106 @@ +//===- AArch64MCLFIExpander.h -----------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +// This file was written by the Native Client authors, modified for LFI. +// +//===----------------------------------------------------------------------===// +// +// This file declares the AArch64MCLFIExpander class, the AArch64 specific +// subclass of MCLFIExpander. +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_MC_AARCH64MCLFIEXPANDER_H +#define LLVM_MC_AARCH64MCLFIEXPANDER_H + +#include "Utils/AArch64BaseInfo.h" +#include "llvm/MC/MCInstrInfo.h" +#include "llvm/MC/MCLFIExpander.h" +#include "llvm/MC/MCRegisterInfo.h" + +namespace llvm { +class MCContext; +class MCInst; +class MCStreamer; +class MCSubtargetInfo; + +namespace AArch64 { +class AArch64MCLFIExpander : public MCLFIExpander { +public: + AArch64MCLFIExpander(MCContext &Ctx, std::unique_ptr &&RI, + std::unique_ptr &&II) + : MCLFIExpander(Ctx, std::move(RI), std::move(II)) {} + + bool expandInst(const MCInst &Inst, MCStreamer &Out, + const MCSubtargetInfo &STI) override; + +protected: + bool isValidScratchRegister(MCRegister Reg) const override; + +private: + bool Guard = false; // recursion guard + + MCRegister getScratch(); + + void expandIndirectBranch(const MCInst &Inst, MCStreamer &Out, + const MCSubtargetInfo &STI, bool isCall); + + void expandCall(const MCInst &Inst, MCStreamer &Out, + const MCSubtargetInfo &STI); + + void expandReturn(const MCInst &Inst, MCStreamer &Out, + const MCSubtargetInfo &STI); + + bool mayModifyStack(const MCInst &Inst); + + bool mayModifyReserved(const MCInst &Inst); + + bool mayModifyLR(const MCInst &Inst); + + void expandControlFlow(const MCInst &Inst, MCStreamer &Out, + const MCSubtargetInfo &STI); + + void expandStackModification(const MCInst &Inst, MCStreamer &Out, + const MCSubtargetInfo &STI); + + void expandLRModification(const MCInst &Inst, MCStreamer &Out, + const MCSubtargetInfo &STI); + + void expandPrefetch(const MCInst &Inst, MCStreamer &Out, + const MCSubtargetInfo &STI); + + void expandLoadStore(const MCInst &Inst, MCStreamer &Out, + const MCSubtargetInfo &STI); + + void expandLoadStoreBasic(const MCInst &Inst, MemInstInfo &InstInfo, + MCStreamer &Out, const MCSubtargetInfo &STI); + + void expandLoadStoreRoW(const MCInst &Inst, MemInstInfo &InstInfo, + MCStreamer &Out, const MCSubtargetInfo &STI); + + void expandSyscall(const MCInst &Inst, MCStreamer &Out, + const MCSubtargetInfo &STI); + + void expandTLSRead(const MCInst &Inst, MCStreamer &Out, + const MCSubtargetInfo &STI); + + void expandTLSWrite(const MCInst &Inst, MCStreamer &Out, + const MCSubtargetInfo &STI); + + void doExpandInst(const MCInst &Inst, MCStreamer &Out, + const MCSubtargetInfo &STI); + + enum LFICallType { + LFISyscall, + LFITLSRead, + LFITLSWrite, + }; + + void emitLFICall(LFICallType CallType, MCStreamer &Out, + const MCSubtargetInfo &STI); +}; +} // namespace AArch64 +} // namespace llvm +#endif diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp index c7f44ec018f5a..c7a044346baab 100644 --- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp +++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp @@ -13,6 +13,7 @@ #include "AArch64MCTargetDesc.h" #include "AArch64ELFStreamer.h" #include "AArch64MCAsmInfo.h" +#include "AArch64MCLFIExpander.h" #include "AArch64WinCOFFStreamer.h" #include "MCTargetDesc/AArch64AddressingModes.h" #include "MCTargetDesc/AArch64InstPrinter.h" @@ -518,6 +519,16 @@ static MCInstrAnalysis *createAArch64InstrAnalysis(const MCInstrInfo *Info) { return new AArch64MCInstrAnalysis(Info); } +static MCLFIExpander *createAArch64MCLFIExpander(MCStreamer &S, + std::unique_ptr &&RegInfo, + std::unique_ptr &&InstInfo) { + auto *Exp = new AArch64::AArch64MCLFIExpander(S.getContext(), + std::move(RegInfo), + std::move(InstInfo)); + S.setLFIExpander(Exp); + return Exp; +} + // Force static initialization. extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAArch64TargetMC() { for (Target *T : {&getTheAArch64leTarget(), &getTheAArch64beTarget(), @@ -546,6 +557,8 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAArch64TargetMC() { TargetRegistry::RegisterMachOStreamer(*T, createMachOStreamer); TargetRegistry::RegisterCOFFStreamer(*T, createWinCOFFStreamer); + TargetRegistry::RegisterMCLFIExpander(*T, createAArch64MCLFIExpander); + // Register the obj target streamer. TargetRegistry::RegisterObjectTargetStreamer( *T, createAArch64ObjectTargetStreamer); diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/CMakeLists.txt b/llvm/lib/Target/AArch64/MCTargetDesc/CMakeLists.txt index 7f220657e45f8..bd144eaddc0f4 100644 --- a/llvm/lib/Target/AArch64/MCTargetDesc/CMakeLists.txt +++ b/llvm/lib/Target/AArch64/MCTargetDesc/CMakeLists.txt @@ -6,6 +6,7 @@ add_llvm_component_library(LLVMAArch64Desc AArch64MCAsmInfo.cpp AArch64MCCodeEmitter.cpp AArch64MCExpr.cpp + AArch64MCLFIExpander.cpp AArch64MCTargetDesc.cpp AArch64MachObjectWriter.cpp AArch64TargetStreamer.cpp diff --git a/llvm/lib/Target/AArch64/Utils/AArch64BaseInfo.cpp b/llvm/lib/Target/AArch64/Utils/AArch64BaseInfo.cpp index 7767028c20bcc..756fdca32dcc6 100644 --- a/llvm/lib/Target/AArch64/Utils/AArch64BaseInfo.cpp +++ b/llvm/lib/Target/AArch64/Utils/AArch64BaseInfo.cpp @@ -190,4 +190,356 @@ namespace llvm { #define GET_SVCRsList_IMPL #include "AArch64GenSystemOperands.inc" } + +static const LdStNInstrDesc LdStNInstInfo[] = { + { AArch64::LD1i8, "ld1", ".b", 1, true, 0 }, + { AArch64::LD1i16, "ld1", ".h", 1, true, 0 }, + { AArch64::LD1i32, "ld1", ".s", 1, true, 0 }, + { AArch64::LD1i64, "ld1", ".d", 1, true, 0 }, + { AArch64::LD1i8_POST, "ld1", ".b", 2, true, 1 }, + { AArch64::LD1i16_POST, "ld1", ".h", 2, true, 2 }, + { AArch64::LD1i32_POST, "ld1", ".s", 2, true, 4 }, + { AArch64::LD1i64_POST, "ld1", ".d", 2, true, 8 }, + { AArch64::LD1Rv16b, "ld1r", ".16b", 0, false, 0 }, + { AArch64::LD1Rv8h, "ld1r", ".8h", 0, false, 0 }, + { AArch64::LD1Rv4s, "ld1r", ".4s", 0, false, 0 }, + { AArch64::LD1Rv2d, "ld1r", ".2d", 0, false, 0 }, + { AArch64::LD1Rv8b, "ld1r", ".8b", 0, false, 0 }, + { AArch64::LD1Rv4h, "ld1r", ".4h", 0, false, 0 }, + { AArch64::LD1Rv2s, "ld1r", ".2s", 0, false, 0 }, + { AArch64::LD1Rv1d, "ld1r", ".1d", 0, false, 0 }, + { AArch64::LD1Rv16b_POST, "ld1r", ".16b", 1, false, 1 }, + { AArch64::LD1Rv8h_POST, "ld1r", ".8h", 1, false, 2 }, + { AArch64::LD1Rv4s_POST, "ld1r", ".4s", 1, false, 4 }, + { AArch64::LD1Rv2d_POST, "ld1r", ".2d", 1, false, 8 }, + { AArch64::LD1Rv8b_POST, "ld1r", ".8b", 1, false, 1 }, + { AArch64::LD1Rv4h_POST, "ld1r", ".4h", 1, false, 2 }, + { AArch64::LD1Rv2s_POST, "ld1r", ".2s", 1, false, 4 }, + { AArch64::LD1Rv1d_POST, "ld1r", ".1d", 1, false, 8 }, + { AArch64::LD1Onev16b, "ld1", ".16b", 0, false, 0 }, + { AArch64::LD1Onev8h, "ld1", ".8h", 0, false, 0 }, + { AArch64::LD1Onev4s, "ld1", ".4s", 0, false, 0 }, + { AArch64::LD1Onev2d, "ld1", ".2d", 0, false, 0 }, + { AArch64::LD1Onev8b, "ld1", ".8b", 0, false, 0 }, + { AArch64::LD1Onev4h, "ld1", ".4h", 0, false, 0 }, + { AArch64::LD1Onev2s, "ld1", ".2s", 0, false, 0 }, + { AArch64::LD1Onev1d, "ld1", ".1d", 0, false, 0 }, + { AArch64::LD1Onev16b_POST, "ld1", ".16b", 1, false, 16 }, + { AArch64::LD1Onev8h_POST, "ld1", ".8h", 1, false, 16 }, + { AArch64::LD1Onev4s_POST, "ld1", ".4s", 1, false, 16 }, + { AArch64::LD1Onev2d_POST, "ld1", ".2d", 1, false, 16 }, + { AArch64::LD1Onev8b_POST, "ld1", ".8b", 1, false, 8 }, + { AArch64::LD1Onev4h_POST, "ld1", ".4h", 1, false, 8 }, + { AArch64::LD1Onev2s_POST, "ld1", ".2s", 1, false, 8 }, + { AArch64::LD1Onev1d_POST, "ld1", ".1d", 1, false, 8 }, + { AArch64::LD1Twov16b, "ld1", ".16b", 0, false, 0 }, + { AArch64::LD1Twov8h, "ld1", ".8h", 0, false, 0 }, + { AArch64::LD1Twov4s, "ld1", ".4s", 0, false, 0 }, + { AArch64::LD1Twov2d, "ld1", ".2d", 0, false, 0 }, + { AArch64::LD1Twov8b, "ld1", ".8b", 0, false, 0 }, + { AArch64::LD1Twov4h, "ld1", ".4h", 0, false, 0 }, + { AArch64::LD1Twov2s, "ld1", ".2s", 0, false, 0 }, + { AArch64::LD1Twov1d, "ld1", ".1d", 0, false, 0 }, + { AArch64::LD1Twov16b_POST, "ld1", ".16b", 1, false, 32 }, + { AArch64::LD1Twov8h_POST, "ld1", ".8h", 1, false, 32 }, + { AArch64::LD1Twov4s_POST, "ld1", ".4s", 1, false, 32 }, + { AArch64::LD1Twov2d_POST, "ld1", ".2d", 1, false, 32 }, + { AArch64::LD1Twov8b_POST, "ld1", ".8b", 1, false, 16 }, + { AArch64::LD1Twov4h_POST, "ld1", ".4h", 1, false, 16 }, + { AArch64::LD1Twov2s_POST, "ld1", ".2s", 1, false, 16 }, + { AArch64::LD1Twov1d_POST, "ld1", ".1d", 1, false, 16 }, + { AArch64::LD1Threev16b, "ld1", ".16b", 0, false, 0 }, + { AArch64::LD1Threev8h, "ld1", ".8h", 0, false, 0 }, + { AArch64::LD1Threev4s, "ld1", ".4s", 0, false, 0 }, + { AArch64::LD1Threev2d, "ld1", ".2d", 0, false, 0 }, + { AArch64::LD1Threev8b, "ld1", ".8b", 0, false, 0 }, + { AArch64::LD1Threev4h, "ld1", ".4h", 0, false, 0 }, + { AArch64::LD1Threev2s, "ld1", ".2s", 0, false, 0 }, + { AArch64::LD1Threev1d, "ld1", ".1d", 0, false, 0 }, + { AArch64::LD1Threev16b_POST, "ld1", ".16b", 1, false, 48 }, + { AArch64::LD1Threev8h_POST, "ld1", ".8h", 1, false, 48 }, + { AArch64::LD1Threev4s_POST, "ld1", ".4s", 1, false, 48 }, + { AArch64::LD1Threev2d_POST, "ld1", ".2d", 1, false, 48 }, + { AArch64::LD1Threev8b_POST, "ld1", ".8b", 1, false, 24 }, + { AArch64::LD1Threev4h_POST, "ld1", ".4h", 1, false, 24 }, + { AArch64::LD1Threev2s_POST, "ld1", ".2s", 1, false, 24 }, + { AArch64::LD1Threev1d_POST, "ld1", ".1d", 1, false, 24 }, + { AArch64::LD1Fourv16b, "ld1", ".16b", 0, false, 0 }, + { AArch64::LD1Fourv8h, "ld1", ".8h", 0, false, 0 }, + { AArch64::LD1Fourv4s, "ld1", ".4s", 0, false, 0 }, + { AArch64::LD1Fourv2d, "ld1", ".2d", 0, false, 0 }, + { AArch64::LD1Fourv8b, "ld1", ".8b", 0, false, 0 }, + { AArch64::LD1Fourv4h, "ld1", ".4h", 0, false, 0 }, + { AArch64::LD1Fourv2s, "ld1", ".2s", 0, false, 0 }, + { AArch64::LD1Fourv1d, "ld1", ".1d", 0, false, 0 }, + { AArch64::LD1Fourv16b_POST, "ld1", ".16b", 1, false, 64 }, + { AArch64::LD1Fourv8h_POST, "ld1", ".8h", 1, false, 64 }, + { AArch64::LD1Fourv4s_POST, "ld1", ".4s", 1, false, 64 }, + { AArch64::LD1Fourv2d_POST, "ld1", ".2d", 1, false, 64 }, + { AArch64::LD1Fourv8b_POST, "ld1", ".8b", 1, false, 32 }, + { AArch64::LD1Fourv4h_POST, "ld1", ".4h", 1, false, 32 }, + { AArch64::LD1Fourv2s_POST, "ld1", ".2s", 1, false, 32 }, + { AArch64::LD1Fourv1d_POST, "ld1", ".1d", 1, false, 32 }, + { AArch64::LD2i8, "ld2", ".b", 1, true, 0 }, + { AArch64::LD2i16, "ld2", ".h", 1, true, 0 }, + { AArch64::LD2i32, "ld2", ".s", 1, true, 0 }, + { AArch64::LD2i64, "ld2", ".d", 1, true, 0 }, + { AArch64::LD2i8_POST, "ld2", ".b", 2, true, 2 }, + { AArch64::LD2i16_POST, "ld2", ".h", 2, true, 4 }, + { AArch64::LD2i32_POST, "ld2", ".s", 2, true, 8 }, + { AArch64::LD2i64_POST, "ld2", ".d", 2, true, 16 }, + { AArch64::LD2Rv16b, "ld2r", ".16b", 0, false, 0 }, + { AArch64::LD2Rv8h, "ld2r", ".8h", 0, false, 0 }, + { AArch64::LD2Rv4s, "ld2r", ".4s", 0, false, 0 }, + { AArch64::LD2Rv2d, "ld2r", ".2d", 0, false, 0 }, + { AArch64::LD2Rv8b, "ld2r", ".8b", 0, false, 0 }, + { AArch64::LD2Rv4h, "ld2r", ".4h", 0, false, 0 }, + { AArch64::LD2Rv2s, "ld2r", ".2s", 0, false, 0 }, + { AArch64::LD2Rv1d, "ld2r", ".1d", 0, false, 0 }, + { AArch64::LD2Rv16b_POST, "ld2r", ".16b", 1, false, 2 }, + { AArch64::LD2Rv8h_POST, "ld2r", ".8h", 1, false, 4 }, + { AArch64::LD2Rv4s_POST, "ld2r", ".4s", 1, false, 8 }, + { AArch64::LD2Rv2d_POST, "ld2r", ".2d", 1, false, 16 }, + { AArch64::LD2Rv8b_POST, "ld2r", ".8b", 1, false, 2 }, + { AArch64::LD2Rv4h_POST, "ld2r", ".4h", 1, false, 4 }, + { AArch64::LD2Rv2s_POST, "ld2r", ".2s", 1, false, 8 }, + { AArch64::LD2Rv1d_POST, "ld2r", ".1d", 1, false, 16 }, + { AArch64::LD2Twov16b, "ld2", ".16b", 0, false, 0 }, + { AArch64::LD2Twov8h, "ld2", ".8h", 0, false, 0 }, + { AArch64::LD2Twov4s, "ld2", ".4s", 0, false, 0 }, + { AArch64::LD2Twov2d, "ld2", ".2d", 0, false, 0 }, + { AArch64::LD2Twov8b, "ld2", ".8b", 0, false, 0 }, + { AArch64::LD2Twov4h, "ld2", ".4h", 0, false, 0 }, + { AArch64::LD2Twov2s, "ld2", ".2s", 0, false, 0 }, + { AArch64::LD2Twov16b_POST, "ld2", ".16b", 1, false, 32 }, + { AArch64::LD2Twov8h_POST, "ld2", ".8h", 1, false, 32 }, + { AArch64::LD2Twov4s_POST, "ld2", ".4s", 1, false, 32 }, + { AArch64::LD2Twov2d_POST, "ld2", ".2d", 1, false, 32 }, + { AArch64::LD2Twov8b_POST, "ld2", ".8b", 1, false, 16 }, + { AArch64::LD2Twov4h_POST, "ld2", ".4h", 1, false, 16 }, + { AArch64::LD2Twov2s_POST, "ld2", ".2s", 1, false, 16 }, + { AArch64::LD3i8, "ld3", ".b", 1, true, 0 }, + { AArch64::LD3i16, "ld3", ".h", 1, true, 0 }, + { AArch64::LD3i32, "ld3", ".s", 1, true, 0 }, + { AArch64::LD3i64, "ld3", ".d", 1, true, 0 }, + { AArch64::LD3i8_POST, "ld3", ".b", 2, true, 3 }, + { AArch64::LD3i16_POST, "ld3", ".h", 2, true, 6 }, + { AArch64::LD3i32_POST, "ld3", ".s", 2, true, 12 }, + { AArch64::LD3i64_POST, "ld3", ".d", 2, true, 24 }, + { AArch64::LD3Rv16b, "ld3r", ".16b", 0, false, 0 }, + { AArch64::LD3Rv8h, "ld3r", ".8h", 0, false, 0 }, + { AArch64::LD3Rv4s, "ld3r", ".4s", 0, false, 0 }, + { AArch64::LD3Rv2d, "ld3r", ".2d", 0, false, 0 }, + { AArch64::LD3Rv8b, "ld3r", ".8b", 0, false, 0 }, + { AArch64::LD3Rv4h, "ld3r", ".4h", 0, false, 0 }, + { AArch64::LD3Rv2s, "ld3r", ".2s", 0, false, 0 }, + { AArch64::LD3Rv1d, "ld3r", ".1d", 0, false, 0 }, + { AArch64::LD3Rv16b_POST, "ld3r", ".16b", 1, false, 3 }, + { AArch64::LD3Rv8h_POST, "ld3r", ".8h", 1, false, 6 }, + { AArch64::LD3Rv4s_POST, "ld3r", ".4s", 1, false, 12 }, + { AArch64::LD3Rv2d_POST, "ld3r", ".2d", 1, false, 24 }, + { AArch64::LD3Rv8b_POST, "ld3r", ".8b", 1, false, 3 }, + { AArch64::LD3Rv4h_POST, "ld3r", ".4h", 1, false, 6 }, + { AArch64::LD3Rv2s_POST, "ld3r", ".2s", 1, false, 12 }, + { AArch64::LD3Rv1d_POST, "ld3r", ".1d", 1, false, 24 }, + { AArch64::LD3Threev16b, "ld3", ".16b", 0, false, 0 }, + { AArch64::LD3Threev8h, "ld3", ".8h", 0, false, 0 }, + { AArch64::LD3Threev4s, "ld3", ".4s", 0, false, 0 }, + { AArch64::LD3Threev2d, "ld3", ".2d", 0, false, 0 }, + { AArch64::LD3Threev8b, "ld3", ".8b", 0, false, 0 }, + { AArch64::LD3Threev4h, "ld3", ".4h", 0, false, 0 }, + { AArch64::LD3Threev2s, "ld3", ".2s", 0, false, 0 }, + { AArch64::LD3Threev16b_POST, "ld3", ".16b", 1, false, 48 }, + { AArch64::LD3Threev8h_POST, "ld3", ".8h", 1, false, 48 }, + { AArch64::LD3Threev4s_POST, "ld3", ".4s", 1, false, 48 }, + { AArch64::LD3Threev2d_POST, "ld3", ".2d", 1, false, 48 }, + { AArch64::LD3Threev8b_POST, "ld3", ".8b", 1, false, 24 }, + { AArch64::LD3Threev4h_POST, "ld3", ".4h", 1, false, 24 }, + { AArch64::LD3Threev2s_POST, "ld3", ".2s", 1, false, 24 }, + { AArch64::LD4i8, "ld4", ".b", 1, true, 0 }, + { AArch64::LD4i16, "ld4", ".h", 1, true, 0 }, + { AArch64::LD4i32, "ld4", ".s", 1, true, 0 }, + { AArch64::LD4i64, "ld4", ".d", 1, true, 0 }, + { AArch64::LD4i8_POST, "ld4", ".b", 2, true, 4 }, + { AArch64::LD4i16_POST, "ld4", ".h", 2, true, 8 }, + { AArch64::LD4i32_POST, "ld4", ".s", 2, true, 16 }, + { AArch64::LD4i64_POST, "ld4", ".d", 2, true, 32 }, + { AArch64::LD4Rv16b, "ld4r", ".16b", 0, false, 0 }, + { AArch64::LD4Rv8h, "ld4r", ".8h", 0, false, 0 }, + { AArch64::LD4Rv4s, "ld4r", ".4s", 0, false, 0 }, + { AArch64::LD4Rv2d, "ld4r", ".2d", 0, false, 0 }, + { AArch64::LD4Rv8b, "ld4r", ".8b", 0, false, 0 }, + { AArch64::LD4Rv4h, "ld4r", ".4h", 0, false, 0 }, + { AArch64::LD4Rv2s, "ld4r", ".2s", 0, false, 0 }, + { AArch64::LD4Rv1d, "ld4r", ".1d", 0, false, 0 }, + { AArch64::LD4Rv16b_POST, "ld4r", ".16b", 1, false, 4 }, + { AArch64::LD4Rv8h_POST, "ld4r", ".8h", 1, false, 8 }, + { AArch64::LD4Rv4s_POST, "ld4r", ".4s", 1, false, 16 }, + { AArch64::LD4Rv2d_POST, "ld4r", ".2d", 1, false, 32 }, + { AArch64::LD4Rv8b_POST, "ld4r", ".8b", 1, false, 4 }, + { AArch64::LD4Rv4h_POST, "ld4r", ".4h", 1, false, 8 }, + { AArch64::LD4Rv2s_POST, "ld4r", ".2s", 1, false, 16 }, + { AArch64::LD4Rv1d_POST, "ld4r", ".1d", 1, false, 32 }, + { AArch64::LD4Fourv16b, "ld4", ".16b", 0, false, 0 }, + { AArch64::LD4Fourv8h, "ld4", ".8h", 0, false, 0 }, + { AArch64::LD4Fourv4s, "ld4", ".4s", 0, false, 0 }, + { AArch64::LD4Fourv2d, "ld4", ".2d", 0, false, 0 }, + { AArch64::LD4Fourv8b, "ld4", ".8b", 0, false, 0 }, + { AArch64::LD4Fourv4h, "ld4", ".4h", 0, false, 0 }, + { AArch64::LD4Fourv2s, "ld4", ".2s", 0, false, 0 }, + { AArch64::LD4Fourv16b_POST, "ld4", ".16b", 1, false, 64 }, + { AArch64::LD4Fourv8h_POST, "ld4", ".8h", 1, false, 64 }, + { AArch64::LD4Fourv4s_POST, "ld4", ".4s", 1, false, 64 }, + { AArch64::LD4Fourv2d_POST, "ld4", ".2d", 1, false, 64 }, + { AArch64::LD4Fourv8b_POST, "ld4", ".8b", 1, false, 32 }, + { AArch64::LD4Fourv4h_POST, "ld4", ".4h", 1, false, 32 }, + { AArch64::LD4Fourv2s_POST, "ld4", ".2s", 1, false, 32 }, + { AArch64::ST1i8, "st1", ".b", 0, true, 0 }, + { AArch64::ST1i16, "st1", ".h", 0, true, 0 }, + { AArch64::ST1i32, "st1", ".s", 0, true, 0 }, + { AArch64::ST1i64, "st1", ".d", 0, true, 0 }, + { AArch64::ST1i8_POST, "st1", ".b", 1, true, 1 }, + { AArch64::ST1i16_POST, "st1", ".h", 1, true, 2 }, + { AArch64::ST1i32_POST, "st1", ".s", 1, true, 4 }, + { AArch64::ST1i64_POST, "st1", ".d", 1, true, 8 }, + { AArch64::ST1Onev16b, "st1", ".16b", 0, false, 0 }, + { AArch64::ST1Onev8h, "st1", ".8h", 0, false, 0 }, + { AArch64::ST1Onev4s, "st1", ".4s", 0, false, 0 }, + { AArch64::ST1Onev2d, "st1", ".2d", 0, false, 0 }, + { AArch64::ST1Onev8b, "st1", ".8b", 0, false, 0 }, + { AArch64::ST1Onev4h, "st1", ".4h", 0, false, 0 }, + { AArch64::ST1Onev2s, "st1", ".2s", 0, false, 0 }, + { AArch64::ST1Onev1d, "st1", ".1d", 0, false, 0 }, + { AArch64::ST1Onev16b_POST, "st1", ".16b", 1, false, 16 }, + { AArch64::ST1Onev8h_POST, "st1", ".8h", 1, false, 16 }, + { AArch64::ST1Onev4s_POST, "st1", ".4s", 1, false, 16 }, + { AArch64::ST1Onev2d_POST, "st1", ".2d", 1, false, 16 }, + { AArch64::ST1Onev8b_POST, "st1", ".8b", 1, false, 8 }, + { AArch64::ST1Onev4h_POST, "st1", ".4h", 1, false, 8 }, + { AArch64::ST1Onev2s_POST, "st1", ".2s", 1, false, 8 }, + { AArch64::ST1Onev1d_POST, "st1", ".1d", 1, false, 8 }, + { AArch64::ST1Twov16b, "st1", ".16b", 0, false, 0 }, + { AArch64::ST1Twov8h, "st1", ".8h", 0, false, 0 }, + { AArch64::ST1Twov4s, "st1", ".4s", 0, false, 0 }, + { AArch64::ST1Twov2d, "st1", ".2d", 0, false, 0 }, + { AArch64::ST1Twov8b, "st1", ".8b", 0, false, 0 }, + { AArch64::ST1Twov4h, "st1", ".4h", 0, false, 0 }, + { AArch64::ST1Twov2s, "st1", ".2s", 0, false, 0 }, + { AArch64::ST1Twov1d, "st1", ".1d", 0, false, 0 }, + { AArch64::ST1Twov16b_POST, "st1", ".16b", 1, false, 32 }, + { AArch64::ST1Twov8h_POST, "st1", ".8h", 1, false, 32 }, + { AArch64::ST1Twov4s_POST, "st1", ".4s", 1, false, 32 }, + { AArch64::ST1Twov2d_POST, "st1", ".2d", 1, false, 32 }, + { AArch64::ST1Twov8b_POST, "st1", ".8b", 1, false, 16 }, + { AArch64::ST1Twov4h_POST, "st1", ".4h", 1, false, 16 }, + { AArch64::ST1Twov2s_POST, "st1", ".2s", 1, false, 16 }, + { AArch64::ST1Twov1d_POST, "st1", ".1d", 1, false, 16 }, + { AArch64::ST1Threev16b, "st1", ".16b", 0, false, 0 }, + { AArch64::ST1Threev8h, "st1", ".8h", 0, false, 0 }, + { AArch64::ST1Threev4s, "st1", ".4s", 0, false, 0 }, + { AArch64::ST1Threev2d, "st1", ".2d", 0, false, 0 }, + { AArch64::ST1Threev8b, "st1", ".8b", 0, false, 0 }, + { AArch64::ST1Threev4h, "st1", ".4h", 0, false, 0 }, + { AArch64::ST1Threev2s, "st1", ".2s", 0, false, 0 }, + { AArch64::ST1Threev1d, "st1", ".1d", 0, false, 0 }, + { AArch64::ST1Threev16b_POST, "st1", ".16b", 1, false, 48 }, + { AArch64::ST1Threev8h_POST, "st1", ".8h", 1, false, 48 }, + { AArch64::ST1Threev4s_POST, "st1", ".4s", 1, false, 48 }, + { AArch64::ST1Threev2d_POST, "st1", ".2d", 1, false, 48 }, + { AArch64::ST1Threev8b_POST, "st1", ".8b", 1, false, 24 }, + { AArch64::ST1Threev4h_POST, "st1", ".4h", 1, false, 24 }, + { AArch64::ST1Threev2s_POST, "st1", ".2s", 1, false, 24 }, + { AArch64::ST1Threev1d_POST, "st1", ".1d", 1, false, 24 }, + { AArch64::ST1Fourv16b, "st1", ".16b", 0, false, 0 }, + { AArch64::ST1Fourv8h, "st1", ".8h", 0, false, 0 }, + { AArch64::ST1Fourv4s, "st1", ".4s", 0, false, 0 }, + { AArch64::ST1Fourv2d, "st1", ".2d", 0, false, 0 }, + { AArch64::ST1Fourv8b, "st1", ".8b", 0, false, 0 }, + { AArch64::ST1Fourv4h, "st1", ".4h", 0, false, 0 }, + { AArch64::ST1Fourv2s, "st1", ".2s", 0, false, 0 }, + { AArch64::ST1Fourv1d, "st1", ".1d", 0, false, 0 }, + { AArch64::ST1Fourv16b_POST, "st1", ".16b", 1, false, 64 }, + { AArch64::ST1Fourv8h_POST, "st1", ".8h", 1, false, 64 }, + { AArch64::ST1Fourv4s_POST, "st1", ".4s", 1, false, 64 }, + { AArch64::ST1Fourv2d_POST, "st1", ".2d", 1, false, 64 }, + { AArch64::ST1Fourv8b_POST, "st1", ".8b", 1, false, 32 }, + { AArch64::ST1Fourv4h_POST, "st1", ".4h", 1, false, 32 }, + { AArch64::ST1Fourv2s_POST, "st1", ".2s", 1, false, 32 }, + { AArch64::ST1Fourv1d_POST, "st1", ".1d", 1, false, 32 }, + { AArch64::ST2i8, "st2", ".b", 0, true, 0 }, + { AArch64::ST2i16, "st2", ".h", 0, true, 0 }, + { AArch64::ST2i32, "st2", ".s", 0, true, 0 }, + { AArch64::ST2i64, "st2", ".d", 0, true, 0 }, + { AArch64::ST2i8_POST, "st2", ".b", 1, true, 2 }, + { AArch64::ST2i16_POST, "st2", ".h", 1, true, 4 }, + { AArch64::ST2i32_POST, "st2", ".s", 1, true, 8 }, + { AArch64::ST2i64_POST, "st2", ".d", 1, true, 16 }, + { AArch64::ST2Twov16b, "st2", ".16b", 0, false, 0 }, + { AArch64::ST2Twov8h, "st2", ".8h", 0, false, 0 }, + { AArch64::ST2Twov4s, "st2", ".4s", 0, false, 0 }, + { AArch64::ST2Twov2d, "st2", ".2d", 0, false, 0 }, + { AArch64::ST2Twov8b, "st2", ".8b", 0, false, 0 }, + { AArch64::ST2Twov4h, "st2", ".4h", 0, false, 0 }, + { AArch64::ST2Twov2s, "st2", ".2s", 0, false, 0 }, + { AArch64::ST2Twov16b_POST, "st2", ".16b", 1, false, 32 }, + { AArch64::ST2Twov8h_POST, "st2", ".8h", 1, false, 32 }, + { AArch64::ST2Twov4s_POST, "st2", ".4s", 1, false, 32 }, + { AArch64::ST2Twov2d_POST, "st2", ".2d", 1, false, 32 }, + { AArch64::ST2Twov8b_POST, "st2", ".8b", 1, false, 16 }, + { AArch64::ST2Twov4h_POST, "st2", ".4h", 1, false, 16 }, + { AArch64::ST2Twov2s_POST, "st2", ".2s", 1, false, 16 }, + { AArch64::ST3i8, "st3", ".b", 0, true, 0 }, + { AArch64::ST3i16, "st3", ".h", 0, true, 0 }, + { AArch64::ST3i32, "st3", ".s", 0, true, 0 }, + { AArch64::ST3i64, "st3", ".d", 0, true, 0 }, + { AArch64::ST3i8_POST, "st3", ".b", 1, true, 3 }, + { AArch64::ST3i16_POST, "st3", ".h", 1, true, 6 }, + { AArch64::ST3i32_POST, "st3", ".s", 1, true, 12 }, + { AArch64::ST3i64_POST, "st3", ".d", 1, true, 24 }, + { AArch64::ST3Threev16b, "st3", ".16b", 0, false, 0 }, + { AArch64::ST3Threev8h, "st3", ".8h", 0, false, 0 }, + { AArch64::ST3Threev4s, "st3", ".4s", 0, false, 0 }, + { AArch64::ST3Threev2d, "st3", ".2d", 0, false, 0 }, + { AArch64::ST3Threev8b, "st3", ".8b", 0, false, 0 }, + { AArch64::ST3Threev4h, "st3", ".4h", 0, false, 0 }, + { AArch64::ST3Threev2s, "st3", ".2s", 0, false, 0 }, + { AArch64::ST3Threev16b_POST, "st3", ".16b", 1, false, 48 }, + { AArch64::ST3Threev8h_POST, "st3", ".8h", 1, false, 48 }, + { AArch64::ST3Threev4s_POST, "st3", ".4s", 1, false, 48 }, + { AArch64::ST3Threev2d_POST, "st3", ".2d", 1, false, 48 }, + { AArch64::ST3Threev8b_POST, "st3", ".8b", 1, false, 24 }, + { AArch64::ST3Threev4h_POST, "st3", ".4h", 1, false, 24 }, + { AArch64::ST3Threev2s_POST, "st3", ".2s", 1, false, 24 }, + { AArch64::ST4i8, "st4", ".b", 0, true, 0 }, + { AArch64::ST4i16, "st4", ".h", 0, true, 0 }, + { AArch64::ST4i32, "st4", ".s", 0, true, 0 }, + { AArch64::ST4i64, "st4", ".d", 0, true, 0 }, + { AArch64::ST4i8_POST, "st4", ".b", 1, true, 4 }, + { AArch64::ST4i16_POST, "st4", ".h", 1, true, 8 }, + { AArch64::ST4i32_POST, "st4", ".s", 1, true, 16 }, + { AArch64::ST4i64_POST, "st4", ".d", 1, true, 32 }, + { AArch64::ST4Fourv16b, "st4", ".16b", 0, false, 0 }, + { AArch64::ST4Fourv8h, "st4", ".8h", 0, false, 0 }, + { AArch64::ST4Fourv4s, "st4", ".4s", 0, false, 0 }, + { AArch64::ST4Fourv2d, "st4", ".2d", 0, false, 0 }, + { AArch64::ST4Fourv8b, "st4", ".8b", 0, false, 0 }, + { AArch64::ST4Fourv4h, "st4", ".4h", 0, false, 0 }, + { AArch64::ST4Fourv2s, "st4", ".2s", 0, false, 0 }, + { AArch64::ST4Fourv16b_POST, "st4", ".16b", 1, false, 64 }, + { AArch64::ST4Fourv8h_POST, "st4", ".8h", 1, false, 64 }, + { AArch64::ST4Fourv4s_POST, "st4", ".4s", 1, false, 64 }, + { AArch64::ST4Fourv2d_POST, "st4", ".2d", 1, false, 64 }, + { AArch64::ST4Fourv8b_POST, "st4", ".8b", 1, false, 32 }, + { AArch64::ST4Fourv4h_POST, "st4", ".4h", 1, false, 32 }, + { AArch64::ST4Fourv2s_POST, "st4", ".2s", 1, false, 32 }, +}; + +const LdStNInstrDesc *getLdStNInstrDesc(unsigned Opcode) { + for (const auto &Info : LdStNInstInfo) + if (Info.Opcode == Opcode) + return &Info; + + return nullptr; +} + } diff --git a/llvm/lib/Target/AArch64/Utils/AArch64BaseInfo.h b/llvm/lib/Target/AArch64/Utils/AArch64BaseInfo.h index 9671fa3b3d92f..ffbc69af0dffc 100644 --- a/llvm/lib/Target/AArch64/Utils/AArch64BaseInfo.h +++ b/llvm/lib/Target/AArch64/Utils/AArch64BaseInfo.h @@ -22,6 +22,7 @@ #include "llvm/ADT/BitmaskEnum.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/StringSwitch.h" +#include "llvm/MC/MCInst.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/TargetParser/SubtargetFeature.h" @@ -247,6 +248,826 @@ static inline bool atomicBarrierDroppedOnZero(unsigned Opcode) { return false; } +struct MemInstInfo { + MemInstInfo() = default; + + int DestRegIdx; + int BaseRegIdx; + int OffsetIdx; + bool IsPrePost = false; + bool IsPair = false; +}; + +static inline std::optional getLoadInfo(const MCInst &Inst) { + int DestRegIdx; + int BaseRegIdx; + int OffsetIdx; + bool IsPrePost = false; + bool IsPair = false; + switch (Inst.getOpcode()) { + default: + return std::nullopt; + + case AArch64::LD1i64: + case AArch64::LD2i64: + DestRegIdx = 0; + BaseRegIdx = 3; + OffsetIdx = -1; + IsPrePost = false; + break; + + case AArch64::LD1i8: + case AArch64::LD1i16: + case AArch64::LD1i32: + case AArch64::LD2i8: + case AArch64::LD2i16: + case AArch64::LD2i32: + case AArch64::LD3i8: + case AArch64::LD3i16: + case AArch64::LD3i32: + case AArch64::LD3i64: + case AArch64::LD4i8: + case AArch64::LD4i16: + case AArch64::LD4i32: + case AArch64::LD4i64: + DestRegIdx = -1; + BaseRegIdx = 3; + OffsetIdx = -1; + IsPrePost = false; + break; + + case AArch64::LD1Onev1d: + case AArch64::LD1Onev2s: + case AArch64::LD1Onev4h: + case AArch64::LD1Onev8b: + case AArch64::LD1Onev2d: + case AArch64::LD1Onev4s: + case AArch64::LD1Onev8h: + case AArch64::LD1Onev16b: + case AArch64::LD1Rv1d: + case AArch64::LD1Rv2s: + case AArch64::LD1Rv4h: + case AArch64::LD1Rv8b: + case AArch64::LD1Rv2d: + case AArch64::LD1Rv4s: + case AArch64::LD1Rv8h: + case AArch64::LD1Rv16b: + DestRegIdx = 0; + BaseRegIdx = 1; + OffsetIdx = -1; + IsPrePost = false; + break; + + case AArch64::LD1Twov1d: + case AArch64::LD1Twov2s: + case AArch64::LD1Twov4h: + case AArch64::LD1Twov8b: + case AArch64::LD1Twov2d: + case AArch64::LD1Twov4s: + case AArch64::LD1Twov8h: + case AArch64::LD1Twov16b: + case AArch64::LD1Threev1d: + case AArch64::LD1Threev2s: + case AArch64::LD1Threev4h: + case AArch64::LD1Threev8b: + case AArch64::LD1Threev2d: + case AArch64::LD1Threev4s: + case AArch64::LD1Threev8h: + case AArch64::LD1Threev16b: + case AArch64::LD1Fourv1d: + case AArch64::LD1Fourv2s: + case AArch64::LD1Fourv4h: + case AArch64::LD1Fourv8b: + case AArch64::LD1Fourv2d: + case AArch64::LD1Fourv4s: + case AArch64::LD1Fourv8h: + case AArch64::LD1Fourv16b: + case AArch64::LD2Twov2s: + case AArch64::LD2Twov4s: + case AArch64::LD2Twov8b: + case AArch64::LD2Twov2d: + case AArch64::LD2Twov4h: + case AArch64::LD2Twov8h: + case AArch64::LD2Twov16b: + case AArch64::LD2Rv1d: + case AArch64::LD2Rv2s: + case AArch64::LD2Rv4s: + case AArch64::LD2Rv8b: + case AArch64::LD2Rv2d: + case AArch64::LD2Rv4h: + case AArch64::LD2Rv8h: + case AArch64::LD2Rv16b: + case AArch64::LD3Threev2s: + case AArch64::LD3Threev4h: + case AArch64::LD3Threev8b: + case AArch64::LD3Threev2d: + case AArch64::LD3Threev4s: + case AArch64::LD3Threev8h: + case AArch64::LD3Threev16b: + case AArch64::LD3Rv1d: + case AArch64::LD3Rv2s: + case AArch64::LD3Rv4h: + case AArch64::LD3Rv8b: + case AArch64::LD3Rv2d: + case AArch64::LD3Rv4s: + case AArch64::LD3Rv8h: + case AArch64::LD3Rv16b: + case AArch64::LD4Fourv2s: + case AArch64::LD4Fourv4h: + case AArch64::LD4Fourv8b: + case AArch64::LD4Fourv2d: + case AArch64::LD4Fourv4s: + case AArch64::LD4Fourv8h: + case AArch64::LD4Fourv16b: + case AArch64::LD4Rv1d: + case AArch64::LD4Rv2s: + case AArch64::LD4Rv4h: + case AArch64::LD4Rv8b: + case AArch64::LD4Rv2d: + case AArch64::LD4Rv4s: + case AArch64::LD4Rv8h: + case AArch64::LD4Rv16b: + DestRegIdx = -1; + BaseRegIdx = 1; + OffsetIdx = -1; + IsPrePost = false; + break; + + case AArch64::LD1i64_POST: + case AArch64::LD2i64_POST: + DestRegIdx = 1; + BaseRegIdx = 4; + OffsetIdx = 5; + IsPrePost = true; + break; + + case AArch64::LD1i8_POST: + case AArch64::LD1i16_POST: + case AArch64::LD1i32_POST: + case AArch64::LD2i8_POST: + case AArch64::LD2i16_POST: + case AArch64::LD2i32_POST: + case AArch64::LD3i8_POST: + case AArch64::LD3i16_POST: + case AArch64::LD3i32_POST: + case AArch64::LD3i64_POST: + case AArch64::LD4i8_POST: + case AArch64::LD4i16_POST: + case AArch64::LD4i32_POST: + case AArch64::LD4i64_POST: + DestRegIdx = -1; + BaseRegIdx = 4; + OffsetIdx = 5; + IsPrePost = true; + break; + + case AArch64::LD1Onev1d_POST: + case AArch64::LD1Onev2s_POST: + case AArch64::LD1Onev4h_POST: + case AArch64::LD1Onev8b_POST: + case AArch64::LD1Onev2d_POST: + case AArch64::LD1Onev4s_POST: + case AArch64::LD1Onev8h_POST: + case AArch64::LD1Onev16b_POST: + case AArch64::LD1Rv1d_POST: + case AArch64::LD1Rv2s_POST: + case AArch64::LD1Rv4h_POST: + case AArch64::LD1Rv8b_POST: + case AArch64::LD1Rv2d_POST: + case AArch64::LD1Rv4s_POST: + case AArch64::LD1Rv8h_POST: + case AArch64::LD1Rv16b_POST: + DestRegIdx = 1; + BaseRegIdx = 2; + OffsetIdx = 3; + IsPrePost = true; + break; + + case AArch64::LD1Twov1d_POST: + case AArch64::LD1Twov2s_POST: + case AArch64::LD1Twov4h_POST: + case AArch64::LD1Twov8b_POST: + case AArch64::LD1Twov2d_POST: + case AArch64::LD1Twov4s_POST: + case AArch64::LD1Twov8h_POST: + case AArch64::LD1Twov16b_POST: + case AArch64::LD1Threev1d_POST: + case AArch64::LD1Threev2s_POST: + case AArch64::LD1Threev4h_POST: + case AArch64::LD1Threev8b_POST: + case AArch64::LD1Threev2d_POST: + case AArch64::LD1Threev4s_POST: + case AArch64::LD1Threev8h_POST: + case AArch64::LD1Threev16b_POST: + case AArch64::LD1Fourv1d_POST: + case AArch64::LD1Fourv2s_POST: + case AArch64::LD1Fourv4h_POST: + case AArch64::LD1Fourv8b_POST: + case AArch64::LD1Fourv2d_POST: + case AArch64::LD1Fourv4s_POST: + case AArch64::LD1Fourv8h_POST: + case AArch64::LD1Fourv16b_POST: + case AArch64::LD2Twov2s_POST: + case AArch64::LD2Twov4s_POST: + case AArch64::LD2Twov8b_POST: + case AArch64::LD2Twov2d_POST: + case AArch64::LD2Twov4h_POST: + case AArch64::LD2Twov8h_POST: + case AArch64::LD2Twov16b_POST: + case AArch64::LD2Rv1d_POST: + case AArch64::LD2Rv2s_POST: + case AArch64::LD2Rv4s_POST: + case AArch64::LD2Rv8b_POST: + case AArch64::LD2Rv2d_POST: + case AArch64::LD2Rv4h_POST: + case AArch64::LD2Rv8h_POST: + case AArch64::LD2Rv16b_POST: + case AArch64::LD3Threev2s_POST: + case AArch64::LD3Threev4h_POST: + case AArch64::LD3Threev8b_POST: + case AArch64::LD3Threev2d_POST: + case AArch64::LD3Threev4s_POST: + case AArch64::LD3Threev8h_POST: + case AArch64::LD3Threev16b_POST: + case AArch64::LD3Rv1d_POST: + case AArch64::LD3Rv2s_POST: + case AArch64::LD3Rv4h_POST: + case AArch64::LD3Rv8b_POST: + case AArch64::LD3Rv2d_POST: + case AArch64::LD3Rv4s_POST: + case AArch64::LD3Rv8h_POST: + case AArch64::LD3Rv16b_POST: + case AArch64::LD4Fourv2s_POST: + case AArch64::LD4Fourv4h_POST: + case AArch64::LD4Fourv8b_POST: + case AArch64::LD4Fourv2d_POST: + case AArch64::LD4Fourv4s_POST: + case AArch64::LD4Fourv8h_POST: + case AArch64::LD4Fourv16b_POST: + case AArch64::LD4Rv1d_POST: + case AArch64::LD4Rv2s_POST: + case AArch64::LD4Rv4h_POST: + case AArch64::LD4Rv8b_POST: + case AArch64::LD4Rv2d_POST: + case AArch64::LD4Rv4s_POST: + case AArch64::LD4Rv8h_POST: + case AArch64::LD4Rv16b_POST: + DestRegIdx = -1; + BaseRegIdx = 2; + OffsetIdx = 3; + IsPrePost = true; + break; + + case AArch64::LDRBBroW: + case AArch64::LDRBBroX: + case AArch64::LDRBBui: + case AArch64::LDRBroW: + case AArch64::LDRBroX: + case AArch64::LDRBui: + case AArch64::LDRDl: + case AArch64::LDRDroW: + case AArch64::LDRDroX: + case AArch64::LDRDui: + case AArch64::LDRHHroW: + case AArch64::LDRHHroX: + case AArch64::LDRHHui: + case AArch64::LDRHroW: + case AArch64::LDRHroX: + case AArch64::LDRHui: + case AArch64::LDRQl: + case AArch64::LDRQroW: + case AArch64::LDRQroX: + case AArch64::LDRQui: + case AArch64::LDRSBWroW: + case AArch64::LDRSBWroX: + case AArch64::LDRSBWui: + case AArch64::LDRSBXroW: + case AArch64::LDRSBXroX: + case AArch64::LDRSBXui: + case AArch64::LDRSHWroW: + case AArch64::LDRSHWroX: + case AArch64::LDRSHWui: + case AArch64::LDRSHXroW: + case AArch64::LDRSHXroX: + case AArch64::LDRSHXui: + case AArch64::LDRSWl: + case AArch64::LDRSWroW: + case AArch64::LDRSWroX: + case AArch64::LDRSWui: + case AArch64::LDRSl: + case AArch64::LDRSroW: + case AArch64::LDRSroX: + case AArch64::LDRSui: + case AArch64::LDRWl: + case AArch64::LDRWroW: + case AArch64::LDRWroX: + case AArch64::LDRWui: + case AArch64::LDRXl: + case AArch64::LDRXroW: + case AArch64::LDRXroX: + case AArch64::LDRXui: + case AArch64::LDURBBi: + case AArch64::LDURBi: + case AArch64::LDURDi: + case AArch64::LDURHHi: + case AArch64::LDURHi: + case AArch64::LDURQi: + case AArch64::LDURSBWi: + case AArch64::LDURSBXi: + case AArch64::LDURSHWi: + case AArch64::LDURSHXi: + case AArch64::LDURSWi: + case AArch64::LDURSi: + case AArch64::LDURWi: + case AArch64::LDURXi: + DestRegIdx = 0; + BaseRegIdx = 1; + OffsetIdx = 2; + IsPrePost = false; + break; + + case AArch64::LDRBBpost: + case AArch64::LDRBBpre: + case AArch64::LDRBpost: + case AArch64::LDRBpre: + case AArch64::LDRDpost: + case AArch64::LDRDpre: + case AArch64::LDRHHpost: + case AArch64::LDRHHpre: + case AArch64::LDRHpost: + case AArch64::LDRHpre: + case AArch64::LDRQpost: + case AArch64::LDRQpre: + case AArch64::LDRSBWpost: + case AArch64::LDRSBWpre: + case AArch64::LDRSBXpost: + case AArch64::LDRSBXpre: + case AArch64::LDRSHWpost: + case AArch64::LDRSHWpre: + case AArch64::LDRSHXpost: + case AArch64::LDRSHXpre: + case AArch64::LDRSWpost: + case AArch64::LDRSWpre: + case AArch64::LDRSpost: + case AArch64::LDRSpre: + case AArch64::LDRWpost: + case AArch64::LDRWpre: + case AArch64::LDRXpost: + case AArch64::LDRXpre: + DestRegIdx = 1; + BaseRegIdx = 2; + OffsetIdx = 3; + IsPrePost = true; + break; + + case AArch64::LDNPDi: + case AArch64::LDNPQi: + case AArch64::LDNPSi: + case AArch64::LDPQi: + case AArch64::LDPDi: + case AArch64::LDPSi: + DestRegIdx = 0; + BaseRegIdx = 2; + OffsetIdx = 3; + IsPrePost = false; + IsPair = true; + break; + + case AArch64::LDPSWi: + case AArch64::LDPWi: + case AArch64::LDPXi: + DestRegIdx = 0; + BaseRegIdx = 2; + OffsetIdx = 3; + IsPrePost = false; + IsPair = true; + break; + + case AArch64::LDPQpost: + case AArch64::LDPQpre: + case AArch64::LDPDpost: + case AArch64::LDPDpre: + case AArch64::LDPSpost: + case AArch64::LDPSpre: + DestRegIdx = 1; + BaseRegIdx = 3; + OffsetIdx = 4; + IsPrePost = true; + IsPair = true; + break; + + case AArch64::LDPSWpost: + case AArch64::LDPSWpre: + case AArch64::LDPWpost: + case AArch64::LDPWpre: + case AArch64::LDPXpost: + case AArch64::LDPXpre: + DestRegIdx = 1; + BaseRegIdx = 3; + OffsetIdx = 4; + IsPrePost = true; + IsPair = true; + break; + + // LD(A)?(X)?R(B|H|W|X) + case AArch64::LDXRB: + case AArch64::LDXRH: + case AArch64::LDXRW: + case AArch64::LDXRX: + case AArch64::LDAXRB: + case AArch64::LDAXRH: + case AArch64::LDAXRW: + case AArch64::LDAXRX: + case AArch64::LDARB: + case AArch64::LDARH: + case AArch64::LDARW: + case AArch64::LDARX: + DestRegIdx = 0; + BaseRegIdx = 1; + OffsetIdx = -1; + IsPrePost = false; + break; + + // LD(A|X)P(W|X) + case AArch64::LDXPW: + case AArch64::LDXPX: + case AArch64::LDAXPW: + case AArch64::LDAXPX: + DestRegIdx = 0; + BaseRegIdx = 2; + OffsetIdx = -1; + IsPrePost = false; + IsPair = true; + break; + + } + + return MemInstInfo { DestRegIdx, BaseRegIdx, OffsetIdx, IsPrePost, IsPair }; +} + +static inline std::optional getStoreInfo(const MCInst &Inst) { + int DestRegIdx; + int BaseRegIdx; + int OffsetIdx; + bool IsPrePost = false; + bool IsPair = false; + switch (Inst.getOpcode()) { + default: + return std::nullopt; + + case AArch64::ST1i8: + case AArch64::ST1i16: + case AArch64::ST1i32: + case AArch64::ST1i64: + case AArch64::ST2i8: + case AArch64::ST2i16: + case AArch64::ST2i32: + case AArch64::ST2i64: + case AArch64::ST3i8: + case AArch64::ST3i16: + case AArch64::ST3i32: + case AArch64::ST3i64: + case AArch64::ST4i8: + case AArch64::ST4i16: + case AArch64::ST4i32: + case AArch64::ST4i64: + DestRegIdx = -1; + BaseRegIdx = 2; + OffsetIdx = -1; + IsPrePost = false; + break; + + case AArch64::ST1Onev1d: + case AArch64::ST1Onev2s: + case AArch64::ST1Onev4h: + case AArch64::ST1Onev8b: + case AArch64::ST1Onev2d: + case AArch64::ST1Onev4s: + case AArch64::ST1Onev8h: + case AArch64::ST1Onev16b: + DestRegIdx = 0; + BaseRegIdx = 1; + OffsetIdx = -1; + IsPrePost = false; + break; + + case AArch64::ST1Twov1d: + case AArch64::ST1Twov2s: + case AArch64::ST1Twov4h: + case AArch64::ST1Twov8b: + case AArch64::ST1Twov2d: + case AArch64::ST1Twov4s: + case AArch64::ST1Twov8h: + case AArch64::ST1Twov16b: + case AArch64::ST1Threev1d: + case AArch64::ST1Threev2s: + case AArch64::ST1Threev4h: + case AArch64::ST1Threev8b: + case AArch64::ST1Threev2d: + case AArch64::ST1Threev4s: + case AArch64::ST1Threev8h: + case AArch64::ST1Threev16b: + case AArch64::ST1Fourv1d: + case AArch64::ST1Fourv2s: + case AArch64::ST1Fourv4h: + case AArch64::ST1Fourv8b: + case AArch64::ST1Fourv2d: + case AArch64::ST1Fourv4s: + case AArch64::ST1Fourv8h: + case AArch64::ST1Fourv16b: + case AArch64::ST2Twov2s: + case AArch64::ST2Twov4s: + case AArch64::ST2Twov8b: + case AArch64::ST2Twov2d: + case AArch64::ST2Twov4h: + case AArch64::ST2Twov8h: + case AArch64::ST2Twov16b: + case AArch64::ST3Threev2s: + case AArch64::ST3Threev4h: + case AArch64::ST3Threev8b: + case AArch64::ST3Threev2d: + case AArch64::ST3Threev4s: + case AArch64::ST3Threev8h: + case AArch64::ST3Threev16b: + case AArch64::ST4Fourv2s: + case AArch64::ST4Fourv4h: + case AArch64::ST4Fourv8b: + case AArch64::ST4Fourv2d: + case AArch64::ST4Fourv4s: + case AArch64::ST4Fourv8h: + case AArch64::ST4Fourv16b: + DestRegIdx = -1; + BaseRegIdx = 1; + OffsetIdx = -1; + IsPrePost = false; + break; + + case AArch64::ST1i64_POST: + case AArch64::ST2i64_POST: + DestRegIdx = 1; + BaseRegIdx = 4; + OffsetIdx = 5; + IsPrePost = true; + break; + + // SIMDStSingleS + case AArch64::ST1i8_POST: + case AArch64::ST1i16_POST: + case AArch64::ST1i32_POST: + case AArch64::ST2i8_POST: + case AArch64::ST2i16_POST: + case AArch64::ST2i32_POST: + case AArch64::ST3i8_POST: + case AArch64::ST3i16_POST: + case AArch64::ST3i32_POST: + case AArch64::ST3i64_POST: + case AArch64::ST4i8_POST: + case AArch64::ST4i16_POST: + case AArch64::ST4i32_POST: + case AArch64::ST4i64_POST: + DestRegIdx = -1; + BaseRegIdx = 3; + OffsetIdx = 4; + IsPrePost = true; + break; + + case AArch64::ST1Onev1d_POST: + case AArch64::ST1Onev2s_POST: + case AArch64::ST1Onev4h_POST: + case AArch64::ST1Onev8b_POST: + case AArch64::ST1Onev2d_POST: + case AArch64::ST1Onev4s_POST: + case AArch64::ST1Onev8h_POST: + case AArch64::ST1Onev16b_POST: + DestRegIdx = 1; + BaseRegIdx = 2; + OffsetIdx = 3; + IsPrePost = true; + break; + + case AArch64::ST1Twov1d_POST: + case AArch64::ST1Twov2s_POST: + case AArch64::ST1Twov4h_POST: + case AArch64::ST1Twov8b_POST: + case AArch64::ST1Twov2d_POST: + case AArch64::ST1Twov4s_POST: + case AArch64::ST1Twov8h_POST: + case AArch64::ST1Twov16b_POST: + case AArch64::ST1Threev1d_POST: + case AArch64::ST1Threev2s_POST: + case AArch64::ST1Threev4h_POST: + case AArch64::ST1Threev8b_POST: + case AArch64::ST1Threev2d_POST: + case AArch64::ST1Threev4s_POST: + case AArch64::ST1Threev8h_POST: + case AArch64::ST1Threev16b_POST: + case AArch64::ST1Fourv1d_POST: + case AArch64::ST1Fourv2s_POST: + case AArch64::ST1Fourv4h_POST: + case AArch64::ST1Fourv8b_POST: + case AArch64::ST1Fourv2d_POST: + case AArch64::ST1Fourv4s_POST: + case AArch64::ST1Fourv8h_POST: + case AArch64::ST1Fourv16b_POST: + case AArch64::ST2Twov2s_POST: + case AArch64::ST2Twov4s_POST: + case AArch64::ST2Twov8b_POST: + case AArch64::ST2Twov2d_POST: + case AArch64::ST2Twov4h_POST: + case AArch64::ST2Twov8h_POST: + case AArch64::ST2Twov16b_POST: + case AArch64::ST3Threev2s_POST: + case AArch64::ST3Threev4h_POST: + case AArch64::ST3Threev8b_POST: + case AArch64::ST3Threev2d_POST: + case AArch64::ST3Threev4s_POST: + case AArch64::ST3Threev8h_POST: + case AArch64::ST3Threev16b_POST: + case AArch64::ST4Fourv2s_POST: + case AArch64::ST4Fourv4h_POST: + case AArch64::ST4Fourv8b_POST: + case AArch64::ST4Fourv2d_POST: + case AArch64::ST4Fourv4s_POST: + case AArch64::ST4Fourv8h_POST: + case AArch64::ST4Fourv16b_POST: + DestRegIdx = -1; + BaseRegIdx = 2; + OffsetIdx = 3; + IsPrePost = true; + break; + + case AArch64::STRBBroW: + case AArch64::STRBBroX: + case AArch64::STRBBui: + case AArch64::STRBroW: + case AArch64::STRBroX: + case AArch64::STRBui: + case AArch64::STRDroW: + case AArch64::STRDroX: + case AArch64::STRDui: + case AArch64::STRHHroW: + case AArch64::STRHHroX: + case AArch64::STRHHui: + case AArch64::STRHroW: + case AArch64::STRHroX: + case AArch64::STRHui: + case AArch64::STRQroW: + case AArch64::STRQroX: + case AArch64::STRQui: + case AArch64::STRSroW: + case AArch64::STRSroX: + case AArch64::STRSui: + case AArch64::STRWroW: + case AArch64::STRWroX: + case AArch64::STRWui: + case AArch64::STRXroW: + case AArch64::STRXroX: + case AArch64::STRXui: + case AArch64::STURBBi: + case AArch64::STURBi: + case AArch64::STURDi: + case AArch64::STURHHi: + case AArch64::STURHi: + case AArch64::STURQi: + case AArch64::STURSi: + case AArch64::STURWi: + case AArch64::STURXi: + DestRegIdx = 0; + BaseRegIdx = 1; + OffsetIdx = 2; + IsPrePost = false; + break; + + case AArch64::STRBBpost: + case AArch64::STRBBpre: + case AArch64::STRBpost: + case AArch64::STRBpre: + case AArch64::STRDpost: + case AArch64::STRDpre: + case AArch64::STRHHpost: + case AArch64::STRHHpre: + case AArch64::STRHpost: + case AArch64::STRHpre: + case AArch64::STRQpost: + case AArch64::STRQpre: + case AArch64::STRSpost: + case AArch64::STRSpre: + case AArch64::STRWpost: + case AArch64::STRWpre: + case AArch64::STRXpost: + case AArch64::STRXpre: + DestRegIdx = 1; + BaseRegIdx = 2; + OffsetIdx = 3; + IsPrePost = true; + break; + + case AArch64::STNPDi: + case AArch64::STNPQi: + case AArch64::STNPSi: + case AArch64::STPQi: + case AArch64::STPDi: + case AArch64::STPSi: + DestRegIdx = 0; + BaseRegIdx = 2; + OffsetIdx = 3; + IsPrePost = false; + IsPair = true; + break; + + case AArch64::STPWi: + case AArch64::STPXi: + DestRegIdx = 0; + BaseRegIdx = 2; + OffsetIdx = 3; + IsPrePost = false; + IsPair = true; + break; + + case AArch64::STPQpost: + case AArch64::STPQpre: + case AArch64::STPDpost: + case AArch64::STPDpre: + case AArch64::STPSpost: + case AArch64::STPSpre: + DestRegIdx = 1; + BaseRegIdx = 3; + OffsetIdx = 4; + IsPrePost = true; + IsPair = true; + break; + + case AArch64::STPWpost: + case AArch64::STPWpre: + case AArch64::STPXpost: + case AArch64::STPXpre: + DestRegIdx = 1; + BaseRegIdx = 3; + OffsetIdx = 4; + IsPrePost = true; + IsPair = true; + break; + + // op Dst, [Base] + case AArch64::STLRB: + case AArch64::STLRH: + case AArch64::STLRW: + case AArch64::STLRX: + DestRegIdx = 0; + BaseRegIdx = 1; + OffsetIdx = -1; + IsPrePost = false; + break; + + // Base = op Dst, [Base, (-4|-8)]! + case AArch64::STLRWpre: + case AArch64::STLRXpre: + DestRegIdx = 1; + BaseRegIdx = 2; + OffsetIdx = -1; // constant offset + IsPrePost = true; + break; + + case AArch64::STXRB: + case AArch64::STXRH: + case AArch64::STXRW: + case AArch64::STXRX: + case AArch64::STLXRB: + case AArch64::STLXRH: + case AArch64::STLXRW: + case AArch64::STLXRX: + DestRegIdx = 0; + BaseRegIdx = 2; + OffsetIdx = -1; + IsPrePost = false; + break; + + case AArch64::STXPW: + case AArch64::STXPX: + case AArch64::STLXPW: + case AArch64::STLXPX: + DestRegIdx = 1; // return the status result at 0 + BaseRegIdx = 3; + OffsetIdx = -1; + IsPrePost = false; + IsPair = true; + break; + } + + return MemInstInfo { DestRegIdx, BaseRegIdx, OffsetIdx, IsPrePost, IsPair }; +} + +struct LdStNInstrDesc { + unsigned Opcode; + const char *Mnemonic; + const char *Layout; + int ListOperand; + bool HasLane; + int NaturalOffset; +}; + +const LdStNInstrDesc *getLdStNInstrDesc(unsigned Opcode); + namespace AArch64CC { // The CondCodes constants map directly to the 4-bit encoding of the condition diff --git a/llvm/lib/Target/X86/CMakeLists.txt b/llvm/lib/Target/X86/CMakeLists.txt index 9553a8619feb5..ccdf45b65a621 100644 --- a/llvm/lib/Target/X86/CMakeLists.txt +++ b/llvm/lib/Target/X86/CMakeLists.txt @@ -63,6 +63,7 @@ set(sources X86InstrFoldTables.cpp X86InstrInfo.cpp X86CompressEVEX.cpp + X86LFIRewritePass.cpp X86LoadValueInjectionLoadHardening.cpp X86LoadValueInjectionRetHardening.cpp X86MCInstLower.cpp diff --git a/llvm/lib/Target/X86/MCTargetDesc/CMakeLists.txt b/llvm/lib/Target/X86/MCTargetDesc/CMakeLists.txt index f2e7d43fc17f6..dd03aa612374d 100644 --- a/llvm/lib/Target/X86/MCTargetDesc/CMakeLists.txt +++ b/llvm/lib/Target/X86/MCTargetDesc/CMakeLists.txt @@ -9,6 +9,7 @@ add_llvm_component_library(LLVMX86Desc X86MCTargetDesc.cpp X86MCAsmInfo.cpp X86MCCodeEmitter.cpp + X86MCLFIExpander.cpp X86MachObjectWriter.cpp X86MnemonicTables.cpp X86ELFObjectWriter.cpp diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.cpp new file mode 100644 index 0000000000000..43a19f2f192b9 --- /dev/null +++ b/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.cpp @@ -0,0 +1,1446 @@ +//===- X86MCLFIExpander.cpp - X86 LFI Expander --------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the X86MCLFIExpander class, the X86 specific +// subclass of MCLFIExpander. +// +// This file was written by the Native Client authors, modified for LFI. +// +//===----------------------------------------------------------------------===// +#include "X86MCLFIExpander.h" +#include "X86BaseInfo.h" + +#include "X86MCTargetDesc.h" +#include "llvm/MC/MCExpr.h" +#include "llvm/MC/MCInst.h" +#include "llvm/MC/MCInstrDesc.h" +#include "llvm/MC/MCInstrInfo.h" +#include "llvm/MC/MCLFIExpander.h" +#include "llvm/MC/MCObjectFileInfo.h" +#include "llvm/MC/MCRegisterInfo.h" +#include "llvm/MC/MCStreamer.h" +#include "llvm/MC/MCSubtargetInfo.h" +#include "llvm/Support/Alignment.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Debug.h" + +using namespace llvm; + +#define DEBUG_TYPE "lfi" + +static cl::opt X86LFIErrorReserved( + "x86-lfi-error-reserved", cl::Hidden, cl::init(false), + cl::desc("Produce errors for uses of LFI reserved registers")); + +static const int BundleSize = 32; + +static const MCRegister LFIBaseReg = X86::R14; +static const MCRegister LFIScratchReg = X86::R11; +static const MCRegister LFIBaseSeg = X86::GS; + +static bool hasFeature(const FeatureBitset Feature, + const MCSubtargetInfo &STI) { + return (STI.getFeatureBits() & Feature) == Feature; +} + +static bool hasSegue(const MCSubtargetInfo &STI) { + return !hasFeature(FeatureBitset({X86::FeatureLFINoSegue}), STI); +} + +static bool hasLFIStores(const MCSubtargetInfo &STI) { + return hasFeature(FeatureBitset({X86::FeatureLFIStores}), STI); +} + +static bool hasLFIJumps(const MCSubtargetInfo &STI) { + return hasFeature(FeatureBitset({X86::FeatureLFIJumps}), STI); +} + +static MCRegister getReg64(MCRegister Reg) { + switch (Reg) { + default: + return getX86SubSuperRegister(Reg, 64, false); + case X86::IP: + case X86::EIP: + case X86::RIP: + return X86::RIP; + } +} + +static MCRegister getReg32(MCRegister Reg) { + switch (Reg) { + default: + return getX86SubSuperRegister(Reg, 32, false); + case X86::IP: + case X86::EIP: + return X86::EIP; + case X86::RIP: + llvm_unreachable("Trying to demote %rip"); + } +} + +static bool isStringOperation(const MCInst &Inst) { + switch (Inst.getOpcode()) { + case X86::CMPSB: + case X86::CMPSW: + case X86::CMPSL: + case X86::CMPSQ: + case X86::MOVSB: + case X86::MOVSW: + case X86::MOVSL: + case X86::MOVSQ: + case X86::STOSB: + case X86::STOSW: + case X86::STOSL: + case X86::STOSQ: + return true; + } + return false; +} + +bool X86::X86MCLFIExpander::isValidScratchRegister(MCRegister Reg) const { + return false; +} + +void X86::X86MCLFIExpander::expandDirectCall(const MCInst &Inst, + MCStreamer &Out, + const MCSubtargetInfo &STI) { + Out.emitInstruction(Inst, STI); + Out.emitCodeAlignment(Align(BundleSize), &STI); +} + +void X86::X86MCLFIExpander::emitSandboxBranchReg(MCRegister Reg, + MCStreamer &Out, + const MCSubtargetInfo &STI) { + + MCInst AndInst; + AndInst.setOpcode(X86::AND32ri8); + + MCOperand Target32 = MCOperand::createReg(getReg32(Reg)); + AndInst.addOperand(Target32); + AndInst.addOperand(Target32); + AndInst.addOperand(MCOperand::createImm(-BundleSize)); + Out.emitInstruction(AndInst, STI); + + MCInst Add; + Add.setOpcode(X86::ADD64rr); + + MCOperand Target64 = MCOperand::createReg(getReg64(Reg)); + Add.addOperand(Target64); + Add.addOperand(Target64); + Add.addOperand(MCOperand::createReg(LFIBaseReg)); + Out.emitInstruction(Add, STI); +} + +void X86::X86MCLFIExpander::emitIndirectJumpReg(MCRegister Reg, MCStreamer &Out, + const MCSubtargetInfo &STI) { + Out.emitBundleLock(false); + emitSandboxBranchReg(Reg, Out, STI); + + MCInst Jmp; + Jmp.setOpcode(X86::JMP64r); + MCOperand Target = MCOperand::createReg(getReg64(Reg)); + Jmp.addOperand(Target); + + Out.emitInstruction(Jmp, STI); + Out.emitBundleUnlock(); +} + +void X86::X86MCLFIExpander::emitIndirectCallReg(MCRegister Reg, MCStreamer &Out, + const MCSubtargetInfo &STI) { + MCOperand Target = MCOperand::createReg(getReg64(Reg)); + + Out.emitBundleLock(false); + emitSandboxBranchReg(Reg, Out, STI); + MCInst Call; + Call.setOpcode(X86::CALL64r); + Call.addOperand(Target); + Out.emitInstruction(Call, STI); + Out.emitBundleUnlock(); + Out.emitCodeAlignment(Align(BundleSize), &STI); +} + +void X86::X86MCLFIExpander::expandIndirectBranch(const MCInst &Inst, + MCStreamer &Out, + const MCSubtargetInfo &STI) { + MCRegister Target; + if (mayLoad(Inst)) { + // indirect jmp/call through memory + MCInst Mov; + Mov.setOpcode(X86::MOV64rm); + + Target = X86::R11; + + Mov.addOperand(MCOperand::createReg(getReg64(Target))); + Mov.addOperand(Inst.getOperand(0)); + Mov.addOperand(Inst.getOperand(1)); + Mov.addOperand(Inst.getOperand(2)); + Mov.addOperand(Inst.getOperand(3)); + Mov.addOperand(Inst.getOperand(4)); + doExpandInst(Mov, Out, STI, false); + } else { + Target = Inst.getOperand(0).getReg(); + } + + if (isCall(Inst)) + emitIndirectCallReg(Target, Out, STI); + else + emitIndirectJumpReg(Target, Out, STI); +} + +static bool isValidReturnRegister(const MCRegister &Reg) { + return Reg == X86::EAX || Reg == X86::RAX || Reg == X86::AL || + Reg == X86::AX || Reg == X86::XMM0; +} + +static bool isHighReg(MCRegister Reg) { + return Reg == X86::AH || Reg == X86::BH || Reg == X86::CH || Reg == X86::DH; +} + +void X86::X86MCLFIExpander::expandReturn(const MCInst &Inst, MCStreamer &Out, + const MCSubtargetInfo &STI) { + MCRegister ScratchReg = X86::R11; + + MCInst Pop; + Pop.setOpcode(X86::POP64r); + Pop.addOperand(MCOperand::createReg(ScratchReg)); + Out.emitInstruction(Pop, STI); + + if (Inst.getNumOperands() > 0) { + if (Inst.getOpcode() == X86::RETI32 || Inst.getOpcode() == X86::RETI64) { + MCOperand StackPointer = MCOperand::createReg(X86::RSP); + + MCInst Add; + Add.setOpcode(X86::ADD64ri32); + Add.addOperand(StackPointer); + Add.addOperand(StackPointer); + Add.addOperand(Inst.getOperand(0)); + + doExpandInst(Add, Out, STI, false); + } else if (Inst.getOpcode() == X86::RET32 || + Inst.getOpcode() == X86::RET64) { + LLVM_DEBUG(Inst.dump()); + assert(Inst.getOperand(0).isReg()); + assert(isValidReturnRegister(Inst.getOperand(0).getReg())); + } else { + LLVM_DEBUG(Inst.dump()); + Error(Inst, "Unexpected return instruction."); + } + } + + emitIndirectJumpReg(ScratchReg, Out, STI); +} + +// Expands any operations that load to or store from memory, but do +// not explicitly modify the stack or base pointer. +void X86::X86MCLFIExpander::expandLoadStore(const MCInst &Inst, MCStreamer &Out, + const MCSubtargetInfo &STI, + bool EmitPrefixes) { + assert(!explicitlyModifiesRegister(Inst, X86::RSP)); + + // Optimize if we are doing a mov into a register + bool ElideScratchReg = false; + switch (Inst.getOpcode()) { + case X86::MOV64rm: + case X86::MOV32rm: + case X86::MOV16rm: + case X86::MOV8rm: + ElideScratchReg = true; + break; + default: + break; + } + + MCInst SandboxedInst(Inst); + + MCRegister ScratchReg; + if (ElideScratchReg) + ScratchReg = Inst.getOperand(0).getReg(); + else if (numScratchRegs() > 0) + ScratchReg = getScratchReg(0); + else + ScratchReg = X86::R11D; + + // Determine if the instruction requires sandboxing + bool InstNeedsSandboxing = emitSandboxMemOps(SandboxedInst, + ScratchReg, + Out, + STI, + /*EmitInstructions=*/false); + + // We may want to load/store to a high byte register, which is not possible + // in combination with using Base for sandboxing. + // Instead, rotate the target register so that the load/store operates on the + // low byte instead. + MCRegister RotateRegister = X86::NoRegister; + if (InstNeedsSandboxing && + (SandboxedInst.getOpcode() == X86::MOV8rm_NOREX || + SandboxedInst.getOpcode() == X86::MOV8rm) && + isHighReg(SandboxedInst.getOperand(0).getReg())) { + RotateRegister = SandboxedInst.getOperand(0).getReg(); + SandboxedInst.setOpcode(X86::MOV8rm); + SandboxedInst.getOperand(0).setReg( + getX86SubSuperRegister(RotateRegister, 8, /*High=*/false)); + } else if (InstNeedsSandboxing && + (SandboxedInst.getOpcode() == X86::MOV8mr_NOREX || + SandboxedInst.getOpcode() == X86::MOV8mr) && + isHighReg(SandboxedInst.getOperand(5).getReg())) { + RotateRegister = SandboxedInst.getOperand(5).getReg(); + SandboxedInst.setOpcode(X86::MOV8mr); + SandboxedInst.getOperand(5).setReg( + getX86SubSuperRegister(RotateRegister, 8, /*High=*/false)); + } + + if (RotateRegister != X86::NoRegister) { + MCInst RotateHtoLInst; + RotateHtoLInst.setOpcode(X86::ROR64ri); + RotateHtoLInst.addOperand(MCOperand::createReg(getReg64(RotateRegister))); + RotateHtoLInst.addOperand(MCOperand::createReg(getReg64(RotateRegister))); + RotateHtoLInst.addOperand(MCOperand::createImm(8)); + Out.emitInstruction(RotateHtoLInst, STI); + } + + bool BundleLock = emitSandboxMemOps(SandboxedInst, + ScratchReg, + Out, + STI, + /*EmitInstructions=*/true); + emitInstruction(SandboxedInst, Out, STI, EmitPrefixes); + if (BundleLock) + Out.emitBundleUnlock(); + + if (RotateRegister != X86::NoRegister) { + // Rotate the register back. + MCInst RotateLtoHInst; + RotateLtoHInst.setOpcode(X86::ROL64ri); + RotateLtoHInst.addOperand(MCOperand::createReg(getReg64(RotateRegister))); + RotateLtoHInst.addOperand(MCOperand::createReg(getReg64(RotateRegister))); + RotateLtoHInst.addOperand(MCOperand::createImm(8)); + Out.emitInstruction(RotateLtoHInst, STI); + } +} + +// Emits movl Reg32, Reg32 +// Used as a helper in various places. +static void clearHighBits(const MCOperand &Reg, MCStreamer &Out, + const MCSubtargetInfo &STI) { + MCInst Mov; + Mov.setOpcode(X86::MOV32rr); + MCOperand Op = MCOperand::createReg(getReg32(Reg.getReg())); + + Mov.addOperand(Op); + Mov.addOperand(Op); + Out.emitInstruction(Mov, STI); +} + +static void fixupStringOpReg(const MCOperand &Op, MCStreamer &Out, + const MCSubtargetInfo &STI) { + clearHighBits(Op, Out, STI); + + MCInst Lea; + Lea.setOpcode(X86::LEA64r); + Lea.addOperand(MCOperand::createReg(getReg64(Op.getReg()))); + Lea.addOperand(MCOperand::createReg(LFIBaseReg)); + Lea.addOperand(MCOperand::createImm(1)); + Lea.addOperand(MCOperand::createReg(getReg64(Op.getReg()))); + Lea.addOperand(MCOperand::createImm(0)); + Lea.addOperand(MCOperand::createReg(0)); + Out.emitInstruction(Lea, STI); +} + +void X86::X86MCLFIExpander::expandStringOperation(const MCInst &Inst, + MCStreamer &Out, + const MCSubtargetInfo &STI, + bool EmitPrefixes) { + Out.emitBundleLock(false); + switch (Inst.getOpcode()) { + case X86::CMPSB: + case X86::CMPSW: + case X86::CMPSL: + case X86::CMPSQ: + case X86::MOVSB: + case X86::MOVSW: + case X86::MOVSL: + case X86::MOVSQ: + if (!hasLFIJumps(STI) && !hasLFIStores(STI)) + fixupStringOpReg(Inst.getOperand(1), Out, STI); + if (!hasLFIJumps(STI)) + fixupStringOpReg(Inst.getOperand(0), Out, STI); + break; + case X86::STOSB: + case X86::STOSW: + case X86::STOSL: + case X86::STOSQ: + if (!hasLFIJumps(STI)) + fixupStringOpReg(Inst.getOperand(0), Out, STI); + break; + } + emitInstruction(Inst, Out, STI, EmitPrefixes); + Out.emitBundleUnlock(); +} + +static unsigned demoteOpcode(unsigned Reg); + +static bool isAbsoluteReg(MCRegister Reg) { + Reg = getReg64(Reg); // Normalize to 64 bits + return (Reg == LFIBaseReg || Reg == X86::RSP || Reg == X86::RIP); +} + +static void demoteInst(MCInst &Inst, const MCInstrInfo &InstInfo) { + unsigned NewOpc = demoteOpcode(Inst.getOpcode()); + Inst.setOpcode(NewOpc); + // demote all general purpose 64 bit registers to 32 bits + const llvm::ArrayRef OpInfo = + InstInfo.get(Inst.getOpcode()).operands(); + for (int i = 0, e = Inst.getNumOperands(); i < e; ++i) { + if (OpInfo[i].OperandType == MCOI::OPERAND_REGISTER) { + assert(Inst.getOperand(i).isReg()); + MCRegister Reg = Inst.getOperand(i).getReg(); + if (getReg64(Reg) == Reg) { + Inst.getOperand(i).setReg(getReg32(Reg)); + } + } + } +} + +void X86::X86MCLFIExpander::prepareSandboxMemOp(MCInst &Inst, int MemIdx, + MCRegister ScratchReg, + MCStreamer &Out, + const MCSubtargetInfo &STI) { + MCOperand &Base = Inst.getOperand(MemIdx); + MCOperand &Index = Inst.getOperand(MemIdx + 2); + MCOperand &Segment = Inst.getOperand(MemIdx + 4); + + if (Segment.getReg() == X86::FS) { + MCInst Push; + Push.setOpcode(X86::PUSH64r); + Push.addOperand(MCOperand::createReg(X86::RAX)); + Out.emitInstruction(Push, STI); + MCInst Read; + Read.setOpcode(X86::MOV64rm); + Read.addOperand(MCOperand::createReg(X86::RAX)); + Read.addOperand(Inst.getOperand(MemIdx)); + Read.addOperand(Inst.getOperand(MemIdx + 1)); + Read.addOperand(Inst.getOperand(MemIdx + 2)); + Read.addOperand(Inst.getOperand(MemIdx + 3)); + Read.addOperand(Inst.getOperand(MemIdx + 4)); + expandTLSRead(Read, Out, STI); + MCInst Mov; + Mov.setOpcode(X86::MOV64rr); + Mov.addOperand(MCOperand::createReg(ScratchReg)); + Mov.addOperand(MCOperand::createReg(X86::RAX)); + Out.emitInstruction(Mov, STI); + MCInst Pop; + Pop.setOpcode(X86::POP64r); + Pop.addOperand(MCOperand::createReg(X86::RAX)); + Out.emitInstruction(Pop, STI); + if (Base.getReg() == 0) + Base.setReg(ScratchReg); + else if (Index.getReg() == 0) + Index.setReg(ScratchReg); + else + return Out.getContext().reportError( + Inst.getLoc(), "invalid use of %fs segment register"); + Segment.setReg(0); + } +} + +// Emits the sandboxing operations necessary, and modifies the memory +// operand specified by MemIdx. +// Used as a helper function for emitSandboxMemOps. +void X86::X86MCLFIExpander::emitSandboxMemOp(MCInst &Inst, int MemIdx, + MCRegister ScratchReg, + MCStreamer &Out, + const MCSubtargetInfo &STI) { + if (hasLFIJumps(STI)) + return; + if (hasLFIStores(STI) && !mayStore(Inst)) + return; + + MCOperand &Base = Inst.getOperand(MemIdx); + MCOperand &Scale = Inst.getOperand(MemIdx + 1); + MCOperand &Index = Inst.getOperand(MemIdx + 2); + MCOperand &Offset = Inst.getOperand(MemIdx + 3); + MCOperand &Segment = Inst.getOperand(MemIdx + 4); + + // In the cases below, we want to promote any registers in the + // memory operand to 64 bits. + if (isAbsoluteReg(Base.getReg()) && Index.getReg() == 0) { + Base.setReg(getReg64(Base.getReg())); + return; + } + + if (Base.getReg() == 0 && isAbsoluteReg(Index.getReg()) && + Scale.getImm() == 1) { + Base.setReg(getReg64(Index.getReg())); + Index.setReg(0); + return; + } + + if (Index.getReg() == 0 && Base.getReg() == 0) { + Base.setReg(LFIBaseReg); + return; + } + + if (hasSegue(STI) && Segment.getReg() == 0) { + Segment.setReg(LFIBaseSeg); + Base.setReg(getReg32(Base.getReg())); + Index.setReg(getReg32(Index.getReg())); + return; + } + + MCRegister ScratchReg32 = 0; + if (ScratchReg != 0) { + ScratchReg32 = getReg32(ScratchReg); + } else { + Error(Inst, + "Not enough scratch registers when emitting sandboxed memory operation." + ); + } + + if (isAbsoluteReg(Base.getReg()) && !isAbsoluteReg(Index.getReg()) && + Offset.isImm() && Offset.getImm() == 0) { + // Validation requires that for memory access, a 64-bit register is used, + // with its upper 32 bits clobbered by a 32-bit move just before. + // Move Index to 32 bit ScratchReg, and use ScratchReg32 instead of Index + // memory access. + MCInst MovIdxToScratchReg; + MovIdxToScratchReg.setOpcode(X86::MOV32rr); + MovIdxToScratchReg.addOperand(MCOperand::createReg(ScratchReg32)); + MovIdxToScratchReg.addOperand( + MCOperand::createReg(getReg32(Index.getReg()))); + Out.emitInstruction(MovIdxToScratchReg, STI); + + Base.setReg(getReg64(Base.getReg())); + Index.setReg(getReg64(ScratchReg32)); + return; + } + + if (Index.getReg() == 0 && Base.getReg() != 0 && Offset.isImm() && + Offset.getImm() == 0) { + // Validation requires that for memory access, a 64-bit register is used, + // with its upper 32 bits clobbered by a 32-bit move just before. + // Move Base to 32 bit ScratchReg, and use ScratchReg32 instead of Base for + // memory access. + MCInst MovBaseToScratchReg; + MovBaseToScratchReg.setOpcode(X86::MOV32rr); + MovBaseToScratchReg.addOperand(MCOperand::createReg(ScratchReg32)); + MovBaseToScratchReg.addOperand( + MCOperand::createReg(getReg32(Base.getReg()))); + Out.emitInstruction(MovBaseToScratchReg, STI); + + Index.setReg(getReg64(ScratchReg32)); + Base.setReg(LFIBaseReg); + return; + } + + MCRegister ScratchReg64 = getReg64(ScratchReg32); + MCRegister BaseReg64 = getReg64(Base.getReg()); + MCRegister IndexReg64 = getReg64(Index.getReg()); + + MCInst Lea; + Lea.setOpcode(X86::LEA64_32r); + Lea.addOperand(MCOperand::createReg(ScratchReg32)); + Lea.addOperand(MCOperand::createReg(BaseReg64)); + Lea.addOperand(Scale); + Lea.addOperand(MCOperand::createReg(IndexReg64)); + Lea.addOperand(Offset); + Lea.addOperand(Segment); + + // Specical case if there is no base or scale + if (Base.getReg() == 0 && Scale.getImm() == 1) { + Lea.getOperand(1).setReg(IndexReg64); // Base + Lea.getOperand(3).setReg(0); // Index + } + + Out.emitInstruction(Lea, STI); + + Base.setReg(LFIBaseReg); + Scale.setImm(1); + Index.setReg(ScratchReg64); + if (Offset.isImm()) { + Offset.setImm(0); + } else { + Inst.erase(Inst.begin() + MemIdx + 3); + Inst.insert(Inst.begin() + MemIdx + 3, MCOperand::createImm(0)); + } +} + +// Returns true if sandboxing the memory operand specified at Idx of +// Inst will emit any auxillary instructions. +// Used in emitSandboxMemOps as a helper. +static bool willEmitSandboxInsts(const MCInst &Inst, int Idx) { + const MCOperand &Base = Inst.getOperand(Idx); + const MCOperand &Scale = Inst.getOperand(Idx + 1); + const MCOperand &Index = Inst.getOperand(Idx + 2); + + if (isAbsoluteReg(Base.getReg()) && Index.getReg() == 0) { + return false; + } else if (Base.getReg() == 0 && isAbsoluteReg(Index.getReg()) && + Scale.getImm() == 1) { + return false; + } + + return true; +} + +// Emits the instructions that are used to sandbox the memory operands. +// Modifies memory operands of Inst in place, but does NOT EMIT Inst. +// If any instructions are emitted, it will precede them with a .bundle_lock +// Returns true if any instructions were emitted, otherwise false. +// Note that this method can modify Inst, but still return false if no +// auxiliary sandboxing instructions were emitted. +// If EmitInstructions is set to false, this will not emit anything and return +// whether it would emit any auxiliary instructions. +bool X86::X86MCLFIExpander::emitSandboxMemOps(MCInst &Inst, + MCRegister ScratchReg, + MCStreamer &Out, + const MCSubtargetInfo &STI, + bool EmitInstructions) { + + const llvm::ArrayRef OpInfo = + InstInfo->get(Inst.getOpcode()).operands(); + + bool anyInstsEmitted = false; + + for (int i = 0, e = Inst.getNumOperands(); i < e; ++i) { + if (OpInfo[i].OperandType == MCOI::OPERAND_MEMORY) { + prepareSandboxMemOp(Inst, i, ScratchReg, Out, STI); + + if (!anyInstsEmitted && willEmitSandboxInsts(Inst, i)) { + if (!EmitInstructions) + return true; + + if (!hasSegue(STI)) { + Out.emitBundleLock(false); + anyInstsEmitted = true; + } + } + emitSandboxMemOp(Inst, i, ScratchReg, Out, STI); + i += 4; + } + } + + return anyInstsEmitted; +} + +static void emitStackFixup(MCRegister StackReg, MCStreamer &Out, + const MCSubtargetInfo &STI) { + MCInst Lea; + Lea.setOpcode(X86::LEA64r); + Lea.addOperand(MCOperand::createReg(StackReg)); + Lea.addOperand(MCOperand::createReg(StackReg)); + Lea.addOperand(MCOperand::createImm(1)); + Lea.addOperand(MCOperand::createReg(LFIBaseReg)); + Lea.addOperand(MCOperand::createImm(0)); + Lea.addOperand(MCOperand::createReg(0)); + Out.emitInstruction(Lea, STI); +} + +void X86::X86MCLFIExpander::expandExplicitStackManipulation( + MCRegister StackReg, const MCInst &Inst, MCStreamer &Out, + const MCSubtargetInfo &STI, bool EmitPrefixes) { + if (hasLFIJumps(STI)) + return emitInstruction(Inst, Out, STI, EmitPrefixes); + + if (Inst.getOpcode() == X86::POP64r) { + // Transform + // pop %rsp + // into + // pop %r11 + // .bundle_lock + // movl %r11d, %esp + // add %r14, %rsp + // .bundle_unlock + // where %r11 is the scratch register, and %rsp the stack register. + + MCInst PopR11Inst; + PopR11Inst.setOpcode(X86::POP64r); + PopR11Inst.addOperand(MCOperand::createReg(X86::R11)); + Out.emitInstruction(PopR11Inst, STI); + + Out.emitBundleLock(false); + + MCInst MovR11ToESP; + MovR11ToESP.setOpcode(X86::MOV32rr); + MovR11ToESP.addOperand(MCOperand::createReg(getReg32(StackReg))); + MovR11ToESP.addOperand(MCOperand::createReg(X86::R11D)); + Out.emitInstruction(MovR11ToESP, STI); + + MCInst AddBaseInst; + AddBaseInst.setOpcode(X86::ADD64rr); + AddBaseInst.addOperand(MCOperand::createReg(StackReg)); + AddBaseInst.addOperand(MCOperand::createReg(StackReg)); + AddBaseInst.addOperand(MCOperand::createReg(LFIBaseReg)); + Out.emitInstruction(AddBaseInst, STI); + + return Out.emitBundleUnlock(); + } + + MCInst SandboxedInst(Inst); + demoteInst(SandboxedInst, *InstInfo); + + MCRegister ScratchReg; + if (numScratchRegs() > 0) + ScratchReg = getScratchReg(0); + else + ScratchReg = 0; + + bool MemSandboxed = emitSandboxMemOps(SandboxedInst, ScratchReg, Out, STI, + /*emitInstructions=*/true); + + Out.emitBundleLock(false); // for stack fixup + + emitInstruction(SandboxedInst, Out, STI, EmitPrefixes); + if (MemSandboxed) + Out.emitBundleUnlock(); // for memory reference + emitStackFixup(StackReg, Out, STI); + + Out.emitBundleUnlock(); // for stack fixup +} + +// Returns true if Inst is an X86 prefix +static bool isPrefix(const MCInst &Inst) { + switch (Inst.getOpcode()) { + case X86::LOCK_PREFIX: + case X86::REP_PREFIX: + case X86::REPNE_PREFIX: + case X86::REX64_PREFIX: + return true; + default: + return false; + } +} + +static bool isDirectCall(const MCInst &Inst) { + switch (Inst.getOpcode()) { + case X86::CALLpcrel32: + case X86::CALL64pcrel32: + return true; + default: + return false; + } +} + +// Emit prefixes + instruction if EmitPrefixes argument is true. +// Otherwise, emit the bare instruction. +void X86::X86MCLFIExpander::emitInstruction(const MCInst &Inst, MCStreamer &Out, + const MCSubtargetInfo &STI, + bool EmitPrefixes) { + if (EmitPrefixes) { + for (const MCInst &Prefix : Prefixes) + Out.emitInstruction(Prefix, STI); + Prefixes.clear(); + } + Out.emitInstruction(Inst, STI); +} + +// returns the stack register that is used in xchg instruction +static MCRegister xchgStackReg(const MCInst &Inst) { + MCRegister Reg1 = 0, Reg2 = 0; + switch (Inst.getOpcode()) { + case X86::XCHG64ar: + case X86::XCHG64rm: + Reg1 = Inst.getOperand(0).getReg(); + break; + case X86::XCHG64rr: + Reg1 = Inst.getOperand(0).getReg(); + Reg2 = Inst.getOperand(2).getReg(); + break; + default: + return 0; + } + if (Reg1 == X86::RSP) + return Reg1; + + if (Reg2 == X86::RSP) + return Reg2; + + return 0; +} + +static bool isSyscall(const MCInst &Inst) { + return Inst.getOpcode() == X86::SYSCALL; +} + +static bool isTLSRead(const MCInst &Inst) { + return Inst.getOpcode() == X86::MOV64rm && + Inst.getOperand(1).getReg() == X86::NoRegister && + Inst.getOperand(2).getImm() == 1 && + Inst.getOperand(3).getReg() == X86::NoRegister && + Inst.getOperand(4).getImm() == 0 && + Inst.getOperand(5).getReg() == X86::FS; +} + +void X86::X86MCLFIExpander::emitLFICall(LFICallType CallType, MCStreamer &Out, + const MCSubtargetInfo &STI) { + Out.emitBundleLock(false); + + MCSymbol *Symbol = Ctx.createTempSymbol(); + + MCInst Lea; + Lea.setOpcode(X86::LEA64r); + Lea.addOperand(MCOperand::createReg(X86::R11)); + Lea.addOperand(MCOperand::createReg(X86::RIP)); + Lea.addOperand(MCOperand::createImm(1)); + Lea.addOperand(MCOperand::createReg(X86::NoRegister)); + Lea.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create(Symbol, Ctx))); + Lea.addOperand(MCOperand::createReg(0)); + Out.emitInstruction(Lea, STI); + + unsigned Offset; + switch (CallType) { + case LFISyscall: + Offset = 0; + break; + case LFITLSRead: + Offset = 8; + break; + case LFITLSWrite: + Offset = 16; + break; + } + + MCInst Jmp; + Jmp.setOpcode(X86::JMP64m); + Jmp.addOperand(MCOperand::createReg(LFIBaseReg)); + Jmp.addOperand(MCOperand::createImm(1)); + Jmp.addOperand(MCOperand::createReg(X86::NoRegister)); + Jmp.addOperand(MCOperand::createImm(Offset)); + Jmp.addOperand(MCOperand::createReg(0)); + Out.emitInstruction(Jmp, STI); + + Out.emitLabel(Symbol); + + Out.emitBundleUnlock(); +} + +void X86::X86MCLFIExpander::expandSyscall(const MCInst &Inst, MCStreamer &Out, + const MCSubtargetInfo &STI) { + emitLFICall(LFISyscall, Out, STI); +} + +void X86::X86MCLFIExpander::expandTLSRead(const MCInst &Inst, MCStreamer &Out, + const MCSubtargetInfo &STI) { + if (Inst.getOperand(0).getReg() != X86::RAX) { + MCInst Mov; + Mov.setOpcode(X86::MOV64rr); + Mov.addOperand(Inst.getOperand(0)); + Mov.addOperand(MCOperand::createReg(X86::RAX)); + Out.emitInstruction(Mov, STI); + } + + emitLFICall(LFITLSRead, Out, STI); + + if (Inst.getOperand(0).getReg() != X86::RAX) { + MCInst Xchg; + Xchg.setOpcode(X86::XCHG64rr); + Xchg.addOperand(Inst.getOperand(0)); + Xchg.addOperand(MCOperand::createReg(X86::RAX)); + Xchg.addOperand(Inst.getOperand(0)); + Xchg.addOperand(MCOperand::createReg(X86::RAX)); + Out.emitInstruction(Xchg, STI); + } +} + +static MCInst replaceReg(const MCInst &Inst, MCRegister Dest, MCRegister Src) { + MCInst New; + New.setOpcode(Inst.getOpcode()); + New.setLoc(Inst.getLoc()); + for (unsigned I = 0; I < Inst.getNumOperands(); ++I) { + const MCOperand &Op = Inst.getOperand(I); + if (Op.isReg() && Op.getReg() == Src) { + New.addOperand(MCOperand::createReg(Dest)); + } else { + New.addOperand(Op); + } + } + return New; +} + +void X86::X86MCLFIExpander::doExpandInst(const MCInst &Inst, MCStreamer &Out, + const MCSubtargetInfo &STI, + bool EmitPrefixes) { + if (isPrefix(Inst)) { + return Prefixes.push_back(Inst); + } + if (explicitlyModifiesRegister(Inst, LFIBaseReg)) { + if (X86LFIErrorReserved) + return Out.getContext().reportError( + Inst.getLoc(), "illegal modification of reserved LFI register"); + Out.getContext().reportWarning( + Inst.getLoc(), "deleting modification of reserved LFI register"); + MCInst New = replaceReg(Inst, LFIScratchReg, LFIBaseReg); + return doExpandInst(New, Out, STI, EmitPrefixes); + } else if (isSyscall(Inst)) { + expandSyscall(Inst, Out, STI); + } else if (isTLSRead(Inst)) { + expandTLSRead(Inst, Out, STI); + } else if (isDirectCall(Inst)) { + expandDirectCall(Inst, Out, STI); + } else if (isIndirectBranch(Inst) || isCall(Inst)) { + expandIndirectBranch(Inst, Out, STI); + } else if (isReturn(Inst)) { + expandReturn(Inst, Out, STI); + } else if (isStringOperation(Inst)) { + expandStringOperation(Inst, Out, STI, EmitPrefixes); + } else if (explicitlyModifiesRegister(Inst, X86::RSP)) { + expandExplicitStackManipulation(X86::RSP, Inst, Out, STI, EmitPrefixes); + } else if (MCRegister StackReg = xchgStackReg(Inst)) { + // The previous case doesn't catch xchg so special case it. + expandExplicitStackManipulation(X86::RSP, Inst, Out, STI, EmitPrefixes); + } else { + for (int i = 0, e = Inst.getNumOperands(); i < e; ++i) { + if (Inst.getOperand(i).isReg()) { + MCRegister Reg = Inst.getOperand(i).getReg(); + if (Reg == X86::GS) + return Out.getContext().reportError( + Inst.getLoc(), "invalid use of %gs segment register"); + } + } + + expandLoadStore(Inst, Out, STI, EmitPrefixes); + } +} + +bool X86::X86MCLFIExpander::expandInst(const MCInst &Inst, MCStreamer &Out, + const MCSubtargetInfo &STI) { + if (Guard) + return false; + Guard = true; + + doExpandInst(Inst, Out, STI, true); + invalidateScratchRegs(Inst); + + Guard = false; + return true; +} + +static unsigned demoteOpcode(unsigned Opcode) { + switch (Opcode) { + case X86::ADC64rr: + return X86::ADC32rr; + case X86::ADC64ri8: + return X86::ADC32ri8; + case X86::ADC64ri32: + return X86::ADC32ri; + case X86::ADC64rm: + return X86::ADC32rm; + case X86::ADCX64rr: + return X86::ADCX32rr; + case X86::ADCX64rm: + return X86::ADCX32rm; + case X86::ADD64rr: + return X86::ADD32rr; + case X86::ADD64ri8: + return X86::ADD32ri8; + case X86::ADD64ri32: + return X86::ADD32ri; + case X86::ADD64rm: + return X86::ADD32rm; + case X86::ADOX64rr: + return X86::ADOX32rr; + case X86::ADOX64rm: + return X86::ADOX32rm; + case X86::ANDN64rr: + return X86::ANDN32rr; + case X86::ANDN64rm: + return X86::ANDN32rm; + case X86::AND64rr: + return X86::AND32rr; + case X86::AND64ri8: + return X86::AND32ri8; + case X86::AND64ri32: + return X86::AND32ri; + case X86::AND64rm: + return X86::AND32rm; + case X86::BEXTRI64ri: + return X86::BEXTRI32ri; + case X86::BEXTRI64mi: + return X86::BEXTRI32mi; + case X86::BEXTR64rr: + return X86::BEXTR32rr; + case X86::BEXTR64rm: + return X86::BEXTR32rm; + case X86::BLCFILL64rr: + return X86::BLCFILL32rr; + case X86::BLCFILL64rm: + return X86::BLCFILL32rm; + case X86::BLCI64rr: + return X86::BLCI32rr; + case X86::BLCI64rm: + return X86::BLCI32rm; + case X86::BLCIC64rr: + return X86::BLCIC32rr; + case X86::BLCIC64rm: + return X86::BLCIC32rm; + case X86::BLCMSK64rr: + return X86::BLCMSK32rr; + case X86::BLCMSK64rm: + return X86::BLCMSK32rm; + case X86::BLCS64rr: + return X86::BLCS32rr; + case X86::BLCS64rm: + return X86::BLCS32rm; + case X86::BLSFILL64rr: + return X86::BLSFILL32rr; + case X86::BLSFILL64rm: + return X86::BLSFILL32rm; + case X86::BLSIC64rr: + return X86::BLSIC32rr; + case X86::BLSIC64rm: + return X86::BLSIC32rm; + case X86::BLSI64rr: + return X86::BLSI32rr; + case X86::BLSI64rm: + return X86::BLSI32rm; + case X86::BLSMSK64rr: + return X86::BLSMSK32rr; + case X86::BLSMSK64rm: + return X86::BLSMSK32rm; + case X86::BLSR64rr: + return X86::BLSR32rr; + case X86::BLSR64rm: + return X86::BLSR32rm; + case X86::BSF64rr: + return X86::BSF32rr; + case X86::BSF64rm: + return X86::BSF32rm; + case X86::BSR64rr: + return X86::BSR32rr; + case X86::BSR64rm: + return X86::BSR32rm; + case X86::BSWAP64r: + return X86::BSWAP32r; + case X86::BTC64rr: + return X86::BTC32rr; + case X86::BTC64ri8: + return X86::BTC32ri8; + case X86::BT64rr: + return X86::BT32rr; + case X86::BT64ri8: + return X86::BT32ri8; + case X86::BTR64rr: + return X86::BTR32rr; + case X86::BTR64ri8: + return X86::BTR32ri8; + case X86::BTS64rr: + return X86::BTS32rr; + case X86::BTS64ri8: + return X86::BTS32ri8; + case X86::BZHI64rr: + return X86::BZHI32rr; + case X86::BZHI64rm: + return X86::BZHI32rm; + case X86::CALL64r: + return X86::CALL32r; + case X86::XOR64rr: + return X86::XOR32rr; + // NaCl 2020 note: there used to be a lot of cmov opcodes, now there are only + // two left since they were unified. The condition (above, equal, ...) used + // to be part of the opcode, now it is encoded differently as per + // https://github.com/llvm/llvm-project/commit/e0bfeb5f24979416144c16e8b99204f5f163b889 + // If there are errors, maybe the reason could be that here it is not + // sufficient to just lower the opcode, but the operation needs to be + // constructed in a more sophisticated way. + case X86::CMOV64rr: + return X86::CMOV32rr; + case X86::CMOV64rm: + return X86::CMOV32rm; + case X86::CMP64rr: + return X86::CMP32rr; + case X86::CMP64ri8: + return X86::CMP32ri8; + case X86::CMP64ri32: + return X86::CMP32ri; + case X86::CMP64rm: + return X86::CMP32rm; + case X86::CMPXCHG64rr: + return X86::CMPXCHG32rr; + case X86::CRC32r64r8: + return X86::CRC32r32r8; + case X86::CRC32r64m8: + return X86::CRC32r64m8; + case X86::CRC32r64r64: + return X86::CRC32r32r32; + case X86::CRC32r64m64: + return X86::CRC32r32m32; + case X86::CVTSD2SI64rr_Int: + return X86::CVTSD2SIrr_Int; + case X86::CVTSD2SI64rm_Int: + return X86::CVTSD2SIrm_Int; + case X86::CVTSS2SI64rr_Int: + return X86::CVTSS2SIrr_Int; + case X86::CVTSS2SI64rm_Int: + return X86::CVTSS2SIrm_Int; + case X86::CVTTSD2SI64rr: + return X86::CVTTSD2SIrr; + case X86::CVTTSD2SI64rm: + return X86::CVTTSD2SIrm; + case X86::CVTTSS2SI64rr: + return X86::CVTTSS2SIrr; + case X86::CVTTSS2SI64rm: + return X86::CVTTSS2SIrm; + case X86::DEC64r: + return X86::DEC32r; + case X86::DIV64r: + return X86::DIV32r; + case X86::IDIV64r: + return X86::IDIV32r; + case X86::IMUL64r: + return X86::IMUL32r; + case X86::IMUL64rr: + return X86::IMUL32rr; + case X86::IMUL64rri8: + return X86::IMUL32rri8; + case X86::IMUL64rri32: + return X86::IMUL32rri; + case X86::IMUL64rm: + return X86::IMUL32rm; + case X86::IMUL64rmi8: + return X86::IMUL32rmi8; + case X86::IMUL64rmi32: + return X86::IMUL32rmi; + case X86::INC64r: + return X86::INC32r; + case X86::INVEPT64: + return X86::INVEPT32; + case X86::INVPCID64: + return X86::INVPCID32; + case X86::INVVPID64: + return X86::INVVPID32; + case X86::JMP64r: + return X86::JMP32r; + case X86::KMOVQrk: + return X86::KMOVQrk; + case X86::LAR64rr: + return X86::LAR32rr; + case X86::LAR64rm: + return X86::LAR32rm; + case X86::LEA64r: + return X86::LEA32r; + case X86::LFS64rm: + return X86::LFS32rm; + case X86::LGS64rm: + return X86::LGS32rm; + case X86::LSL64rr: + return X86::LSL32rr; + case X86::LSL64rm: + return X86::LSL32rm; + case X86::LSS64rm: + return X86::LSS32rm; + case X86::LZCNT64rr: + return X86::LZCNT32rr; + case X86::LZCNT64rm: + return X86::LZCNT32rm; + case X86::MOV64ri: + return X86::MOV32ri; + case X86::MOVBE64rm: + return X86::MOVBE32rm; + case X86::MOV64rr: + return X86::MOV32rr; + case X86::MMX_MOVD64from64rr: + return X86::MMX_MOVD64grr; + case X86::MOVPQIto64rr: + return X86::MOVPDI2DIrr; + case X86::MOV64rs: + return X86::MOV32rs; + case X86::MOV64rd: + return X86::MOV32rd; + case X86::MOV64rc: + return X86::MOV32rc; + case X86::MOV64ri32: + return X86::MOV32ri; + case X86::MOV64rm: + return X86::MOV32rm; + case X86::MOVSX64rr8: + return X86::MOVSX32rr8; + case X86::MOVSX64rm8: + return X86::MOVSX32rm8; + case X86::MOVSX64rr32: + return X86::MOV32rr; + case X86::MOVSX64rm32: + return X86::MOV32rm; + case X86::MOVSX64rr16: + return X86::MOVSX32rr16; + case X86::MOVSX64rm16: + return X86::MOVSX32rm16; + case X86::MOVZX64rr8: + return X86::MOVZX32rr8; + case X86::MOVZX64rm8: + return X86::MOVZX32rm8; + case X86::MOVZX64rr16: + return X86::MOVZX32rr16; + case X86::MOVZX64rm16: + return X86::MOVZX32rm16; + case X86::MUL64r: + return X86::MUL32r; + case X86::MULX64rr: + return X86::MULX32rr; + case X86::MULX64rm: + return X86::MULX32rm; + case X86::NEG64r: + return X86::NEG32r; + case X86::NOT64r: + return X86::NOT32r; + case X86::OR64rr: + return X86::OR32rr; + case X86::OR64ri8: + return X86::OR32ri8; + case X86::OR64ri32: + return X86::OR32ri; + case X86::OR64rm: + return X86::OR32rm; + case X86::PDEP64rr: + return X86::PDEP32rr; + case X86::PDEP64rm: + return X86::PDEP32rm; + case X86::PEXT64rr: + return X86::PEXT32rr; + case X86::PEXT64rm: + return X86::PEXT32rm; + case X86::PEXTRQrri: + return X86::PEXTRQrri; + case X86::POPCNT64rr: + return X86::POPCNT32rr; + case X86::POPCNT64rm: + return X86::POPCNT32rm; + case X86::POP64r: + return X86::POP32r; + case X86::POP64rmr: + return X86::POP32rmr; + // TODO POP64rmm? + case X86::PUSH64r: + return X86::PUSH32r; + case X86::PUSH64rmr: + return X86::PUSH32rmr; + case X86::RCL64r1: + return X86::RCL32r1; + case X86::RCL64rCL: + return X86::RCL32rCL; + case X86::RCL64ri: + return X86::RCL32ri; + case X86::RCR64r1: + return X86::RCR32r1; + case X86::RCR64rCL: + return X86::RCR32rCL; + case X86::RCR64ri: + return X86::RCR32ri; + case X86::RDFSBASE64: + return X86::RDFSBASE; + case X86::RDGSBASE64: + return X86::RDGSBASE; + case X86::RDRAND64r: + return X86::RDRAND32r; + case X86::RDSEED64r: + return X86::RDSEED32r; + case X86::ROL64r1: + return X86::ROL32r1; + case X86::ROL64rCL: + return X86::ROL32rCL; + case X86::ROL64ri: + return X86::ROL32ri; + case X86::ROR64r1: + return X86::ROR32r1; + case X86::ROR64rCL: + return X86::ROR32rCL; + case X86::ROR64ri: + return X86::ROR32ri; + case X86::RORX64ri: + return X86::RORX32ri; + case X86::RORX64mi: + return X86::RORX64mi; + case X86::SAR64r1: + return X86::SAR32r1; + case X86::SAR64rCL: + return X86::SAR32rCL; + case X86::SAR64ri: + return X86::SAR32ri; + case X86::SARX64rr: + return X86::SARX32rr; + case X86::SARX64rm: + return X86::SARX32rm; + case X86::SBB64rr: + return X86::SBB32rr; + case X86::SBB64ri8: + return X86::SBB32ri8; + case X86::SBB64ri32: + return X86::SBB32ri; + case X86::SBB64rm: + return X86::SBB32rm; + case X86::SHLD64rrCL: + return X86::SHLD32rrCL; + case X86::SHLD64rri8: + return X86::SHLD32rri8; + case X86::SHL64r1: + return X86::SHL32r1; + case X86::SHL64rCL: + return X86::SHL32rCL; + case X86::SHL64ri: + return X86::SHL32ri; + case X86::SHLX64rr: + return X86::SHLX32rr; + case X86::SHLX64rm: + return X86::SHLX32rm; + case X86::SHRD64rrCL: + return X86::SHRD32rrCL; + case X86::SHRD64rri8: + return X86::SHRD32rri8; + case X86::SHR64r1: + return X86::SHR32r1; + case X86::SHR64rCL: + return X86::SHR32rCL; + case X86::SHR64ri: + return X86::SHR32ri; + case X86::SHRX64rr: + return X86::SHRX32rr; + case X86::SHRX64rm: + return X86::SHRX32rm; + case X86::SLDT64r: + return X86::SLDT32r; + case X86::SMSW64r: + return X86::SMSW32r; + case X86::STR64r: + return X86::STR32r; + case X86::SUB64rr: + return X86::SUB32rr; + case X86::SUB64ri8: + return X86::SUB32ri8; + case X86::SUB64ri32: + return X86::SUB32ri; + case X86::SUB64rm: + return X86::SUB32rm; + case X86::T1MSKC64rr: + return X86::T1MSKC32rr; + case X86::T1MSKC64rm: + return X86::T1MSKC32rm; + case X86::TEST64rr: + return X86::TEST32rr; + case X86::TEST64ri32: + return X86::TEST32ri; + case X86::TEST64mr: + return X86::TEST32mr; + case X86::TZCNT64rr: + return X86::TZCNT32rr; + case X86::TZCNT64rm: + return X86::TZCNT32rm; + case X86::TZMSK64rr: + return X86::TZMSK32rr; + case X86::TZMSK64rm: + return X86::TZMSK32rm; + /* + Some more Candidates for this are: + // Candidates start + case X86::VCVTSI2SSZrr: + case X86::VCVTSI2SSZrm: + case X86::Int_VCVTSI2SSZrr: + case X86::Int_VCVTSI2SSZrm: + case X86::VCVTSI2SSZrr_Int: + case X86::VCVTSI2SSZrm_Int: + case X86::VCVTSI642SSZrr: + case X86::VCVTSI642SSZrm: + case X86::Int_VCVTSI2SS64Zrr: + case X86::Int_VCVTSI2SS64Zrm: + case X86::VCVTSI642SSZrr_Int: + case X86::VCVTSI642SSZrm_Int: + case X86::VCVTSI2SDZrr: + case X86::VCVTSI2SDZrm: + case X86::Int_VCVTSI2SDZrr: + case X86::Int_VCVTSI2SDZrm: + case X86::VCVTSI2SDZrr_Int: + case X86::VCVTSI2SDZrm_Int: + case X86::VCVTSI642SDZrr: + case X86::VCVTSI642SDZrm: + case X86::Int_VCVTSI2SD64Zrr: + case X86::Int_VCVTSI2SD64Zrm: + case X86::VCVTSI642SDZrr_Int: + case X86::VCVTSI642SDZrm_Int: + Maybe everything that is listed in hasUndefRegUpdate() in X86InstrInfo.cpp + // Candidates end + */ + case X86::VCVTSD2SI64rr_Int: + return X86::VCVTSD2SIrr_Int; + case X86::VCVTSD2SI64Zrr_Int: + return X86::VCVTSD2SIZrr_Int; + case X86::VCVTSD2SI64Zrm_Int: + return X86::VCVTSD2SIZrm_Int; + case X86::VCVTSD2SI64rm_Int: + return X86::VCVTSD2SIrm_Int; + case X86::VCVTSD2USI64Zrr_Int: + return X86::VCVTSD2USIZrr_Int; + case X86::VCVTSD2USI64Zrm_Int: + return X86::VCVTSD2USIZrm_Int; + case X86::VCVTSS2SI64rr_Int: + return X86::VCVTSS2SIrr_Int; + case X86::VCVTSS2SI64Zrr_Int: + return X86::VCVTSS2SIZrr_Int; + case X86::VCVTSS2SI64Zrm_Int: + return X86::VCVTSS2SIZrm_Int; + case X86::VCVTSS2SI64rm_Int: + return X86::VCVTSS2SIrm_Int; + case X86::VCVTSS2USI64Zrr_Int: + return X86::VCVTSS2USIZrr_Int; + case X86::VCVTSS2USI64Zrm_Int: + return X86::VCVTSS2USIZrm_Int; + case X86::VCVTTSD2SI64rr: + return X86::VCVTTSD2SIrr; + case X86::VCVTTSD2SI64Zrr: + return X86::VCVTTSD2SIZrr; + case X86::VCVTTSD2SI64Zrm: + return X86::VCVTTSD2SIZrm; + case X86::VCVTTSD2SI64rm: + return X86::VCVTTSD2SIrm; + case X86::VCVTTSD2USI64Zrr: + return X86::VCVTTSD2USIZrr; + case X86::VCVTTSD2USI64Zrm: + return X86::VCVTTSD2USIZrm; + case X86::VCVTTSS2SI64rr: + return X86::VCVTTSS2SIrr; + case X86::VCVTTSS2SI64Zrr: + return X86::VCVTTSS2SIZrr; + case X86::VCVTTSS2SI64Zrm: + return X86::VCVTTSS2SIZrm; + case X86::VCVTTSS2SI64rm: + return X86::VCVTTSS2SIrm; + case X86::VCVTTSS2USI64Zrr: + return X86::VCVTTSS2USIZrr; + case X86::VCVTTSS2USI64Zrm: + return X86::VCVTTSS2USIZrm; + case X86::VMOVPQIto64rr: + return X86::VMOVPDI2DIrr; + case X86::VMOVPQIto64Zrr: + return X86::VMOVPDI2DIZrr; + case X86::VMREAD64rr: + return X86::VMREAD32rr; + case X86::VMWRITE64rr: + return X86::VMWRITE32rr; + case X86::VMWRITE64rm: + return X86::VMWRITE32rm; + case X86::VPEXTRQrri: + return X86::VPEXTRQrri; + case X86::WRFSBASE64: + return X86::WRFSBASE; + case X86::WRGSBASE64: + return X86::WRGSBASE; + case X86::XADD64rr: + return X86::XADD32rr; + case X86::XCHG64ar: + return X86::XCHG32ar; + case X86::XCHG64rr: + return X86::XCHG32rr; + case X86::XCHG64rm: + return X86::XCHG32rm; + case X86::XOR64ri8: + return X86::XOR32ri8; + case X86::XOR64ri32: + return X86::XOR32ri; + case X86::XOR64rm: + return X86::XOR32rm; + default: + return Opcode; + } +} diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.h b/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.h new file mode 100644 index 0000000000000..5bde669aa3d8e --- /dev/null +++ b/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.h @@ -0,0 +1,107 @@ +//===- X86MCLFIExpander.h - X86 LFI Expander -------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +// This file was written by the Native Client authors. +// +//===----------------------------------------------------------------------===// +// +// This file declares the X86MCLFIExpander class, the X86 specific +// subclass of MCLFIExpander. +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_MC_X86MCNACLEXPANDER_H +#define LLVM_MC_X86MCNACLEXPANDER_H + +#include "llvm/ADT/SmallVector.h" +#include "llvm/MC/MCInst.h" +#include "llvm/MC/MCInstrInfo.h" +#include "llvm/MC/MCLFIExpander.h" +#include "llvm/MC/MCRegisterInfo.h" + +namespace llvm { +class MCContext; +class MCStreamer; +class MCSubtargetInfo; + +namespace X86 { +class X86MCLFIExpander : public MCLFIExpander { +public: + X86MCLFIExpander(MCContext &Ctx, std::unique_ptr &&RI, + std::unique_ptr &&II) + : MCLFIExpander(Ctx, std::move(RI), std::move(II)) {} + + bool expandInst(const MCInst &Inst, MCStreamer &Out, + const MCSubtargetInfo &STI) override; + +protected: + bool isValidScratchRegister(MCRegister Reg) const override; + +private: + bool Guard = false; // recursion guard + SmallVector Prefixes; + + void emitPrefixes(MCStreamer &Out, const MCSubtargetInfo &STI); + + void expandDirectCall(const MCInst &Inst, MCStreamer &Out, + const MCSubtargetInfo &STI); + + void expandIndirectBranch(const MCInst &Inst, MCStreamer &Out, + const MCSubtargetInfo &STI); + + void expandReturn(const MCInst &Inst, MCStreamer &Out, + const MCSubtargetInfo &STI); + + void expandSyscall(const MCInst &Inst, MCStreamer &Out, + const MCSubtargetInfo &STI); + + void expandTLSRead(const MCInst &Inst, MCStreamer &Out, + const MCSubtargetInfo &STI); + + void expandLoadStore(const MCInst &Inst, MCStreamer &Out, + const MCSubtargetInfo &STI, bool EmitPrefixes); + + void expandStringOperation(const MCInst &Inst, MCStreamer &Out, + const MCSubtargetInfo &STI, bool EmitPrefixes); + + void doExpandInst(const MCInst &Inst, MCStreamer &Out, + const MCSubtargetInfo &STI, bool EmitPrefixes); + + void expandExplicitStackManipulation(MCRegister StackReg, const MCInst &Inst, + MCStreamer &Out, + const MCSubtargetInfo &STI, + bool EmitPrefixes); + + void prepareSandboxMemOp(MCInst &Inst, int MemIdx, MCRegister ScratchReg, + MCStreamer &Out, const MCSubtargetInfo &STI); + + void emitSandboxMemOp(MCInst &Inst, int MemIdx, MCRegister ScratchReg, + MCStreamer &Out, const MCSubtargetInfo &STI); + + bool emitSandboxMemOps(MCInst &Inst, MCRegister ScratchReg, MCStreamer &Out, + const MCSubtargetInfo &STI, bool EmitInstructions); + + void emitInstruction(const MCInst &Inst, MCStreamer &Out, + const MCSubtargetInfo &STI, bool EmitPrefixes); + + void emitSandboxBranchReg(MCRegister Reg, MCStreamer &Out, + const MCSubtargetInfo &STI); + void emitIndirectJumpReg(MCRegister Reg, MCStreamer &Out, + const MCSubtargetInfo &STI); + void emitIndirectCallReg(MCRegister Reg, MCStreamer &Out, + const MCSubtargetInfo &STI); + + enum LFICallType { + LFISyscall, + LFITLSRead, + LFITLSWrite, + }; + + void emitLFICall(LFICallType CallType, MCStreamer &Out, + const MCSubtargetInfo &STI); +}; +} // namespace X86 +} // namespace llvm +#endif diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.cpp index 1c4d68d5448d6..f649bdf09b714 100644 --- a/llvm/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.cpp +++ b/llvm/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.cpp @@ -16,6 +16,7 @@ #include "X86BaseInfo.h" #include "X86IntelInstPrinter.h" #include "X86MCAsmInfo.h" +#include "X86MCLFIExpander.h" #include "X86TargetStreamer.h" #include "llvm/ADT/APInt.h" #include "llvm/DebugInfo/CodeView/CodeView.h" @@ -708,6 +709,14 @@ static MCInstrAnalysis *createX86MCInstrAnalysis(const MCInstrInfo *Info) { return new X86_MC::X86MCInstrAnalysis(Info); } +static MCLFIExpander *createX86_64MCLFIExpander(MCStreamer &S, std::unique_ptr &&RegInfo, std::unique_ptr &&InstInfo) { + auto *Exp = + new X86::X86MCLFIExpander(S.getContext(), std::move(RegInfo), + std::move(InstInfo)); + S.setLFIExpander(Exp); + return Exp; +} + // Force static initialization. extern "C" LLVM_C_ABI void LLVMInitializeX86TargetMC() { for (Target *T : {&getTheX86_32Target(), &getTheX86_64Target()}) { @@ -755,6 +764,9 @@ extern "C" LLVM_C_ABI void LLVMInitializeX86TargetMC() { createX86_32AsmBackend); TargetRegistry::RegisterMCAsmBackend(getTheX86_64Target(), createX86_64AsmBackend); + + TargetRegistry::RegisterMCLFIExpander(getTheX86_64Target(), + createX86_64MCLFIExpander); } MCRegister llvm::getX86SubSuperRegister(MCRegister Reg, unsigned Size, diff --git a/llvm/lib/Target/X86/X86.h b/llvm/lib/Target/X86/X86.h index 48a3fe1934a96..d3fd0315d18ae 100644 --- a/llvm/lib/Target/X86/X86.h +++ b/llvm/lib/Target/X86/X86.h @@ -42,6 +42,9 @@ FunctionPass *createCleanupLocalDynamicTLSPass(); /// physical instructions. FunctionPass *createX86FloatingPointStackifierPass(); +// Creates a pass to modify code generation for LFI rules. +FunctionPass *createX86LFIRewritePass(); + /// This pass inserts AVX vzeroupper instructions before each call to avoid /// transition penalty between functions encoded with AVX and SSE. FunctionPass *createX86IssueVZeroUpperPass(); diff --git a/llvm/lib/Target/X86/X86.td b/llvm/lib/Target/X86/X86.td index 577428cad6d61..e285eaac4ce18 100644 --- a/llvm/lib/Target/X86/X86.td +++ b/llvm/lib/Target/X86/X86.td @@ -772,6 +772,22 @@ def TuningUseGLMDivSqrtCosts def TuningBranchHint: SubtargetFeature<"branch-hint", "HasBranchHint", "true", "Target has branch hint feature">; +//===----------------------------------------------------------------------===// +// LFI Subtarget Features +//===----------------------------------------------------------------------===// + +def FeatureLFIStores : SubtargetFeature<"lfi-stores", "LFIStores", "true", + "Isolate only store+jump instructions">; + +def FeatureLFIJumps : SubtargetFeature<"lfi-jumps", "LFIJumps", "true", + "Isolate only jump instructions">; + +def FeatureLFITLSReg : SubtargetFeature<"lfi-tls-reg", "LFITLSReg", "true", + "Reserve a register for thread-local LFI data">; + +def FeatureLFINoSegue : SubtargetFeature<"lfi-no-segue", "LFINoSegue", "true", + "Do not use segmentation register for LFI">; + //===----------------------------------------------------------------------===// // X86 CPU Families // TODO: Remove these - use general tuning features to determine codegen. diff --git a/llvm/lib/Target/X86/X86AsmPrinter.cpp b/llvm/lib/Target/X86/X86AsmPrinter.cpp index f01e47b41cf5e..141c3af015c45 100644 --- a/llvm/lib/Target/X86/X86AsmPrinter.cpp +++ b/llvm/lib/Target/X86/X86AsmPrinter.cpp @@ -38,6 +38,7 @@ #include "llvm/MC/MCExpr.h" #include "llvm/MC/MCInst.h" #include "llvm/MC/MCInstBuilder.h" +#include "llvm/MC/MCLFI.h" #include "llvm/MC/MCSectionCOFF.h" #include "llvm/MC/MCSectionELF.h" #include "llvm/MC/MCSectionMachO.h" @@ -928,6 +929,9 @@ void X86AsmPrinter::emitStartOfAsmFile(Module &M) { bool is16 = TT.getEnvironment() == Triple::CODE16; if (M.getModuleInlineAsm().empty() && is16) OutStreamer->emitAssemblerFlag(MCAF_Code16); + + if (TT.isLFI()) + initializeLFIMCStreamer(*OutStreamer.get(), OutContext, TT); } static void diff --git a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp index 84bcdae520885..3a60a668fff56 100644 --- a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp +++ b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp @@ -182,6 +182,10 @@ namespace { IndirectTlsSegRefs = MF.getFunction().hasFnAttribute( "indirect-tls-seg-refs"); + // Always enable IndirectTlsSegRefs for LFI. + if (Subtarget->isLFI()) + IndirectTlsSegRefs = true; + // OptFor[Min]Size are used in pattern predicates that isel is matching. OptForMinSize = MF.getFunction().hasMinSize(); assert((!OptForMinSize || MF.getFunction().hasOptSize()) && diff --git a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp index 6835c7e336a5c..d5181466edfc9 100644 --- a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp +++ b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp @@ -2820,6 +2820,11 @@ bool X86TargetLowering::IsEligibleForTailCallOptimization( if (IsCalleeWin64 || IsCallerWin64) return false; + // Do not optimize vararg calls with 6 arguments for LFI because LFI + // reserves %r11. + if (Subtarget.isLFI() && ArgLocs.size() > 5) + return false; + for (const auto &VA : ArgLocs) if (!VA.isRegLoc()) return false; diff --git a/llvm/lib/Target/X86/X86LFIRewritePass.cpp b/llvm/lib/Target/X86/X86LFIRewritePass.cpp new file mode 100644 index 0000000000000..bc1f3040fbbbc --- /dev/null +++ b/llvm/lib/Target/X86/X86LFIRewritePass.cpp @@ -0,0 +1,57 @@ +//=== X86LFIRewritePAss.cpp - Modify instructions for LFI ---------*- C++ -*-=// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +#define DEBUG_TYPE "lfi-rewrite-pass" + +#include "X86.h" +#include "X86InstrInfo.h" +#include "X86Subtarget.h" +#include "llvm/CodeGen/MachineFunctionPass.h" +#include "llvm/CodeGen/MachineInstr.h" +#include "llvm/Support/Alignment.h" +#include "llvm/Support/Debug.h" + +using namespace llvm; + +namespace { +class X86LFIRewritePass : public MachineFunctionPass { +public: + static char ID; + X86LFIRewritePass() : MachineFunctionPass(ID) {} + + virtual bool runOnMachineFunction(MachineFunction &Fn) override; + + virtual StringRef getPassName() const override { return "LFI Rewrites"; } + +private: + const TargetInstrInfo *TII; + const X86Subtarget *Subtarget; +}; + +char X86LFIRewritePass::ID = 0; +} // namespace + +bool X86LFIRewritePass::runOnMachineFunction(MachineFunction &MF) { + bool Modified = false; + + TII = MF.getSubtarget().getInstrInfo(); + Subtarget = &MF.getSubtarget(); + assert(Subtarget->isLFI() && "Unexpected target in LFIRewritePass!"); + + MF.setAlignment(llvm::Align(32)); + + for (MachineBasicBlock &MBB : MF) { + // TODO: consider only setting this if MBB.hasAddressTaken() is true. + MBB.setAlignment(llvm::Align(32)); + } + return Modified; +} + +/// createX86LFIRewritePassPass - returns an instance of the pass. +namespace llvm { +FunctionPass *createX86LFIRewritePass() { return new X86LFIRewritePass(); } +} // namespace llvm diff --git a/llvm/lib/Target/X86/X86RegisterInfo.cpp b/llvm/lib/Target/X86/X86RegisterInfo.cpp index 4faf8bca4f9e0..0d33b2ac2af73 100644 --- a/llvm/lib/Target/X86/X86RegisterInfo.cpp +++ b/llvm/lib/Target/X86/X86RegisterInfo.cpp @@ -557,6 +557,13 @@ BitVector X86RegisterInfo::getReservedRegs(const MachineFunction &MF) const { // Set the SIMD floating point control register as reserved. Reserved.set(X86::MXCSR); + if (MF.getSubtarget().isLFI()) { + for (const MCPhysReg &SubReg : subregs_inclusive(X86::R11)) // scratch + Reserved.set(SubReg); + for (const MCPhysReg &SubReg : subregs_inclusive(X86::R14)) // base + Reserved.set(SubReg); + } + // Set the stack-pointer register and its aliases as reserved. for (const MCPhysReg &SubReg : subregs_inclusive(X86::RSP)) Reserved.set(SubReg); diff --git a/llvm/lib/Target/X86/X86Subtarget.h b/llvm/lib/Target/X86/X86Subtarget.h index 722076ca88c9c..db7376ba56612 100644 --- a/llvm/lib/Target/X86/X86Subtarget.h +++ b/llvm/lib/Target/X86/X86Subtarget.h @@ -179,6 +179,8 @@ class X86Subtarget final : public X86GenSubtargetInfo { return Is64Bit && (!TargetTriple.isX32() && !TargetTriple.isOSNaCl()); } + bool isLFI() const { return TargetTriple.isX8664LFI(); } + PICStyles::Style getPICStyle() const { return PICStyle; } void setPICStyle(PICStyles::Style Style) { PICStyle = Style; } diff --git a/llvm/lib/Target/X86/X86TargetMachine.cpp b/llvm/lib/Target/X86/X86TargetMachine.cpp index 20dfdd27b33df..4a8785b0f2b96 100644 --- a/llvm/lib/Target/X86/X86TargetMachine.cpp +++ b/llvm/lib/Target/X86/X86TargetMachine.cpp @@ -611,6 +611,9 @@ void X86PassConfig::addPreEmitPass() { addPass(createX86DiscriminateMemOpsPass()); addPass(createX86InsertPrefetchPass()); addPass(createX86InsertX87waitPass()); + + if (Triple(TM->getTargetTriple()).isLFI()) + addPass(createX86LFIRewritePass()); } void X86PassConfig::addPreEmitPass2() { diff --git a/llvm/lib/TargetParser/Triple.cpp b/llvm/lib/TargetParser/Triple.cpp index e9e6f130f757c..4aeb1bf625112 100644 --- a/llvm/lib/TargetParser/Triple.cpp +++ b/llvm/lib/TargetParser/Triple.cpp @@ -91,6 +91,10 @@ StringRef Triple::getArchTypeName(ArchType Kind) { StringRef Triple::getArchName(ArchType Kind, SubArchType SubArch) { switch (Kind) { + case Triple::x86_64: + if (SubArch == X8664SubArch_lfi) + return "x86_64_lfi"; + break; case Triple::mips: if (SubArch == MipsSubArch_r6) return "mipsisa32r6"; @@ -112,6 +116,8 @@ StringRef Triple::getArchName(ArchType Kind, SubArchType SubArch) { return "arm64ec"; if (SubArch == AArch64SubArch_arm64e) return "arm64e"; + if (SubArch == AArch64SubArch_lfi) + return "aarch64_lfi"; break; case Triple::spirv: switch (SubArch) { @@ -561,6 +567,7 @@ static Triple::ArchType parseArch(StringRef ArchName) { // FIXME: Do we need to support these? .Cases("i786", "i886", "i986", Triple::x86) .Cases("amd64", "x86_64", "x86_64h", Triple::x86_64) + .Cases("amd64_lfi", "x86_64_lfi", Triple::x86_64) .Cases("powerpc", "powerpcspe", "ppc", "ppc32", Triple::ppc) .Cases("powerpcle", "ppcle", "ppc32le", Triple::ppcle) .Cases("powerpc64", "ppu", "ppc64", Triple::ppc64) @@ -575,6 +582,7 @@ static Triple::ArchType parseArch(StringRef ArchName) { .Case("arm64_32", Triple::aarch64_32) .Case("arm64e", Triple::aarch64) .Case("arm64ec", Triple::aarch64) + .Case("aarch64_lfi", Triple::aarch64) .Case("arm", Triple::arm) .Case("armeb", Triple::armeb) .Case("thumb", Triple::thumb) @@ -797,6 +805,12 @@ static Triple::SubArchType parseSubArch(StringRef SubArchName) { if (SubArchName == "arm64ec") return Triple::AArch64SubArch_arm64ec; + if (SubArchName == "aarch64_lfi") + return Triple::AArch64SubArch_lfi; + + if (SubArchName == "x86_64_lfi") + return Triple::X8664SubArch_lfi; + if (SubArchName.starts_with("spirv")) return StringSwitch(SubArchName) .EndsWith("v1.0", Triple::SPIRVSubArch_v10) diff --git a/llvm/lib/Transforms/Scalar/TailRecursionElimination.cpp b/llvm/lib/Transforms/Scalar/TailRecursionElimination.cpp index 3f27166080d5a..f78a88832db15 100644 --- a/llvm/lib/Transforms/Scalar/TailRecursionElimination.cpp +++ b/llvm/lib/Transforms/Scalar/TailRecursionElimination.cpp @@ -77,6 +77,7 @@ #include "llvm/Pass.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" +#include "llvm/TargetParser/Triple.h" #include "llvm/Transforms/Scalar.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" using namespace llvm; diff --git a/llvm/test/MC/AArch64/LFI/branch.s b/llvm/test/MC/AArch64/LFI/branch.s new file mode 100644 index 0000000000000..e2c126cf24db7 --- /dev/null +++ b/llvm/test/MC/AArch64/LFI/branch.s @@ -0,0 +1,20 @@ +// RUN: llvm-mc -filetype asm -triple aarch64_lfi %s | FileCheck %s + +foo: + b foo +// CHECK: b foo + + br x0 +// CHECK: add x28, x27, w0, uxtw +// CHECK-NEXT: br x28 + + blr x0 +// CHECK: add x28, x27, w0, uxtw +// CHECK-NEXT: blr x28 + + ret +// CHECK: ret + + ret x0 +// CHECK: add x28, x27, w0, uxtw +// CHECK-NEXT: ret x28 diff --git a/llvm/test/MC/AArch64/LFI/lse.s b/llvm/test/MC/AArch64/LFI/lse.s new file mode 100644 index 0000000000000..bda6c45c7a850 --- /dev/null +++ b/llvm/test/MC/AArch64/LFI/lse.s @@ -0,0 +1,15 @@ +// RUN: llvm-mc -filetype asm -triple aarch64_lfi %s | FileCheck %s + +.arch_extension lse + +ldadd x0, x1, [x2] +// CHECK: add x28, x27, w2, uxtw +// CHECK-NEXT: ldadd x0, x1, [x28] + +swpal w0, w0, [x1] +// CHECK: add x28, x27, w1, uxtw +// CHECK-NEXT: swpal w0, w0, [x28] + +caspal w0, w1, w2, w3, [x4] +// CHECK: add x28, x27, w4, uxtw +// CHECK-NEXT: caspal w0, w1, w2, w3, [x28] diff --git a/llvm/test/MC/AArch64/LFI/mem.s b/llvm/test/MC/AArch64/LFI/mem.s new file mode 100644 index 0000000000000..6d87d3b29ba8f --- /dev/null +++ b/llvm/test/MC/AArch64/LFI/mem.s @@ -0,0 +1,181 @@ +// RUN: llvm-mc -filetype asm -triple aarch64_lfi %s | FileCheck %s + +ldr x0, [sp] +// CHECK: ldr x0, [sp] + +ldr x0, [sp, #8] +// CHECK: ldr x0, [sp, #8] + +ldp x0, x1, [sp, #8] +// CHECK: ldp x0, x1, [sp, #8] + +str x0, [sp] +// CHECK: str x0, [sp] + +str x0, [sp, #8] +// CHECK: str x0, [sp, #8] + +stp x0, x1, [sp, #8] +// CHECK: stp x0, x1, [sp, #8] + +ldur x0, [x1] +// CHECK: add x28, x27, w1, uxtw +// CHECK-NEXT: ldur x0, [x28] + +stur x0, [x1] +// CHECK: add x28, x27, w1, uxtw +// CHECK-NEXT: stur x0, [x28] + +ldp x0, x1, [x2] +// CHECK: add x28, x27, w2, uxtw +// CHECK-NEXT: ldp x0, x1, [x28] + +stp x0, x1, [x2] +// CHECK: add x28, x27, w2, uxtw +// CHECK-NEXT: stp x0, x1, [x28] + +ldr x0, [x1] +// CHECK: ldr x0, [x27, w1, uxtw] + +ldr x0, [x1, #8] +// CHECK: add x28, x27, w1, uxtw +// CHECK-NEXT: ldr x0, [x28, #8] + +ldr x0, [x1, #8]! +// CHECK: add x1, x1, #8 +// CHECK-NEXT: ldr x0, [x27, w1, uxtw] + +str x0, [x1, #8]! +// CHECK: add x1, x1, #8 +// CHECK-NEXT: str x0, [x27, w1, uxtw] + +ldr x0, [x1, #-8]! +// CHECK: sub x1, x1, #8 +// CHECK-NEXT: ldr x0, [x27, w1, uxtw] + +str x0, [x1, #-8]! +// CHECK: sub x1, x1, #8 +// CHECK-NEXT: str x0, [x27, w1, uxtw] + +ldr x0, [x1], #8 +// CHECK: ldr x0, [x27, w1, uxtw] +// CHECK-NEXT: add x1, x1, #8 + +str x0, [x1], #8 +// CHECK: str x0, [x27, w1, uxtw] +// CHECK-NEXT: add x1, x1, #8 + +ldr x0, [x1], #-8 +// CHECK: ldr x0, [x27, w1, uxtw] +// CHECK-NEXT: sub x1, x1, #8 + +str x0, [x1], #-8 +// CHECK: str x0, [x27, w1, uxtw] +// CHECK-NEXT: sub x1, x1, #8 + +ldr x0, [x1, x2] +// CHECK: add x26, x1, x2 +// CHECK-NEXT: ldr x0, [x27, w26, uxtw] + +ldr x0, [x1, x2, lsl #3] +// CHECK: add x26, x1, x2, lsl #3 +// CHECK-NEXT: ldr x0, [x27, w26, uxtw] + +ldr x0, [x1, x2, sxtx #0] +// CHECK: add x26, x1, x2, sxtx +// CHECK-NEXT: ldr x0, [x27, w26, uxtw] + +ldr x0, [x1, x2, sxtx #3] +// CHECK: add x26, x1, x2, sxtx #3 +// CHECK-NEXT: ldr x0, [x27, w26, uxtw] + +ldr x0, [x1, w2, uxtw] +// CHECK: add x26, x1, w2, uxtw +// CHECK-NEXT: ldr x0, [x27, w26, uxtw] + +ldr x0, [x1, w2, uxtw #3] +// CHECK: add x26, x1, w2, uxtw #3 +// CHECK-NEXT: ldr x0, [x27, w26, uxtw] + +ldr x0, [x1, w2, sxtw] +// CHECK: add x26, x1, w2, sxtw +// CHECK-NEXT: ldr x0, [x27, w26, uxtw] + +ldr x0, [x1, w2, sxtw #3] +// CHECK: add x26, x1, w2, sxtw #3 +// CHECK-NEXT: ldr x0, [x27, w26, uxtw] + +ldp x0, x1, [sp], #8 +// CHECK: ldp x0, x1, [sp], #8 + +ldp x0, x1, [x2], #8 +// CHECK: add x28, x27, w2, uxtw +// CHECK-NEXT: ldp x0, x1, [x28] +// CHECK-NEXT: add x2, x2, #8 + +ldp x0, x1, [x2, #8]! +// CHECK: add x28, x27, w2, uxtw +// CHECK-NEXT: ldp x0, x1, [x28, #8] +// CHECK-NEXT: add x2, x2, #8 + +ldp x0, x1, [x2], #-8 +// CHECK: add x28, x27, w2, uxtw +// CHECK-NEXT: ldp x0, x1, [x28] +// CHECK-NEXT: sub x2, x2, #8 + +ldp x0, x1, [x2, #-8]! +// CHECK: add x28, x27, w2, uxtw +// CHECK-NEXT: ldp x0, x1, [x28, #-8] +// CHECK-NEXT: sub x2, x2, #8 + +stp x0, x1, [x2, #-8]! +// CHECK: add x28, x27, w2, uxtw +// CHECK-NEXT: stp x0, x1, [x28, #-8] +// CHECK-NEXT: sub x2, x2, #8 + +ld3 { v0.4s, v1.4s, v2.4s }, [x0], #48 +// CHECK: add x28, x27, w0, uxtw +// CHECK-NEXT: ld3 { v0.4s, v1.4s, v2.4s }, [x28] +// CHECK-NEXT: add x0, x0, #48 + +st2 { v1.8b, v2.8b }, [x14], #16 +// CHECK: add x28, x27, w14, uxtw +// CHECK-NEXT: st2 { v1.8b, v2.8b }, [x28] +// CHECK-NEXT: add x14, x14, #16 + +st2 { v1.8b, v2.8b }, [x14] +// CHECK: add x28, x27, w14, uxtw +// CHECK-NEXT: st2 { v1.8b, v2.8b }, [x28] + +ld1 { v0.s }[1], [x8] +// CHECK: add x28, x27, w8, uxtw +// CHECK-NEXT: ld1 { v0.s }[1], [x28] + +ld1r { v3.2d }, [x9] +// CHECK: add x28, x27, w9, uxtw +// CHECK-NEXT: ld1r { v3.2d }, [x28] + +ld1 { v0.s }[1], [x8], x10 +// CHECK: add x28, x27, w8, uxtw +// CHECK-NEXT: ld1 { v0.s }[1], [x28] +// CHECK-NEXT: add x8, x8, x10 + +ldaxr x0, [x2] +// CHECK: add x28, x27, w2, uxtw +// CHECK-NEXT: ldaxr x0, [x28] + +stlxr w15, w17, [x1] +// CHECK: add x28, x27, w1, uxtw +// CHECK-NEXT: stlxr w15, w17, [x28] + +ldr w4, [sp, w3, uxtw #2] +// CHECK: add x26, sp, w3, uxtw #2 +// CHECK-NEXT: ldr w4, [x27, w26, uxtw] + +stxrb w11, w10, [x8] +// CHECK: add x28, x27, w8, uxtw +// CHECK-NEXT: stxrb w11, w10, [x28] + +ldr x0, [x0, :got_lo12:x] +// CHECK: add x28, x27, w0, uxtw +// CHECK-NEXT: ldr x0, [x28, :got_lo12:x] diff --git a/llvm/test/MC/AArch64/LFI/other.s b/llvm/test/MC/AArch64/LFI/other.s new file mode 100644 index 0000000000000..c63ee1cd5d88f --- /dev/null +++ b/llvm/test/MC/AArch64/LFI/other.s @@ -0,0 +1,6 @@ +// RUN: llvm-mc -filetype asm -triple aarch64_lfi %s | FileCheck %s + +.no_expand +ldr x0, [x1] +// CHECK: ldr x0, [x1] +.expand diff --git a/llvm/test/MC/AArch64/LFI/reserved.s b/llvm/test/MC/AArch64/LFI/reserved.s new file mode 100644 index 0000000000000..e387ec5ec21e2 --- /dev/null +++ b/llvm/test/MC/AArch64/LFI/reserved.s @@ -0,0 +1,21 @@ +// RUN: llvm-mc -filetype asm -triple aarch64_lfi %s 2> %t +// RUN: FileCheck --check-prefix=CHECK-ERROR %s <%t + +mov x28, x0 +// CHECK: mov x26, x0 +// CHECK-ERROR: warning: deleting modification of reserved LFI register +// CHECK-ERROR: mov x28, x0 +// CHECK-ERROR: ^ + +ldr x27, [x0] +// CHECK: ldr xzr, [x27, w0, uxtw] +// CHECK-ERROR: warning: deleting modification of reserved LFI register +// CHECK-ERROR: ldr x27, [x0] +// CHECK-ERROR: ^ + +ldp x27, x28, [x0] +// CHECK: add x28, x27, w0, uxtw +// CHECK-NEXT: ldp xzr, x26, [x28] +// CHECK-ERROR: warning: deleting modification of reserved LFI register +// CHECK-ERROR: ldp x27, x28, [x0] +// CHECK-ERROR: ^ diff --git a/llvm/test/MC/AArch64/LFI/return.s b/llvm/test/MC/AArch64/LFI/return.s new file mode 100644 index 0000000000000..1295626134969 --- /dev/null +++ b/llvm/test/MC/AArch64/LFI/return.s @@ -0,0 +1,19 @@ +// RUN: llvm-mc -filetype asm -triple aarch64_lfi %s | FileCheck %s + +mov x30, x0 +// CHECK: mov x26, x0 +// CHECK-NEXT: add x30, x27, w26, uxtw + +ldr x30, [sp] +// CHECK: ldr x26, [sp] +// CHECK-NEXT: add x30, x27, w26, uxtw + +ldp x29, x30, [sp] +// CHECK: ldp x29, x26, [sp] +// CHECK-NEXT: add x30, x27, w26, uxtw + +.scratch x10 +ldr x30, [sp] +// CHECK: ldr x10, [sp] +// CHECK-NEXT: add x30, x27, w10, uxtw +.scratch_clear diff --git a/llvm/test/MC/AArch64/LFI/stack.s b/llvm/test/MC/AArch64/LFI/stack.s new file mode 100644 index 0000000000000..bae875071dc72 --- /dev/null +++ b/llvm/test/MC/AArch64/LFI/stack.s @@ -0,0 +1,43 @@ +// RUN: llvm-mc -filetype asm -triple aarch64_lfi %s | FileCheck %s + +ldr x0, [sp, #16]! +// CHECK: ldr x0, [sp, #16]! + +ldr x0, [sp], #16 +// CHECK: ldr x0, [sp], #16 + +str x0, [sp, #16]! +// CHECK: str x0, [sp, #16]! + +str x0, [sp], #16 +// CHECK: str x0, [sp], #16 + +mov sp, x0 +// CHECK: add x26, x0, #0 +// CHECK-NEXT: add sp, x27, w26, uxtw + +add sp, sp, #8 +// CHECK: add x26, sp, #8 +// CHECK-NEXT: add sp, x27, w26, uxtw + +sub sp, sp, #8 +// CHECK: sub x26, sp, #8 +// CHECK-NEXT: add sp, x27, w26, uxtw + +add sp, sp, x0 +// CHECK: add x26, sp, x0 +// CHECK-NEXT: add sp, x27, w26, uxtw + +sub sp, sp, x0 +// CHECK: sub x26, sp, x0 +// CHECK-NEXT: add sp, x27, w26, uxtw + +sub sp, sp, #1, lsl #12 +// CHECK: sub x26, sp, #1, lsl #12 +// CHECK-NEXT: add sp, x27, w26, uxtw + +.scratch x10 +add sp, sp, x0 +// CHECK: add x10, sp, x0 +// CHECK-NEXT: add sp, x27, w10, uxtw +.scratch_clear diff --git a/llvm/test/MC/AArch64/LFI/sys.s b/llvm/test/MC/AArch64/LFI/sys.s new file mode 100644 index 0000000000000..6f54fcad754d6 --- /dev/null +++ b/llvm/test/MC/AArch64/LFI/sys.s @@ -0,0 +1,49 @@ +// RUN: llvm-mc -filetype asm -triple aarch64_lfi %s | FileCheck %s + +svc #0 +// CHECK: mov x26, x30 +// CHECK-NEXT: ldr x30, [x27] +// CHECK-NEXT: blr x30 +// CHECK-NEXT: add x30, x27, w26, uxtw + +.scratch x10 +svc #0 +// CHECK: mov x10, x30 +// CHECK-NEXT: ldr x30, [x27] +// CHECK-NEXT: blr x30 +// CHECK-NEXT: add x30, x27, w10, uxtw +.scratch_clear + +msr tpidr_el0, x0 +// CHECK: mov x26, x30 +// CHECK-NEXT: ldr x30, [x27, #16] +// CHECK-NEXT: blr x30 +// CHECK-NEXT: add x30, x27, w26, uxtw + +msr tpidr_el0, x1 +// CHECK: eor x1, x1, x0 +// CHECK-NEXT: eor x0, x1, x0 +// CHECK-NEXT: eor x1, x1, x0 +// CHECK-NEXT: mov x26, x30 +// CHECK-NEXT: ldr x30, [x27, #16] +// CHECK-NEXT: blr x30 +// CHECK-NEXT: add x30, x27, w26, uxtw +// CHECK-NEXT: eor x0, x0, x1 +// CHECK-NEXT: eor x1, x0, x1 +// CHECK-NEXT: eor x0, x0, x1 + +mrs x0, tpidr_el0 +// CHECK: mov x26, x30 +// CHECK-NEXT: ldr x30, [x27, #8] +// CHECK-NEXT: blr x30 +// CHECK-NEXT: add x30, x27, w26, uxtw + +mrs x1, tpidr_el0 +// CHECK: mov x1, x0 +// CHECK-NEXT: mov x26, x30 +// CHECK-NEXT: ldr x30, [x27, #8] +// CHECK-NEXT: blr x30 +// CHECK-NEXT: add x30, x27, w26, uxtw +// CHECK-NEXT: eor x0, x0, x1 +// CHECK-NEXT: eor x1, x0, x1 +// CHECK-NEXT: eor x0, x0, x1 diff --git a/llvm/test/MC/AArch64/LFI/tls-reg.s b/llvm/test/MC/AArch64/LFI/tls-reg.s new file mode 100644 index 0000000000000..b079f8316f89e --- /dev/null +++ b/llvm/test/MC/AArch64/LFI/tls-reg.s @@ -0,0 +1,13 @@ +// RUN: llvm-mc -filetype asm -triple aarch64_lfi -mattr +lfi-tls-reg %s | FileCheck %s + +msr tpidr_el0, x0 +// CHECK: mov x26, x30 +// CHECK-NEXT: ldr x30, [x27, #16] +// CHECK-NEXT: blr x30 +// CHECK-NEXT: add x30, x27, w26, uxtw + +mrs x0, tpidr_el0 +// CHECK: ldr x0, [x25] + +mrs x1, tpidr_el0 +// CHECK: ldr x1, [x25] diff --git a/llvm/test/MC/X86/LFI/branch.s b/llvm/test/MC/X86/LFI/branch.s new file mode 100644 index 0000000000000..0f039b43f7cf7 --- /dev/null +++ b/llvm/test/MC/X86/LFI/branch.s @@ -0,0 +1,36 @@ +// RUN: llvm-mc -filetype asm -triple x86_64_lfi %s | FileCheck %s + +callq foo +// CHECK: callq foo +// CHECK-NEXT: .p2align 5 + +callq *%rax +// CHECK: .bundle_lock +// CHECK-NEXT: andl $-32, %eax +// CHECK-NEXT: addq %r14, %rax +// CHECK-NEXT: callq *%rax +// CHECK-NEXT: .bundle_unlock + +callq *(%rax) +// CHECK: movq %gs:(%eax), %r11 +// CHECK-NEXT: .bundle_lock +// CHECK-NEXT: andl $-32, %r11d +// CHECK-NEXT: addq %r14, %r11 +// CHECK-NEXT: callq *%r11 +// CHECK-NEXT: .bundle_unlock +// CHECK-NEXT: .p2align 5 + +jmpq *%rax +// CHECK: .bundle_lock +// CHECK-NEXT: andl $-32, %eax +// CHECK-NEXT: addq %r14, %rax +// CHECK-NEXT: jmpq *%rax +// CHECK-NEXT: .bundle_unlock + +jmpq *(%rax) +// CHECK: movq %gs:(%eax), %r11 +// CHECK-NEXT: .bundle_lock +// CHECK-NEXT: andl $-32, %r11d +// CHECK-NEXT: addq %r14, %r11 +// CHECK-NEXT: jmpq *%r11 +// CHECK-NEXT: .bundle_unlock diff --git a/llvm/test/MC/X86/LFI/mem.s b/llvm/test/MC/X86/LFI/mem.s new file mode 100644 index 0000000000000..b4ed56dfd4c0c --- /dev/null +++ b/llvm/test/MC/X86/LFI/mem.s @@ -0,0 +1,10 @@ +// RUN: llvm-mc -filetype asm -triple x86_64_lfi %s | FileCheck %s + +movq (%rax), %rdi +// CHECK: movq %gs:(%eax), %rdi + +movq %rcx, (%rax) +// CHECK: movq %rcx, %gs:(%eax) + +movq %rcx, (%rax, %rdi) +// CHECK: movq %rcx, %gs:(%eax,%edi) diff --git a/llvm/test/MC/X86/LFI/return.s b/llvm/test/MC/X86/LFI/return.s new file mode 100644 index 0000000000000..1920f1cf489fe --- /dev/null +++ b/llvm/test/MC/X86/LFI/return.s @@ -0,0 +1,9 @@ +// RUN: llvm-mc -filetype asm -triple x86_64_lfi %s | FileCheck %s + +ret +// CHECK: popq %r11 +// CHECK-NEXT: .bundle_lock +// CHECK-NEXT: andl $-32, %r11d +// CHECK-NEXT: addq %r14, %r11 +// CHECK-NEXT: jmpq *%r11 +// CHECK-NEXT: .bundle_unlock diff --git a/llvm/test/MC/X86/LFI/stack.s b/llvm/test/MC/X86/LFI/stack.s new file mode 100644 index 0000000000000..98d5f05b1a7b7 --- /dev/null +++ b/llvm/test/MC/X86/LFI/stack.s @@ -0,0 +1,19 @@ +// RUN: llvm-mc -filetype asm -triple x86_64_lfi %s | FileCheck %s + +movq %rdi, %rsp +// CHECK: .bundle_lock +// CHECK-NEXT: movl %edi, %esp +// CHECK-NEXT: leaq (%rsp,%r14), %rsp +// CHECK-NEXT: .bundle_unlock + +addq %rax, %rsp +// CHECK: .bundle_lock +// CHECK-NEXT: addl %eax, %esp +// CHECK-NEXT: leaq (%rsp,%r14), %rsp +// CHECK-NEXT: .bundle_unlock + +addq $8, %rsp +// CHECK: .bundle_lock +// CHECK-NEXT: addl $8, %esp +// CHECK-NEXT: leaq (%rsp,%r14), %rsp +// CHECK-NEXT: .bundle_unlock diff --git a/llvm/test/MC/X86/LFI/string.s b/llvm/test/MC/X86/LFI/string.s new file mode 100644 index 0000000000000..beef15ef61a3e --- /dev/null +++ b/llvm/test/MC/X86/LFI/string.s @@ -0,0 +1,17 @@ +// RUN: llvm-mc -filetype asm -triple x86_64_lfi %s | FileCheck %s + +rep stosq +// CHECK: .bundle_lock +// CHECK-NEXT: movl %edi, %edi +// CHECK-NEXT: leaq (%r14,%rdi), %rdi +// CHECK-NEXT: rep stosq %rax, %es:(%rdi) +// CHECK-NEXT: .bundle_unlock + +rep movsq +// CHECK: .bundle_lock +// CHECK-NEXT: movl %esi, %esi +// CHECK-NEXT: leaq (%r14,%rsi), %rsi +// CHECK-NEXT: movl %edi, %edi +// CHECK-NEXT: leaq (%r14,%rdi), %rdi +// CHECK-NEXT: rep movsq (%rsi), %es:(%rdi) +// CHECK-NEXT: .bundle_unlock diff --git a/llvm/test/MC/X86/LFI/sys.s b/llvm/test/MC/X86/LFI/sys.s new file mode 100644 index 0000000000000..4cd495196cb9f --- /dev/null +++ b/llvm/test/MC/X86/LFI/sys.s @@ -0,0 +1,24 @@ +// RUN: llvm-mc -filetype asm -triple x86_64_lfi %s | FileCheck %s + +syscall +// CHECK: .bundle_lock +// CHECK-NEXT: leaq .Ltmp0(%rip), %r11 +// CHECK-NEXT: jmpq *(%r14) +// CHECK-NEXT: .Ltmp0: +// CHECK-NEXT: .bundle_unlock + +movq %fs:0, %rax +// CHECK: .bundle_lock +// CHECK-NEXT: leaq .Ltmp1(%rip), %r11 +// CHECK-NEXT: jmpq *8(%r14) +// CHECK-NEXT: .Ltmp1: +// CHECK-NEXT: .bundle_unlock + +movq %fs:0, %rdi +// CHECK: movq %rax, %rdi +// CHECK-NEXT: .bundle_lock +// CHECK-NEXT: leaq .Ltmp2(%rip), %r11 +// CHECK-NEXT: jmpq *8(%r14) +// CHECK-NEXT: .Ltmp2: +// CHECK-NEXT: .bundle_unlock +// CHECK-NEXT: xchgq %rax, %rdi diff --git a/llvm/tools/llvm-mc/llvm-mc.cpp b/llvm/tools/llvm-mc/llvm-mc.cpp index 70f92d09aded7..6de991c6d1bba 100644 --- a/llvm/tools/llvm-mc/llvm-mc.cpp +++ b/llvm/tools/llvm-mc/llvm-mc.cpp @@ -18,6 +18,7 @@ #include "llvm/MC/MCContext.h" #include "llvm/MC/MCInstPrinter.h" #include "llvm/MC/MCInstrInfo.h" +#include "llvm/MC/MCLFI.h" #include "llvm/MC/MCObjectFileInfo.h" #include "llvm/MC/MCObjectWriter.h" #include "llvm/MC/MCParser/AsmLexer.h" @@ -551,6 +552,12 @@ int main(int argc, char **argv) { Str.reset(TheTarget->createAsmStreamer(Ctx, std::move(FOut), IP, std::move(CE), std::move(MAB))); + Triple T(TripleName); + if (T.isLFI()) { + Str->initSections(NoExecStack, *STI); + initializeLFIMCStreamer(*Str.get(), Ctx, T); + } + } else if (FileType == OFT_Null) { Str.reset(TheTarget->createNullStreamer(Ctx)); } else { @@ -570,6 +577,13 @@ int main(int argc, char **argv) { std::unique_ptr(CE), *STI)); if (NoExecStack) Str->switchSection(Ctx.getAsmInfo()->getNonexecutableStackSection(Ctx)); + + Triple T(TripleName); + if (T.isLFI()) { + Str->initSections(NoExecStack, *STI); + initializeLFIMCStreamer(*Str.get(), Ctx, T); + } + Str->emitVersionForTarget(TheTriple, VersionTuple(), nullptr, VersionTuple()); }