[pull] v2.1 from LuaJIT:v2.1

doc/running.html

@@ -299,9 +299,9 @@ <h3 id="opt_O"><tt>-O[level]</tt><br>
 <tr class="even">
 <td class="param_name">recunroll</td><td class="param_default">2</td><td class="param_desc">Min. unroll factor for true recursion</td></tr>
 <tr class="odd separate">
-<td class="param_name">sizemcode</td><td class="param_default">32</td><td class="param_desc">Size of each machine code area in KBytes (Windows: 64K)</td></tr>
+<td class="param_name">sizemcode</td><td class="param_default">64</td><td class="param_desc">Size of each machine code area in KBytes</td></tr>
 <tr class="even">
-<td class="param_name">maxmcode</td><td class="param_default">512</td><td class="param_desc">Max. total size of all machine code areas in KBytes</td></tr>
+<td class="param_name">maxmcode</td><td class="param_default">2048</td><td class="param_desc">Max. total size of all machine code areas in KBytes</td></tr>
 </table>
 <br class="flush">
 </div>

dynasm/dasm_arm64.lua

@@ -244,6 +244,10 @@ local map_cond = {
   hs = 2, lo = 3,
 }
 
+local map_bti = {
+  c = 0x40, j = 0x80, jc = 0xc0,
+}
+
 ------------------------------------------------------------------------------
 
 local parse_reg_type
@@ -475,6 +479,12 @@ local function parse_cond(expr, inv)
   return shl(bit.bxor(c, inv), 12)
 end
 
+local function parse_map(expr, map)
+  local x = map[expr]
+  if not x then werror("bad operand") end
+  return x
+end
+
 local function parse_load(params, nparams, n, op)
   if params[n+2] then werror("too many operands") end
   local scale = shr(op, 30)
@@ -823,11 +833,21 @@ map_op = {
   tbz_3  = "36000000DTBw|36000000DTBx",
   tbnz_3 = "37000000DTBw|37000000DTBx",
 
+  -- Branch Target Identification.
+  bti_1  = "d503241ft",
+
   -- ARM64e: Pointer authentication codes (PAC).
   blraaz_1  = "d63f081fNx",
+  blrabz_1  = "d63f0c1fNx",
   braa_2    = "d71f0800NDx",
+  brab_2    = "d71f0c00NDx",
   braaz_1   = "d61f081fNx",
+  brabz_1   = "d61f0c1fNx",
+  paciasp_0 = "d503233f",
   pacibsp_0 = "d503237f",
+  autiasp_0 = "d50323bf",
+  autibsp_0 = "d50323ff",
+  retaa_0   = "d65f0bff",
   retab_0   = "d65f0fff",
 
   -- Miscellaneous instructions.
@@ -996,6 +1016,8 @@ local function parse_template(params, template, nparams, pos)
       op = op + parse_cond(q, 0); n = n + 1
     elseif p == "c" then
       op = op + parse_cond(q, 1); n = n + 1
+    elseif p == "t" then
+      op = op + parse_map(q, map_bti); n = n + 1
 
     else
       assert(false)

src/Makefile

@@ -302,6 +302,9 @@ endif
 ifneq (,$(INSTALL_LJLIBD))
   TARGET_XCFLAGS+= -DLUA_LJDIR=\"$(INSTALL_LJLIBD)\"
 endif
+ifeq (,$(shell $(TARGET_CC) -o /dev/null -c -x c /dev/null -fno-strict-float-cast-overflow 2>/dev/null || echo 1))
+  TARGET_XCFLAGS+= -fno-strict-float-cast-overflow
+endif
 
 ##############################################################################
 # Target system detection.
@@ -354,6 +357,9 @@ else
   ifeq (GNU/kFreeBSD,$(TARGET_SYS))
     TARGET_XLIBS+= -ldl
   endif
+  ifeq (GNU,$(TARGET_SYS))
+    TARGET_XLIBS+= -ldl
+  endif
 endif
 endif
 endif
@@ -440,6 +446,14 @@ ifneq (,$(findstring LJ_ABI_PAUTH 1,$(TARGET_TESTARCH)))
   DASM_AFLAGS+= -D PAUTH
   TARGET_ARCH+= -DLJ_ABI_PAUTH=1
 endif
+ifneq (,$(findstring LJ_ABI_BRANCH_TRACK 1,$(TARGET_TESTARCH)))
+  DASM_AFLAGS+= -D BRANCH_TRACK
+  TARGET_ARCH+= -DLJ_ABI_BRANCH_TRACK=1
+endif
+ifneq (,$(findstring LJ_ABI_SHADOW_STACK 1,$(TARGET_TESTARCH)))
+  DASM_AFLAGS+= -D SHADOW_STACK
+  TARGET_ARCH+= -DLJ_ABI_SHADOW_STACK=1
+endif
 DASM_AFLAGS+= -D VER=$(subst LJ_ARCH_VERSION_,,$(filter LJ_ARCH_VERSION_%,$(subst LJ_ARCH_VERSION ,LJ_ARCH_VERSION_,$(TARGET_TESTARCH))))
 ifeq (Windows,$(TARGET_SYS))
   DASM_AFLAGS+= -D WIN

src/jit/bcsave.lua

@@ -465,9 +465,11 @@ typedef struct {
   mach_segment_command_64 seg;
   mach_section_64 sec;
   mach_symtab_command sym;
+} mach_obj_64;
+typedef struct {
   mach_nlist_64 sym_entry;
   uint8_t space[4096];
-} mach_obj_64;
+} mach_obj_64_tail;
 ]]
   local symname = '_'..LJBC_PREFIX..ctx.modname
   local cputype, cpusubtype = 0x01000007, 3
@@ -479,7 +481,10 @@ typedef struct {
 
   -- Create Mach-O object and fill in header.
   local o = ffi.new("mach_obj_64")
-  local mach_size = aligned(ffi.offsetof(o, "space")+#symname+2, 8)
+  local t = ffi.new("mach_obj_64_tail")
+  local ofs_bc = ffi.sizeof(o)
+  local sz_bc = aligned(#s, 8)
+  local ofs_sym = ofs_bc + sz_bc
 
   -- Fill in sections and symbols.
   o.hdr.magic = 0xfeedfacf
@@ -491,30 +496,31 @@ typedef struct {
   o.seg.cmd = 0x19
   o.seg.cmdsize = ffi.sizeof(o.seg)+ffi.sizeof(o.sec)
   o.seg.vmsize = #s
-  o.seg.fileoff = mach_size
+  o.seg.fileoff = ofs_bc
   o.seg.filesize = #s
   o.seg.maxprot = 1
   o.seg.initprot = 1
   o.seg.nsects = 1
   ffi.copy(o.sec.sectname, "__data")
   ffi.copy(o.sec.segname, "__DATA")
   o.sec.size = #s
-  o.sec.offset = mach_size
+  o.sec.offset = ofs_bc
   o.sym.cmd = 2
   o.sym.cmdsize = ffi.sizeof(o.sym)
-  o.sym.symoff = ffi.offsetof(o, "sym_entry")
+  o.sym.symoff = ofs_sym
   o.sym.nsyms = 1
-  o.sym.stroff = ffi.offsetof(o, "sym_entry")+ffi.sizeof(o.sym_entry)
+  o.sym.stroff = ofs_sym + ffi.offsetof(t, "space")
   o.sym.strsize = aligned(#symname+2, 8)
-  o.sym_entry.type = 0xf
-  o.sym_entry.sect = 1
-  o.sym_entry.strx = 1
-  ffi.copy(o.space+1, symname)
+  t.sym_entry.type = 0xf
+  t.sym_entry.sect = 1
+  t.sym_entry.strx = 1
+  ffi.copy(t.space+1, symname)
 
   -- Write Mach-O object file.
   local fp = savefile(output, "wb")
-  fp:write(ffi.string(o, mach_size))
-  bcsave_tail(fp, output, s)
+  fp:write(ffi.string(o, ofs_bc))
+  fp:write(s, ("\0"):rep(sz_bc - #s))
+  bcsave_tail(fp, output, ffi.string(t, ffi.offsetof(t, "space") + o.sym.strsize))
 end
 
 local function bcsave_obj(ctx, output, s)

src/jit/dis_arm64.lua

@@ -695,7 +695,10 @@ local map_br = { -- Branches, exception generating and system instructions.
     },
     { -- System instructions.
       shift = 0, mask = 0x3fffff,
-      [0x03201f] = "nop"
+      [0x03201f] = "nop",
+      [0x03245f] = "bti c",
+      [0x03249f] = "bti j",
+      [0x0324df] = "bti jc",
     },
     { -- Unconditional branch, register.
       shift = 0, mask = 0xfffc1f,
@@ -920,7 +923,7 @@ local function disass_ins(ctx)
     elseif p == "B" then
       local addr = ctx.addr + pos + parse_immpc(op, name)
       ctx.rel = addr
-      x = "0x"..tohex(addr)
+      x = format("0x%08x", addr)
     elseif p == "T" then
       x = bor(band(rshift(op, 26), 32), band(rshift(op, 19), 31))
     elseif p == "V" then
@@ -1171,6 +1174,9 @@ local function disass_ins(ctx)
 	end
       end
       second0 = true
+    elseif p == " " then
+      operands[#operands+1] = pat:match(" (.*)")
+      break
     else
       assert(false)
     end

src/jit/dis_x86.lua

@@ -122,7 +122,7 @@ local map_opc2 = {
 "movlhpsXrm$movhpsXrm|movshdupXrm|movhpdXrm",
 "movhpsXmr||movhpdXmr",
 "$prefetcht!Bm","hintnopVm","hintnopVm","hintnopVm",
-"hintnopVm","hintnopVm","hintnopVm","hintnopVm",
+"hintnopVm","hintnopVm","endbr*hintnopVm","hintnopVm",
 --2x
 "movUmx$","movUmy$","movUxm$","movUym$","movUmz$",nil,"movUzm$",nil,
 "movapsXrm||movapdXrm",
@@ -804,6 +804,24 @@ map_act = {
     return dispatch(ctx, map_opcvm[ctx.mrm])
   end,
 
+  -- Special NOP for endbr64/endbr32.
+  endbr = function(ctx, name, pat)
+    if ctx.rep then
+      local pos = ctx.pos
+      local b = byte(ctx.code, pos)
+      local text
+      if b == 0xfa then text = "endbr64"
+      elseif b == 0xfb then text = "endbr64"
+      end
+      if text then
+	ctx.pos = pos + 1
+	ctx.rep = nil
+	return putop(ctx, text)
+      end
+    end
+    return dispatch(ctx, pat)
+  end,
+
   -- Floating point opcode dispatch.
   fp = function(ctx, name, pat)
     local mrm = getmrm(ctx); if not mrm then return incomplete(ctx) end

src/lib_jit.c

@@ -479,12 +479,21 @@ static int jitopt_param(jit_State *J, const char *str)
     size_t len = *(const uint8_t *)lst;
     lj_assertJ(len != 0, "bad JIT_P_STRING");
     if (strncmp(str, lst+1, len) == 0 && str[len] == '=') {
-      int32_t n = 0;
+      uint32_t n = 0;
       const char *p = &str[len+1];
       while (*p >= '0' && *p <= '9')
 	n = n*10 + (*p++ - '0');
-      if (*p) return 0;  /* Malformed number. */
-      J->param[i] = n;
+      if (*p || (int32_t)n < 0) return 0;  /* Malformed number. */
+      if (i == JIT_P_sizemcode) {  /* Adjust to required range here. */
+#if LJ_TARGET_JUMPRANGE
+	uint32_t maxkb = ((1 << (LJ_TARGET_JUMPRANGE - 10)) - 64);
+#else
+	uint32_t maxkb = ((1 << (31 - 10)) - 64);
+#endif
+	n = (n + (LJ_PAGESIZE >> 10) - 1) & ~((LJ_PAGESIZE >> 10) - 1);
+	if (n > maxkb) n = maxkb;
+      }
+      J->param[i] = (int32_t)n;
       if (i == JIT_P_hotloop)
 	lj_dispatch_init_hotcount(J2G(J));
       return 1;  /* Ok. */
@@ -714,7 +723,16 @@ static void jit_init(lua_State *L)
   jit_State *J = L2J(L);
   J->flags = jit_cpudetect() | JIT_F_ON | JIT_F_OPT_DEFAULT;
   memcpy(J->param, jit_param_default, sizeof(J->param));
+#if LJ_TARGET_UNALIGNED
+  G(L)->tmptv.u64 = U64x(0000504d,4d500000);
+#endif
   lj_dispatch_update(G(L));
+#if LJ_TARGET_UNALIGNED
+  /* If you get a crash below then your toolchain indicates unaligned
+  ** accesses are OK, but your kernel disagrees. I.e. fix your toolchain.
+  */
+  if (*(uint32_t *)((char *)&G(L)->tmptv + 2) != 0x504d4d50u) L->top = NULL;
+#endif
 }
 #endif
 

src/lj_arch.h

@@ -96,6 +96,9 @@
 #elif defined(__QNX__)
 #define LJ_TARGET_QNX		1
 #define LUAJIT_OS	LUAJIT_OS_POSIX
+#elif defined(__GNU__)
+#define LJ_TARGET_HURD		1
+#define LUAJIT_OS	LUAJIT_OS_POSIX
 #else
 #define LUAJIT_OS	LUAJIT_OS_OTHER
 #endif
@@ -216,6 +219,29 @@
 #error "macOS requires GC64 -- don't disable it"
 #endif
 
+#if !defined(LJ_ABI_BRANCH_TRACK) && (__CET__ & 1) && \
+    LJ_TARGET_GC64 && defined(LUAJIT_ENABLE_CET_BR)
+/*
+** Control-Flow Enforcement Technique (CET) indirect branch tracking (IBT).
+** This is not enabled by default because it causes a notable slowdown of
+** the interpreter on all x64 CPUs, whether they have CET enabled or not.
+** If your toolchain enables -fcf-protection=branch by default, you need
+** to build with: make amalg XCFLAGS=-DLUAJIT_ENABLE_CET_BR
+*/
+#define LJ_ABI_BRANCH_TRACK	1
+#endif
+
+#if !defined(LJ_ABI_SHADOW_STACK) && (__CET__ & 2)
+/*
+** Control-Flow Enforcement Technique (CET) shadow stack (CET-SS).
+** It has no code overhead and doesn't cause any slowdowns when unused.
+** It can also be unconditionally enabled since all code already follows
+** a strict CALL to RET correspondence for performance reasons (all modern
+** CPUs use a (non-enforcing) shadow stack for return branch prediction).
+*/
+#define LJ_ABI_SHADOW_STACK	1
+#endif
+
 #elif LUAJIT_TARGET == LUAJIT_ARCH_ARM
 
 #define LJ_ARCH_NAME		"arm"
@@ -262,6 +288,11 @@
 #if !defined(LJ_ABI_PAUTH) && defined(__arm64e__)
 #define LJ_ABI_PAUTH		1
 #endif
+#if !defined(LJ_ABI_BRANCH_TRACK) && (__ARM_FEATURE_BTI_DEFAULT & 1) && \
+    defined(LUAJIT_ENABLE_CET_BR)
+/* See comments about LUAJIT_ENABLE_CET_BR above. */
+#define LJ_ABI_BRANCH_TRACK	1
+#endif
 #define LJ_TARGET_ARM64		1
 #define LJ_TARGET_EHRETREG	0
 #define LJ_TARGET_EHRAREG	30
@@ -270,8 +301,13 @@
 #define LJ_TARGET_MASKROT	1
 #define LJ_TARGET_UNIFYROT	2	/* Want only IR_BROR. */
 #define LJ_TARGET_GC64		1
+#define LJ_PAGESIZE		16384
 #define LJ_ARCH_NUMMODE		LJ_NUMMODE_DUAL
 
+#if __ARM_FEATURE_UNALIGNED
+#define LJ_TARGET_UNALIGNED	1
+#endif
+
 #define LJ_ARCH_VERSION		80
 
 #elif LUAJIT_TARGET == LUAJIT_ARCH_PPC
@@ -425,7 +461,7 @@
 #define LJ_TARGET_MIPS		1
 #define LJ_TARGET_EHRETREG	4
 #define LJ_TARGET_EHRAREG	31
-#define LJ_TARGET_JUMPRANGE	27	/* 2*2^27 = 256MB-aligned region */
+#define LJ_TARGET_JUMPRANGE	28	/* 2^28 = 256MB-aligned region */
 #define LJ_TARGET_MASKSHIFT	1
 #define LJ_TARGET_MASKROT	1
 #define LJ_TARGET_UNIFYROT	2	/* Want only IR_BROR. */

src/lj_asm.c

@@ -93,6 +93,10 @@ typedef struct ASMState {
   MCode *invmcp;	/* Points to invertible loop branch (or NULL). */
   MCode *flagmcp;	/* Pending opportunity to merge flag setting ins. */
   MCode *realign;	/* Realign loop if not NULL. */
+  MCode *mctail;	/* Tail of trace before stack adjust + jmp. */
+#if LJ_TARGET_PPC || LJ_TARGET_ARM64
+  MCode *mcexit;	/* Pointer to exit stubs. */
+#endif
 
 #ifdef LUAJIT_RANDOM_RA
   /* Randomize register allocation. OK for fuzz testing, not for production. */
@@ -2541,7 +2545,7 @@ void lj_asm_trace(jit_State *J, GCtrace *T)
     RA_DBGX((as, "===== STOP ====="));
 
     /* General trace setup. Emit tail of trace. */
-    asm_tail_prep(as);
+    asm_tail_prep(as, T->link);
     as->mcloop = NULL;
     as->flagmcp = NULL;
     as->topslot = 0;
@@ -2586,6 +2590,9 @@ void lj_asm_trace(jit_State *J, GCtrace *T)
       asm_head_side(as);
     else
       asm_head_root(as);
+#if LJ_ABI_BRANCH_TRACK
+    emit_branch_track(as);
+#endif
     asm_phi_fixup(as);
 
     if (J->curfinal->nins >= T->nins) {  /* IR didn't grow? */