Compile table traversals: next(), pairs(), BC_ISNEXT/BC_ITERN.

Sponsored by OpenResty Inc.
loongson-community · Sep 19, 2021 · bb0f241 · bb0f241
1 parent 986bb40
commit bb0f241
Show file tree

Hide file tree

Showing 27 changed files with 781 additions and 47 deletions.
diff --git a/src/jit/dump.lua b/src/jit/dump.lua
@@ -219,8  219,10 @@ local function colorize_text(s)
   return s
 end
 
-local function colorize_ansi(s, t)
-  return format(colortype_ansi[t], s)
 local function colorize_ansi(s, t, extra)
   local out = format(colortype_ansi[t], s)
   if extra then out = "\027[3m"..out end
   return out
 end
 
 local irtype_ansi = setmetatable({},
@@ -229,9  231,10 @@ local irtype_ansi = setmetatable({},
 
 local html_escape = { ["<"] = "&lt;", [">"] = "&gt;", ["&"] = "&amp;", }
 
-local function colorize_html(s, t)
 local function colorize_html(s, t, extra)
   s = gsub(s, "[<>&]", html_escape)
-  return format('<span class="irt_%s">%s</span>', irtype_text[t], s)
   return format('<span class="irt_%s%s">%s</span>',
 		irtype_text[t], extra and " irt_extra" or "", s)
 end
 
 local irtype_html = setmetatable({},
@@ -256,6  259,7 @@ span.irt_tab { color: #c00000; }
 span.irt_udt, span.irt_lud { color: #00c0c0; }
 span.irt_num { color: #4040c0; }
 span.irt_int, span.irt_i8, span.irt_u8, span.irt_i16, span.irt_u16 { color: #b040b0; }
 span.irt_extra { font-style: italic; }
 </style>
 ]]
 
@@ -271,6  275,7 @@ local litname = {
     if band(mode, 8) ~= 0 then s = s.."C" end
     if band(mode, 16) ~= 0 then s = s.."R" end
     if band(mode, 32) ~= 0 then s = s.."I" end
     if band(mode, 64) ~= 0 then s = s.."K" end
     t[mode] = s
     return s
   end}),
@@ -350,7  355,7 @@ local function formatk(tr, idx, sn)
   else
     s = tostring(k) -- For primitives.
   end
-  s = colorize(format("%-4s", s), t)
   s = colorize(format("%-4s", s), t, band(sn or 0, 0x100000) ~= 0)
   if slot then
     s = format("%s @%d", s, slot)
   end
@@ -370,7  375,7 @@ local function printsnap(tr, snap)
 	out:write(colorize(format("d/d", ref, ref 1), 14))
       else
 	local m, ot, op1, op2 = traceir(tr, ref)
-	out:write(colorize(format("d", ref), band(ot, 31)))
 	out:write(colorize(format("d", ref), band(ot, 31), band(sn, 0x100000) ~= 0))
       end
       out:write(band(sn, 0x10000) == 0 and " " or "|") -- SNAP_FRAME
     else

diff --git a/src/lib_base.c b/src/lib_base.c
@@ -76,7  76,7 @@ LJLIB_ASM_(type)		LJLIB_REC(.)
 /* This solves a circular dependency problem -- change FF_next_N as needed. */
 LJ_STATIC_ASSERT((int)FF_next == FF_next_N);
 
-LJLIB_ASM(next)
 LJLIB_ASM(next)			LJLIB_REC(.)
 {
   lj_lib_checktab(L, 1);
   lj_err_msg(L, LJ_ERR_NEXTIDX);

diff --git a/src/lj_asm.c b/src/lj_asm.c
@@ -2225,7  2225,17 @@ static void asm_setup_regsp(ASMState *as)
 	as->modset |= RSET_SCRATCH;
       continue;
       }
-    case IR_CALLN: case IR_CALLA: case IR_CALLL: case IR_CALLS: {
     case IR_CALLL:
       /* lj_vm_next needs two TValues on the stack. */
 #if LJ_TARGET_X64 && LJ_ABI_WIN
       if (ir->op2 == IRCALL_lj_vm_next && as->evenspill < SPS_FIRST   4)
 	as->evenspill = SPS_FIRST   4;
 #else
       if (SPS_FIRST < 4 && ir->op2 == IRCALL_lj_vm_next && as->evenspill < 4)
 	as->evenspill = 4;
 #endif
       /* fallthrough */
     case IR_CALLN: case IR_CALLA: case IR_CALLS: {
       const CCallInfo *ci = &lj_ir_callinfo[ir->op2];
       ir->prev = asm_setup_call_slots(as, ir, ci);
       if (inloop)

diff --git a/src/lj_asm_arm.h b/src/lj_asm_arm.h
@@ -2064,6  2064,8 @@ static void asm_stack_restore(ASMState *as, SnapShot *snap)
       } else if ((sn & SNAP_SOFTFPNUM)) {
 	type = ra_alloc1(as, ref 1, rset_exclude(RSET_GPRODD, RID_BASE));
 #endif
       } else if ((sn & SNAP_KEYINDEX)) {
 	type = ra_allock(as, (int32_t)LJ_KEYINDEX, odd);
       } else {
 	type = ra_allock(as, (int32_t)irt_toitype(ir->t), odd);
       }

diff --git a/src/lj_asm_arm64.h b/src/lj_asm_arm64.h
@@ -1814,7  1814,14 @@ static void asm_stack_restore(ASMState *as, SnapShot *snap)
     IRIns *ir = IR(ref);
     if ((sn & SNAP_NORESTORE))
       continue;
-    if (irt_isnum(ir->t)) {
     if ((sn & SNAP_KEYINDEX)) {
       RegSet allow = rset_exclude(RSET_GPR, RID_BASE);
       Reg r = irref_isk(ref) ? ra_allock(as, ir->i, allow) :
 			       ra_alloc1(as, ref, allow);
       rset_clear(allow, r);
       emit_lso(as, A64I_STRw, r, RID_BASE, ofs);
       emit_lso(as, A64I_STRw, ra_allock(as, LJ_KEYINDEX, allow), RID_BASE, ofs 4);
     } else if (irt_isnum(ir->t)) {
       Reg src = ra_alloc1(as, ref, RSET_FPR);
       emit_lso(as, A64I_STRd, (src & 31), RID_BASE, ofs);
     } else {

diff --git a/src/lj_asm_mips.h b/src/lj_asm_mips.h
@@ -2568,6  2568,8 @@ static void asm_stack_restore(ASMState *as, SnapShot *snap)
       } else if ((sn & SNAP_SOFTFPNUM)) {
 	type = ra_alloc1(as, ref 1, rset_exclude(RSET_GPR, RID_BASE));
 #endif
       } else if ((sn & SNAP_KEYINDEX)) {
 	type = ra_allock(as, (int32_t)LJ_KEYINDEX, allow);
       } else {
 	type = ra_allock(as, (int32_t)irt_toitype(ir->t), allow);
       }

diff --git a/src/lj_asm_ppc.h b/src/lj_asm_ppc.h
@@ -1103,7  1103,8 @@ static void asm_sload(ASMState *as, IRIns *ir)
   lj_assertA(irt_isguard(ir->t) || !(ir->op2 & IRSLOAD_TYPECHECK),
 	     "inconsistent SLOAD variant");
   lj_assertA(LJ_DUALNUM ||
-	     !irt_isint(t) || (ir->op2 & (IRSLOAD_CONVERT|IRSLOAD_FRAME)),
 	     !irt_isint(t) ||
 	     (ir->op2 & (IRSLOAD_CONVERT|IRSLOAD_FRAME|IRSLOAD_KEYINDEX)),
 	     "bad SLOAD type");
 #if LJ_SOFTFP
   lj_assertA(!(ir->op2 & IRSLOAD_CONVERT),
@@ -2096,6  2097,8 @@ static void asm_stack_restore(ASMState *as, SnapShot *snap)
       } else if ((sn & SNAP_SOFTFPNUM)) {
 	type = ra_alloc1(as, ref 1, rset_exclude(RSET_GPR, RID_BASE));
 #endif
       } else if ((sn & SNAP_KEYINDEX)) {
 	type = ra_allock(as, (int32_t)LJ_KEYINDEX, allow);
       } else {
 	type = ra_allock(as, (int32_t)irt_toitype(ir->t), allow);
       }

diff --git a/src/lj_asm_x86.h b/src/lj_asm_x86.h
@@ -1700,7  1700,8 @@ static void asm_sload(ASMState *as, IRIns *ir)
   lj_assertA(irt_isguard(t) || !(ir->op2 & IRSLOAD_TYPECHECK),
 	     "inconsistent SLOAD variant");
   lj_assertA(LJ_DUALNUM ||
-	     !irt_isint(t) || (ir->op2 & (IRSLOAD_CONVERT|IRSLOAD_FRAME)),
 	     !irt_isint(t) ||
 	     (ir->op2 & (IRSLOAD_CONVERT|IRSLOAD_FRAME|IRSLOAD_KEYINDEX)),
 	     "bad SLOAD type");
   if ((ir->op2 & IRSLOAD_CONVERT) && irt_isguard(t) && irt_isint(t)) {
     Reg left = ra_scratch(as, RSET_FPR);
@@ -2727,7  2728,15 @@ static void asm_stack_restore(ASMState *as, SnapShot *snap)
     IRIns *ir = IR(ref);
     if ((sn & SNAP_NORESTORE))
       continue;
-    if (irt_isnum(ir->t)) {
     if ((sn & SNAP_KEYINDEX)) {
       emit_movmroi(as, RID_BASE, ofs 4, LJ_KEYINDEX);
       if (irref_isk(ref)) {
 	emit_movmroi(as, RID_BASE, ofs, ir->i);
       } else {
 	Reg src = ra_alloc1(as, ref, rset_exclude(RSET_GPR, RID_BASE));
 	emit_movtomro(as, src, RID_BASE, ofs);
       }
     } else if (irt_isnum(ir->t)) {
       Reg src = ra_alloc1(as, ref, RSET_FPR);
       emit_rmro(as, XO_MOVSDto, src, RID_BASE, ofs);
     } else {

diff --git a/src/lj_dispatch.c b/src/lj_dispatch.c
@@ -68,6  68,8 @@ void lj_dispatch_init(GG_State *GG)
   /* The JIT engine is off by default. luaopen_jit() turns it on. */
   disp[BC_FORL] = disp[BC_IFORL];
   disp[BC_ITERL] = disp[BC_IITERL];
   /* Workaround for stable v2.1 bytecode. TODO: Replace with BC_IITERN. */
   disp[BC_ITERN] = &lj_vm_IITERN;
   disp[BC_LOOP] = disp[BC_ILOOP];
   disp[BC_FUNCF] = disp[BC_IFUNCF];
   disp[BC_FUNCV] = disp[BC_IFUNCV];
@@ -118,26  120,29 @@ void lj_dispatch_update(global_State *g)
   mode |= (g->hookmask & LUA_MASKRET) ? DISPMODE_RET : 0;
   if (oldmode != mode) {  /* Mode changed? */
     ASMFunction *disp = G2GG(g)->dispatch;
-    ASMFunction f_forl, f_iterl, f_loop, f_funcf, f_funcv;
     ASMFunction f_forl, f_iterl, f_itern, f_loop, f_funcf, f_funcv;
     g->dispatchmode = mode;
 
     /* Hotcount if JIT is on, but not while recording. */
     if ((mode & (DISPMODE_JIT|DISPMODE_REC)) == DISPMODE_JIT) {
       f_forl = makeasmfunc(lj_bc_ofs[BC_FORL]);
       f_iterl = makeasmfunc(lj_bc_ofs[BC_ITERL]);
       f_itern = makeasmfunc(lj_bc_ofs[BC_ITERN]);
       f_loop = makeasmfunc(lj_bc_ofs[BC_LOOP]);
       f_funcf = makeasmfunc(lj_bc_ofs[BC_FUNCF]);
       f_funcv = makeasmfunc(lj_bc_ofs[BC_FUNCV]);
     } else {  /* Otherwise use the non-hotcounting instructions. */
       f_forl = disp[GG_LEN_DDISP BC_IFORL];
       f_iterl = disp[GG_LEN_DDISP BC_IITERL];
       f_itern = &lj_vm_IITERN;
       f_loop = disp[GG_LEN_DDISP BC_ILOOP];
       f_funcf = makeasmfunc(lj_bc_ofs[BC_IFUNCF]);
       f_funcv = makeasmfunc(lj_bc_ofs[BC_IFUNCV]);
     }
     /* Init static counting instruction dispatch first (may be copied below). */
     disp[GG_LEN_DDISP BC_FORL] = f_forl;
     disp[GG_LEN_DDISP BC_ITERL] = f_iterl;
     disp[GG_LEN_DDISP BC_ITERN] = f_itern;
     disp[GG_LEN_DDISP BC_LOOP] = f_loop;
 
     /* Set dynamic instruction dispatch. */
@@ -165,6  170,7 @@ void lj_dispatch_update(global_State *g)
       /* Otherwise set dynamic counting ins. */
       disp[BC_FORL] = f_forl;
       disp[BC_ITERL] = f_iterl;
       disp[BC_ITERN] = f_itern;
       disp[BC_LOOP] = f_loop;
       /* Set dynamic return dispatch. */
       if ((mode & DISPMODE_RET)) {

diff --git a/src/lj_ffrecord.c b/src/lj_ffrecord.c
@@ -521,6  521,40 @@ static void LJ_FASTCALL recff_getfenv(jit_State *J, RecordFFData *rd)
   recff_nyiu(J, rd);
 }
 
 static void LJ_FASTCALL recff_next(jit_State *J, RecordFFData *rd)
 {
 #if LJ_BE
   /* YAGNI: Disabled on big-endian due to issues with lj_vm_next,
   ** IR_HIOP, RID_RETLO/RID_RETHI and ra_destpair.
   */
   recff_nyi(J, rd);
 #else
   TRef tab = J->base[0];
   if (tref_istab(tab)) {
     RecordIndex ix;
     cTValue *keyv;
     ix.tab = tab;
     if (tref_isnil(J->base[1])) {  /* Shortcut for start of traversal. */
       ix.key = lj_ir_kint(J, 0);
       keyv = niltvg(J2G(J));
     } else {
       TRef tmp = recff_tmpref(J, J->base[1], IRTMPREF_IN1);
       ix.key = lj_ir_call(J, IRCALL_lj_tab_keyindex, tab, tmp);
       keyv = &rd->argv[1];
     }
     copyTV(J->L, &ix.tabv, &rd->argv[0]);
     ix.keyv.u32.lo = lj_tab_keyindex(tabV(&ix.tabv), keyv);
     /* Omit the value, if not used by the caller. */
     ix.idxchain = (J->framedepth && frame_islua(J->L->base-1) &&
 		   bc_b(frame_pc(J->L->base-1)[-1]) <= 2);
     ix.mobj = 0;  /* We don't need the next index. */
     rd->nres = lj_record_next(J, &ix);
     J->base[0] = ix.key;
     J->base[1] = ix.val;
   }  /* else: Interpreter will throw. */
 #endif
 }
 
 /* -- Math library fast functions ----------------------------------------- */
 
 static void LJ_FASTCALL recff_math_abs(jit_State *J, RecordFFData *rd)

diff --git a/src/lj_ir.h b/src/lj_ir.h
@@ -236,6  236,7 @@ IRFLDEF(FLENUM)
 #define IRSLOAD_CONVERT		0x08	/* Number to integer conversion. */
 #define IRSLOAD_READONLY	0x10	/* Read-only, omit slot store. */
 #define IRSLOAD_INHERIT		0x20	/* Inherited by exits/side traces. */
 #define IRSLOAD_KEYINDEX	0x40	/* Table traversal key index. */
 
 /* XLOAD mode bits, stored in op2. */
 #define IRXLOAD_READONLY	0x01	/* Load from read-only data. */
@@ -495,6  496,7 @@ typedef uint32_t TRef;
 #define TREF_REFMASK		0x0000ffff
 #define TREF_FRAME		0x00010000
 #define TREF_CONT		0x00020000
 #define TREF_KEYINDEX		0x00100000
 
 #define TREF(ref, t)		((TRef)((ref)   ((t)<<24)))
 

diff --git a/src/lj_ircall.h b/src/lj_ircall.h
@@ -187,6  187,8 @@ typedef struct CCallInfo {
   _(ANY,	lj_tab_dup,		2,  FA, TAB, CCI_L|CCI_T) \
   _(ANY,	lj_tab_clear,		1,  FS, NIL, 0) \
   _(ANY,	lj_tab_newkey,		3,   S, PGC, CCI_L|CCI_T) \
   _(ANY,	lj_tab_keyindex,	2,  FL, INT, 0) \
   _(ANY,	lj_vm_next,		2,  FL, PTR, 0) \
   _(ANY,	lj_tab_len,		1,  FL, INT, 0) \
   _(ANY,	lj_tab_len_hint,	2,  FL, INT, 0) \
   _(ANY,	lj_gc_step_jit,		2,  FS, NIL, CCI_L) \

diff --git a/src/lj_jit.h b/src/lj_jit.h
@@ -150,6  150,7 @@ typedef enum {
   LJ_TRACE_IDLE,	/* Trace compiler idle. */
   LJ_TRACE_ACTIVE = 0x10,
   LJ_TRACE_RECORD,	/* Bytecode recording active. */
   LJ_TRACE_RECORD_1ST,	/* Record 1st instruction, too. */
   LJ_TRACE_START,	/* New trace started. */
   LJ_TRACE_END,		/* End of trace. */
   LJ_TRACE_ASM,		/* Assemble trace. */
@@ -200,12  201,15 @@ typedef uint32_t SnapEntry;
 #define SNAP_CONT		0x020000	/* Continuation slot. */
 #define SNAP_NORESTORE		0x040000	/* No need to restore slot. */
 #define SNAP_SOFTFPNUM		0x080000	/* Soft-float number. */
 #define SNAP_KEYINDEX		0x100000	/* Traversal key index. */
 LJ_STATIC_ASSERT(SNAP_FRAME == TREF_FRAME);
 LJ_STATIC_ASSERT(SNAP_CONT == TREF_CONT);
 LJ_STATIC_ASSERT(SNAP_KEYINDEX == TREF_KEYINDEX);
 
 #define SNAP(slot, flags, ref)	(((SnapEntry)(slot) << 24)   (flags)   (ref))
 #define SNAP_TR(slot, tr) \
-  (((SnapEntry)(slot) << 24)   ((tr) & (TREF_CONT|TREF_FRAME|TREF_REFMASK)))
   (((SnapEntry)(slot) << 24)   \
    ((tr) & (TREF_KEYINDEX|TREF_CONT|TREF_FRAME|TREF_REFMASK)))
 #if !LJ_FR2
 #define SNAP_MKPC(pc)		((SnapEntry)u32ptr(pc))
 #endif

diff --git a/src/lj_opt_fold.c b/src/lj_opt_fold.c
@@ -2320,6  2320,15 @@ LJFOLDF(fload_sbuf)
   return lj_opt_fwd_sbuf(J, tref_ref(tr)) ? tr : EMITFOLD;
 }
 
 /* The fast function ID of function objects is immutable. */
 LJFOLD(FLOAD KGC IRFL_FUNC_FFID)
 LJFOLDF(fload_func_ffid_kgc)
 {
   if (LJ_LIKELY(J->flags & JIT_F_OPT_FOLD))
     return INTFOLD((int32_t)ir_kfunc(fleft)->c.ffid);
   return NEXTFOLD;
 }
 
 /* The C type ID of cdata objects is immutable. */
 LJFOLD(FLOAD KGC IRFL_CDATA_CTYPEID)
 LJFOLDF(fload_cdata_typeid_kgc)

diff --git a/src/lj_opt_mem.c b/src/lj_opt_mem.c
@@ -364,7  364,10 @@ TRef LJ_FASTCALL lj_opt_dse_ahstore(jit_State *J)
       /* Different value: try to eliminate the redundant store. */
       if (ref > J->chain[IR_LOOP]) {  /* Quick check to avoid crossing LOOP. */
 	IRIns *ir;
-	/* Check for any intervening guards (includes conflicting loads). */
 	/* Check for any intervening guards (includes conflicting loads).
 	** Note that lj_tab_keyindex and lj_vm_next don't need guards,
 	** since they are followed by at least one guarded VLOAD.
 	*/
 	for (ir = IR(J->cur.nins-1); ir > store; ir--)
 	  if (irt_isguard(ir->t) || ir->o == IR_ALEN)
 	    goto doemit;  /* No elimination possible. */