Skip to content

Commit ee9da47

Browse files
ktockktock
committed
tcg/wasm: Enable TLB lookup
This commit enables Wasm module's qemu_ld and qemu_st to perform TLB lookups, following the approach used in other backends such as RISC-V. Unlike other backends, the Wasm backend cannot use ldst labels, as jumping to specific code addresses (e.g. raddr) is not possible in Wasm. Instead, each TLB lookup is followed by a if branch: if the lookup succeeds, the memory is accessed directly; otherwise, a fallback helper function is invoked. Support for MO_BSWAP is not yet implemented, so has_memory_bswap is set to false. Signed-off-by: Kohei Tokunaga <ktokunaga.mail@gmail.com>
1 parent 9051ee1 commit ee9da47

File tree

1 file changed

+222
-3
lines changed

1 file changed

+222
-3
lines changed

‎tcg/wasm/tcg-target.c.inc‎

Lines changed: 222 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,12 @@
33
* Tiny Code Generator for QEMU
44
*
55
* Copyright (c) 2009, 2011 Stefan Weil
6+
* Copyright (c) 2018 SiFive, Inc
7+
* Copyright (c) 2008-2009 Arnaud Patard <arnaud.patard@rtp-net.org>
8+
* Copyright (c) 2009 Aurelien Jarno <aurelien@aurel32.net>
9+
* Copyright (c) 2008 Fabrice Bellard
610
*
7-
* Based on tci/tcg-target.c.inc
11+
* Based on tci/tcg-target.c.inc and riscv/tcg-target.c.inc
812
*
913
* Permission is hereby granted, free of charge, to any person obtaining a copy
1014
* of this software and associated documentation files (the "Software"), to deal
@@ -154,6 +158,11 @@ static const uint8_t tcg_target_reg_index[TCG_TARGET_NB_REGS] = {
154158
/* Local variable pointing to WasmContext */
155159
#define CTX_IDX 0
156160

161+
/* Temporary local variables */
162+
#define TMP32_LOCAL_0_IDX 1
163+
#define TMP64_LOCAL_0_IDX 2
164+
#define TMP64_LOCAL_1_IDX 3
165+
157166
/* Function index */
158167
#define CHECK_UNWINDING_IDX 0 /* A function to check the Asyncify status */
159168
#define HELPER_IDX_START 1 /* The first index of helper functions */
@@ -170,6 +179,8 @@ typedef enum {
170179
OPC_RETURN = 0x0f,
171180
OPC_CALL = 0x10,
172181
OPC_LOCAL_GET = 0x20,
182+
OPC_LOCAL_SET = 0x21,
183+
OPC_LOCAL_TEE = 0x22,
173184
OPC_GLOBAL_GET = 0x23,
174185
OPC_GLOBAL_SET = 0x24,
175186

@@ -1217,11 +1228,156 @@ static void *qemu_ld_helper_ptr(uint32_t oi)
12171228
}
12181229
}
12191230

1231+
#define MIN_TLB_MASK_TABLE_OFS INT_MIN
1232+
1233+
static uint8_t prepare_host_addr_wasm(TCGContext *s, uint8_t *hit_var,
1234+
TCGReg addr_reg, MemOpIdx oi,
1235+
bool is_ld)
1236+
{
1237+
MemOp opc = get_memop(oi);
1238+
TCGAtomAlign aa;
1239+
unsigned a_mask;
1240+
unsigned s_bits = opc & MO_SIZE;
1241+
unsigned s_mask = (1u << s_bits) - 1;
1242+
int mem_index = get_mmuidx(oi);
1243+
int fast_ofs = tlb_mask_table_ofs(s, mem_index);
1244+
int mask_ofs = fast_ofs + offsetof(CPUTLBDescFast, mask);
1245+
int table_ofs = fast_ofs + offsetof(CPUTLBDescFast, table);
1246+
int add_off = offsetof(CPUTLBEntry, addend);
1247+
tcg_target_long compare_mask;
1248+
int offset;
1249+
1250+
uint8_t tmp1 = TMP64_LOCAL_0_IDX;
1251+
uint8_t tmp2 = TMP64_LOCAL_1_IDX;
1252+
1253+
if (!tcg_use_softmmu) {
1254+
g_assert_not_reached();
1255+
}
1256+
1257+
*hit_var = TMP32_LOCAL_0_IDX;
1258+
tcg_wasm_out_op_const(s, OPC_I32_CONST, 0);
1259+
tcg_wasm_out_op_idx(s, OPC_LOCAL_SET, *hit_var);
1260+
1261+
aa = atom_and_align_for_opc(s, opc, MO_ATOM_IFALIGN, false);
1262+
a_mask = (1u << aa.align) - 1;
1263+
1264+
/* Get the CPUTLBEntry offset */
1265+
tcg_wasm_out_op_idx(s, OPC_GLOBAL_GET, REG_IDX(addr_reg));
1266+
tcg_wasm_out_op_const(s, OPC_I64_CONST,
1267+
TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS);
1268+
tcg_wasm_out_op(s, OPC_I64_SHR_U);
1269+
1270+
tcg_wasm_out_op_idx(s, OPC_GLOBAL_GET, REG_IDX(TCG_AREG0));
1271+
offset = tcg_wasm_out_norm_ptr(s, mask_ofs);
1272+
tcg_wasm_out_op_ldst(s, OPC_I64_LOAD, 0, offset);
1273+
tcg_wasm_out_op(s, OPC_I64_AND);
1274+
1275+
/* Get the pointer to the target CPUTLBEntry */
1276+
tcg_wasm_out_op_idx(s, OPC_GLOBAL_GET, REG_IDX(TCG_AREG0));
1277+
offset = tcg_wasm_out_norm_ptr(s, table_ofs);
1278+
tcg_wasm_out_op_ldst(s, OPC_I64_LOAD, 0, offset);
1279+
tcg_wasm_out_op(s, OPC_I64_ADD);
1280+
tcg_wasm_out_op_idx(s, OPC_LOCAL_TEE, tmp1);
1281+
1282+
/* Load the tlb copmarator */
1283+
offset = tcg_wasm_out_norm_ptr(s, is_ld ? offsetof(CPUTLBEntry, addr_read)
1284+
: offsetof(CPUTLBEntry, addr_write));
1285+
tcg_wasm_out_op_ldst(s, OPC_I64_LOAD, 0, offset);
1286+
1287+
/*
1288+
* For aligned accesses, we check the first byte and include the
1289+
* alignment bits within the address. For unaligned access, we
1290+
* check that we don't cross pages using the address of the last
1291+
* byte of the access.
1292+
*/
1293+
tcg_wasm_out_op_idx(s, OPC_GLOBAL_GET, REG_IDX(addr_reg));
1294+
if (a_mask < s_mask) {
1295+
tcg_wasm_out_op_const(s, OPC_I64_CONST, s_mask - a_mask);
1296+
tcg_wasm_out_op(s, OPC_I64_ADD);
1297+
}
1298+
compare_mask = (uint64_t)TARGET_PAGE_MASK | a_mask;
1299+
tcg_wasm_out_op_const(s, OPC_I64_CONST, compare_mask);
1300+
tcg_wasm_out_op(s, OPC_I64_AND);
1301+
1302+
/* Compare masked address with the TLB entry. */
1303+
tcg_wasm_out_op(s, OPC_I64_EQ);
1304+
1305+
tcg_wasm_out_op_block(s, OPC_IF, BLOCK_NORET);
1306+
1307+
/* TLB Hit - translate address using addend. */
1308+
tcg_wasm_out_op_idx(s, OPC_LOCAL_GET, tmp1);
1309+
offset = tcg_wasm_out_norm_ptr(s, add_off);
1310+
tcg_wasm_out_op_ldst(s, OPC_I64_LOAD, 0, offset);
1311+
tcg_wasm_out_op_idx(s, OPC_GLOBAL_GET, REG_IDX(addr_reg));
1312+
tcg_wasm_out_op(s, OPC_I64_ADD);
1313+
tcg_wasm_out_op_idx(s, OPC_LOCAL_SET, tmp2);
1314+
tcg_wasm_out_op_const(s, OPC_I32_CONST, 1);
1315+
tcg_wasm_out_op_idx(s, OPC_LOCAL_SET, *hit_var);
1316+
1317+
tcg_wasm_out_op(s, OPC_END);
1318+
1319+
return tmp2;
1320+
}
1321+
1322+
static void tcg_wasm_out_qemu_ld_direct(
1323+
TCGContext *s, TCGReg r, uint8_t base, MemOp opc)
1324+
{
1325+
intptr_t ofs;
1326+
switch (opc & (MO_SSIZE)) {
1327+
case MO_UB:
1328+
tcg_wasm_out_op_idx(s, OPC_LOCAL_GET, base);
1329+
ofs = tcg_wasm_out_norm_ptr(s, 0);
1330+
tcg_wasm_out_op_ldst(s, OPC_I64_LOAD8_U, 0, ofs);
1331+
tcg_wasm_out_op_idx(s, OPC_GLOBAL_SET, REG_IDX(r));
1332+
break;
1333+
case MO_SB:
1334+
tcg_wasm_out_op_idx(s, OPC_LOCAL_GET, base);
1335+
ofs = tcg_wasm_out_norm_ptr(s, 0);
1336+
tcg_wasm_out_op_ldst(s, OPC_I64_LOAD8_S, 0, ofs);
1337+
tcg_wasm_out_op_idx(s, OPC_GLOBAL_SET, REG_IDX(r));
1338+
break;
1339+
case MO_UW:
1340+
tcg_wasm_out_op_idx(s, OPC_LOCAL_GET, base);
1341+
ofs = tcg_wasm_out_norm_ptr(s, 0);
1342+
tcg_wasm_out_op_ldst(s, OPC_I64_LOAD16_U, 0, ofs);
1343+
tcg_wasm_out_op_idx(s, OPC_GLOBAL_SET, REG_IDX(r));
1344+
break;
1345+
case MO_SW:
1346+
tcg_wasm_out_op_idx(s, OPC_LOCAL_GET, base);
1347+
ofs = tcg_wasm_out_norm_ptr(s, 0);
1348+
tcg_wasm_out_op_ldst(s, OPC_I64_LOAD16_S, 0, ofs);
1349+
tcg_wasm_out_op_idx(s, OPC_GLOBAL_SET, REG_IDX(r));
1350+
break;
1351+
case MO_UL:
1352+
tcg_wasm_out_op_idx(s, OPC_LOCAL_GET, base);
1353+
ofs = tcg_wasm_out_norm_ptr(s, 0);
1354+
tcg_wasm_out_op_ldst(s, OPC_I64_LOAD32_U, 0, ofs);
1355+
tcg_wasm_out_op_idx(s, OPC_GLOBAL_SET, REG_IDX(r));
1356+
break;
1357+
case MO_SL:
1358+
tcg_wasm_out_op_idx(s, OPC_LOCAL_GET, base);
1359+
ofs = tcg_wasm_out_norm_ptr(s, 0);
1360+
tcg_wasm_out_op_ldst(s, OPC_I64_LOAD32_S, 0, ofs);
1361+
tcg_wasm_out_op_idx(s, OPC_GLOBAL_SET, REG_IDX(r));
1362+
break;
1363+
case MO_UQ:
1364+
tcg_wasm_out_op_idx(s, OPC_LOCAL_GET, base);
1365+
ofs = tcg_wasm_out_norm_ptr(s, 0);
1366+
tcg_wasm_out_op_ldst(s, OPC_I64_LOAD, 0, ofs);
1367+
tcg_wasm_out_op_idx(s, OPC_GLOBAL_SET, REG_IDX(r));
1368+
break;
1369+
default:
1370+
g_assert_not_reached();
1371+
}
1372+
}
1373+
12201374
static void tcg_wasm_out_qemu_ld(TCGContext *s, TCGReg data_reg,
12211375
TCGReg addr_reg, MemOpIdx oi)
12221376
{
12231377
intptr_t helper_idx;
12241378
int64_t func_idx;
1379+
MemOp mop = get_memop(oi);
1380+
uint8_t base_var, hit_var;
12251381

12261382
helper_idx = (intptr_t)qemu_ld_helper_ptr(oi);
12271383
func_idx = get_helper_idx(s, helper_idx);
@@ -1230,6 +1386,14 @@ static void tcg_wasm_out_qemu_ld(TCGContext *s, TCGReg data_reg,
12301386
gen_func_type_qemu_ld(s, oi);
12311387
}
12321388

1389+
base_var = prepare_host_addr_wasm(s, &hit_var, addr_reg, oi, true);
1390+
tcg_wasm_out_op_idx(s, OPC_LOCAL_GET, hit_var);
1391+
tcg_wasm_out_op_const(s, OPC_I32_CONST, 1);
1392+
tcg_wasm_out_op(s, OPC_I32_EQ);
1393+
tcg_wasm_out_op_block(s, OPC_IF, BLOCK_NORET);
1394+
tcg_wasm_out_qemu_ld_direct(s, data_reg, base_var, mop); /* fast path */
1395+
tcg_wasm_out_op(s, OPC_END);
1396+
12331397
/*
12341398
* update the block index so that the possible rewinding will
12351399
* skip this block
@@ -1238,6 +1402,10 @@ static void tcg_wasm_out_qemu_ld(TCGContext *s, TCGReg data_reg,
12381402
tcg_wasm_out_op_idx(s, OPC_GLOBAL_SET, BLOCK_IDX);
12391403
tcg_wasm_out_new_block(s);
12401404

1405+
tcg_wasm_out_op_idx(s, OPC_LOCAL_GET, hit_var);
1406+
tcg_wasm_out_op(s, OPC_I32_EQZ);
1407+
tcg_wasm_out_op_block(s, OPC_IF, BLOCK_NORET);
1408+
12411409
/* call the target helper */
12421410
tcg_wasm_out_op_idx(s, OPC_GLOBAL_GET, REG_IDX(TCG_AREG0));
12431411
tcg_wasm_out_op_idx(s, OPC_GLOBAL_GET, REG_IDX(addr_reg));
@@ -1247,6 +1415,8 @@ static void tcg_wasm_out_qemu_ld(TCGContext *s, TCGReg data_reg,
12471415
tcg_wasm_out_op_idx(s, OPC_CALL, func_idx);
12481416
tcg_wasm_out_op_idx(s, OPC_GLOBAL_SET, REG_IDX(data_reg));
12491417
tcg_wasm_out_handle_unwinding(s);
1418+
1419+
tcg_wasm_out_op(s, OPC_END);
12501420
}
12511421

12521422
static void *qemu_st_helper_ptr(uint32_t oi)
@@ -1266,12 +1436,47 @@ static void *qemu_st_helper_ptr(uint32_t oi)
12661436
}
12671437
}
12681438

1439+
static void tcg_wasm_out_qemu_st_direct(
1440+
TCGContext *s, TCGReg lo, uint8_t base, MemOp opc)
1441+
{
1442+
intptr_t ofs;
1443+
switch (opc & (MO_SSIZE)) {
1444+
case MO_8:
1445+
tcg_wasm_out_op_idx(s, OPC_LOCAL_GET, base);
1446+
ofs = tcg_wasm_out_norm_ptr(s, 0);
1447+
tcg_wasm_out_op_idx(s, OPC_GLOBAL_GET, REG_IDX(lo));
1448+
tcg_wasm_out_op_ldst(s, OPC_I64_STORE8, 0, ofs);
1449+
break;
1450+
case MO_16:
1451+
tcg_wasm_out_op_idx(s, OPC_LOCAL_GET, base);
1452+
ofs = tcg_wasm_out_norm_ptr(s, 0);
1453+
tcg_wasm_out_op_idx(s, OPC_GLOBAL_GET, REG_IDX(lo));
1454+
tcg_wasm_out_op_ldst(s, OPC_I64_STORE16, 0, ofs);
1455+
break;
1456+
case MO_32:
1457+
tcg_wasm_out_op_idx(s, OPC_LOCAL_GET, base);
1458+
ofs = tcg_wasm_out_norm_ptr(s, 0);
1459+
tcg_wasm_out_op_idx(s, OPC_GLOBAL_GET, REG_IDX(lo));
1460+
tcg_wasm_out_op_ldst(s, OPC_I64_STORE32, 0, ofs);
1461+
break;
1462+
case MO_64:
1463+
tcg_wasm_out_op_idx(s, OPC_LOCAL_GET, base);
1464+
ofs = tcg_wasm_out_norm_ptr(s, 0);
1465+
tcg_wasm_out_op_idx(s, OPC_GLOBAL_GET, REG_IDX(lo));
1466+
tcg_wasm_out_op_ldst(s, OPC_I64_STORE, 0, ofs);
1467+
break;
1468+
default:
1469+
g_assert_not_reached();
1470+
}
1471+
}
1472+
12691473
static void tcg_wasm_out_qemu_st(TCGContext *s, TCGReg data_reg,
12701474
TCGReg addr_reg, MemOpIdx oi)
12711475
{
12721476
intptr_t helper_idx;
12731477
int64_t func_idx;
12741478
MemOp mop = get_memop(oi);
1479+
uint8_t base_var, hit_var;
12751480

12761481
helper_idx = (intptr_t)qemu_st_helper_ptr(oi);
12771482
func_idx = get_helper_idx(s, helper_idx);
@@ -1280,6 +1485,14 @@ static void tcg_wasm_out_qemu_st(TCGContext *s, TCGReg data_reg,
12801485
gen_func_type_qemu_st(s, oi);
12811486
}
12821487

1488+
base_var = prepare_host_addr_wasm(s, &hit_var, addr_reg, oi, false);
1489+
tcg_wasm_out_op_idx(s, OPC_LOCAL_GET, hit_var);
1490+
tcg_wasm_out_op_const(s, OPC_I32_CONST, 1);
1491+
tcg_wasm_out_op(s, OPC_I32_EQ);
1492+
tcg_wasm_out_op_block(s, OPC_IF, BLOCK_NORET);
1493+
tcg_wasm_out_qemu_st_direct(s, data_reg, base_var, mop); /* fast path */
1494+
tcg_wasm_out_op(s, OPC_END);
1495+
12831496
/*
12841497
* update the block index so that the possible rewinding will
12851498
* skip this block
@@ -1288,6 +1501,10 @@ static void tcg_wasm_out_qemu_st(TCGContext *s, TCGReg data_reg,
12881501
tcg_wasm_out_op_idx(s, OPC_GLOBAL_SET, BLOCK_IDX);
12891502
tcg_wasm_out_new_block(s);
12901503

1504+
tcg_wasm_out_op_idx(s, OPC_LOCAL_GET, hit_var);
1505+
tcg_wasm_out_op(s, OPC_I32_EQZ);
1506+
tcg_wasm_out_op_block(s, OPC_IF, BLOCK_NORET);
1507+
12911508
/* call the target helper */
12921509
tcg_wasm_out_op_idx(s, OPC_GLOBAL_GET, REG_IDX(TCG_AREG0));
12931510
tcg_wasm_out_op_idx(s, OPC_GLOBAL_GET, REG_IDX(addr_reg));
@@ -1305,6 +1522,8 @@ static void tcg_wasm_out_qemu_st(TCGContext *s, TCGReg data_reg,
13051522

13061523
tcg_wasm_out_op_idx(s, OPC_CALL, func_idx);
13071524
tcg_wasm_out_handle_unwinding(s);
1525+
1526+
tcg_wasm_out_op(s, OPC_END);
13081527
}
13091528

13101529
static void tcg_out_op_l(TCGContext *s, TCGOpcode op, TCGLabel *l0)
@@ -2152,7 +2371,7 @@ static const TCGOutOpQemuLdSt outop_qemu_st = {
21522371

21532372
bool tcg_target_has_memory_bswap(MemOp memop)
21542373
{
2155-
return true;
2374+
return false;
21562375
}
21572376

21582377
static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
@@ -2384,7 +2603,7 @@ static const uint8_t mod_3[] = {
23842603
0x80, 0x80, 0x80, 0x80, 0x00, /* placeholder for section size*/
23852604
1, /* num of codes */
23862605
0x80, 0x80, 0x80, 0x80, 0x00, /* placeholder for code size */
2387-
0x0, /* local variables (none) */
2606+
0x2, 0x1, 0x7f, 0x2, 0x7e, /* local variables (32bit*1, 64bit*2) */
23882607
};
23892608

23902609
#define MOD_3_PH_EXPORT_START_FUNC_IDX 102

0 commit comments

Comments
 (0)