Change the memory representation of 64-bit, dual-limbed values to be little-endian.

In memory the low (least significant) 32-bit limb is now loaded or
stored from/to `addr` and the high 32-bit limb from/to `addr+1`.  On the
stack the limb order is still big-endian.

Author: otrho

codegen/masm/intrinsics/mem.masm

@@ -288,7 +288,7 @@ export.realign_dw # [chunk_hi, chunk_mid, chunk_lo, offset]
     swap.1 # [x_hi, x_lo]
 end
 
-# Shift a double-word (64-bit, in two 32-bit chunks) value by the given offset.
+# Shift two 32-bit words by the given offset.
 #
 # Returns three 32-bit chunks [chunk_lo, chunk_mid, chunk_hi]
 export.offset_dw # [value_hi, value_lo, offset]
@@ -308,7 +308,7 @@ export.offset_dw # [value_hi, value_lo, offset]
     u32shr       # [ chunk_lo, chunk_mid, chunk_hi]
 end
 
-# Load a machine double-word (64-bit value, in two 32-bit chunks) to the operand stack
+# Load two 32-bit words to the operand stack
 export.load_dw # [addr, offset]
     # check for alignment and offset validity
     dup.1 eq.0                # [offset == 0, addr, offset]
@@ -326,16 +326,17 @@ export.load_dw # [addr, offset]
     else
         # unaligned; an unaligned double-word spans three elements
         #
-        # convert offset from bytes to bits
-        swap.1 push.8 u32wrapping_mul swap.1              # [addr, bit_offset]
-
-        # load the three elements containing the double-word on the stack
-        dup.0 push.2 u32overflowing_add assertz mem_load  # [e2, addr, bit_offset]
-        dup.1 push.1 add mem_load                         # [e1, e2, addr, bit_offset]
-        movup.2 mem_load                                  # [e0, e1, e2, bit_offset]
-
-        # re-align it, and we're done
+        # convert offset from bytes to bitcount from RHS of triplet (32 - offset * 8)
+        swap.1 push.8 mul push.32 swap.1 sub swap.1        # [addr, bit_offset]
+
+        # load the three elements containing the double-word on the stack and re-align
+        # NOTE: realign_dw expects the hi, mid, lo word inputs and returns hi, lo (i.e., everything
+        # is 64-bit big-endian limb oriented), so we must swap the output to be in the correct order
+        dup.0 mem_load                                     # [e0, addr, bit_offset]
+        dup.1 push.1 u32overflowing_add assertz mem_load   # [e1, e0, addr, bit_offset]
+        movup.2 push.2 u32overflowing_add assertz mem_load # [e2, e1, e0, bit_offset]
         exec.realign_dw
+        swap.1
     end
 end
 
@@ -434,7 +435,7 @@ export.store_sw # [addr, offset, value]
     end
 end
 
-# Store a 64-bit value, i.e. two 32-bit machine words from the given native pointer tuple.
+# Store two 32-bit words to the given native pointer tuple.
 #
 # A native pointer tuple consists of an element address where the data begins, and a byte offset,
 # which is the offset of the first byte, in the 32-bit representation of that element.
@@ -536,7 +537,7 @@ end
 # Write `count` copies of `value` to memory, starting at `dst`.
 #
 # * `dst` is expected to be an address in byte-addressable space, _not_ an element address.
-# * `value` must be a 64-bit value or smaller
+# * `value` must be a two 32-bit words.
 export.memset_dw # [size, dst, count, value_hi, value_lo]
     # prepare to loop until `count` iterations have been performed
     push.0        # [i, dst, size, count, value_hi, value_lo]

codegen/masm/src/emit/mem.rs

@@ -62,7 +62,7 @@ impl OpEmitter<'_> {
                 );
                 match &ty {
                     Type::I128 => self.load_quad_word(None, span),
-                    Type::I64 | Type::U64 => self.load_double_word(None, span),
+                    Type::I64 | Type::U64 => self.load_double_word_int(None, span),
                     Type::Felt => self.load_felt(None, span),
                     Type::I32 | Type::U32 => self.load_word(None, span),
                     ty @ (Type::I16 | Type::U16 | Type::U8 | Type::I8 | Type::I1) => {
@@ -87,7 +87,7 @@ impl OpEmitter<'_> {
         let ptr = NativePtr::from_ptr(addr);
         match &ty {
             Type::I128 => self.load_quad_word(Some(ptr), span),
-            Type::I64 | Type::U64 => self.load_double_word(Some(ptr), span),
+            Type::I64 | Type::U64 => self.load_double_word_int(Some(ptr), span),
             Type::Felt => self.load_felt(Some(ptr), span),
             Type::I32 | Type::U32 => self.load_word(Some(ptr), span),
             Type::I16 | Type::U16 | Type::U8 | Type::I8 | Type::I1 => {
@@ -170,13 +170,17 @@ impl OpEmitter<'_> {
         }
     }
 
-    /// Load a pair of machine words (32-bit elements) to the operand stack
-    fn load_double_word(&mut self, ptr: Option<NativePtr>, span: SourceSpan) {
+    /// Load a 64-bit word from the given address.
+    fn load_double_word_int(&mut self, ptr: Option<NativePtr>, span: SourceSpan) {
         if let Some(imm) = ptr {
-            return self.load_double_word_imm(imm, span);
+            self.load_double_word_imm(imm, span);
+        } else {
+            self.raw_exec("intrinsics::mem::load_dw", span);
         }
 
-        self.raw_exec("intrinsics::mem::load_dw", span);
+        // The mem::intrinsic loads two 32-bit words with the first at the top of the stack.  Swap
+        // them to make a big-endian-limbed stack value.
+        self.emit(masm::Instruction::Swap1, span);
     }
 
     /// Load a sub-word value (u8, u16, etc.) from memory
@@ -538,7 +542,7 @@ impl OpEmitter<'_> {
                 );
                 match value_ty {
                     Type::I128 => self.store_quad_word(None, span),
-                    Type::I64 | Type::U64 => self.store_double_word(None, span),
+                    Type::I64 | Type::U64 => self.store_double_word_int(None, span),
                     Type::Felt => self.store_felt(None, span),
                     Type::I32 | Type::U32 => self.store_word(None, span),
                     ref ty if ty.size_in_bytes() <= 4 => self.store_small(ty, None, span),
@@ -566,7 +570,7 @@ impl OpEmitter<'_> {
         let ptr = NativePtr::from_ptr(addr);
         match value_ty {
             Type::I128 => self.store_quad_word(Some(ptr), span),
-            Type::I64 | Type::U64 => self.store_double_word(Some(ptr), span),
+            Type::I64 | Type::U64 => self.store_double_word_int(Some(ptr), span),
             Type::Felt => self.store_felt(Some(ptr), span),
             Type::I32 | Type::U32 => self.store_word(Some(ptr), span),
             ref ty if ty.size_in_bytes() <= 4 => self.store_small(ty, Some(ptr), span),
@@ -853,13 +857,18 @@ impl OpEmitter<'_> {
         }
     }
 
-    /// Store a pair of machine words (32-bit elements) to the operand stack
-    fn store_double_word(&mut self, ptr: Option<NativePtr>, span: SourceSpan) {
+    /// Store a 64-bit word to the operand stack
+    fn store_double_word_int(&mut self, ptr: Option<NativePtr>, span: SourceSpan) {
+        // The mem::intrinsic stores two 32-bit words in stack order.  Swap them (the 3rd and 4th
+        // params) first to make a little-endian-limbed memory value.
+        self.emit(masm::Instruction::MovUp2, span);
+        self.emit(masm::Instruction::MovDn3, span);
+
         if let Some(imm) = ptr {
-            return self.store_double_word_imm(imm, span);
+            self.store_double_word_imm(imm, span);
+        } else {
+            self.raw_exec("intrinsics::mem::store_dw", span);
         }
-
-        self.raw_exec("intrinsics::mem::store_dw", span);
     }
 
     fn store_double_word_imm(&mut self, ptr: NativePtr, span: SourceSpan) {

tests/integration/expected/abi_transform_stdlib_blake3_hash.masm

@@ -156,11 +156,14 @@ pub proc entrypoint(i32, i32)
     exec.::intrinsics::mem::load_dw
     trace.252
     nop
+    swap.1
     push.24
     dup.5
     u32wrapping_add
     u32divmod.4
     swap.1
+    movup.2
+    movdn.3
     trace.240
     nop
     exec.::intrinsics::mem::store_dw
@@ -183,11 +186,14 @@ pub proc entrypoint(i32, i32)
     exec.::intrinsics::mem::load_dw
     trace.252
     nop
+    swap.1
     push.16
     dup.5
     u32wrapping_add
     u32divmod.4
     swap.1
+    movup.2
+    movdn.3
     trace.240
     nop
     exec.::intrinsics::mem::store_dw
@@ -210,11 +216,14 @@ pub proc entrypoint(i32, i32)
     exec.::intrinsics::mem::load_dw
     trace.252
     nop
+    swap.1
     push.8
     dup.5
     u32wrapping_add
     u32divmod.4
     swap.1
+    movup.2
+    movdn.3
     trace.240
     nop
     exec.::intrinsics::mem::store_dw
@@ -233,9 +242,12 @@ pub proc entrypoint(i32, i32)
     exec.::intrinsics::mem::load_dw
     trace.252
     nop
+    swap.1
     movup.3
     u32divmod.4
     swap.1
+    movup.2
+    movdn.3
     trace.240
     nop
     exec.::intrinsics::mem::store_dw

tests/integration/expected/add_i128.masm

@@ -44,6 +44,8 @@ pub proc entrypoint(i32, [u32; 2], [u32; 2], [u32; 2], [u32; 2])
     assertz
     u32divmod.4
     swap.1
+    movup.2
+    movdn.3
     trace.240
     nop
     exec.::intrinsics::mem::store_dw
@@ -58,6 +60,8 @@ pub proc entrypoint(i32, [u32; 2], [u32; 2], [u32; 2], [u32; 2])
     assertz
     u32divmod.4
     swap.1
+    movup.2
+    movdn.3
     trace.240
     nop
     exec.::intrinsics::mem::store_dw

tests/integration/expected/add_u128.masm

@@ -44,6 +44,8 @@ pub proc entrypoint(i32, [u32; 2], [u32; 2], [u32; 2], [u32; 2])
     assertz
     u32divmod.4
     swap.1
+    movup.2
+    movdn.3
     trace.240
     nop
     exec.::intrinsics::mem::store_dw
@@ -58,6 +60,8 @@ pub proc entrypoint(i32, [u32; 2], [u32; 2], [u32; 2], [u32; 2])
     assertz
     u32divmod.4
     swap.1
+    movup.2
+    movdn.3
     trace.240
     nop
     exec.::intrinsics::mem::store_dw

tests/integration/expected/examples/auth_component_no_auth.masm

@@ -109,6 +109,7 @@ proc miden:base/authentication-component@1.0.0#auth-procedure(
     exec.::intrinsics::mem::load_dw
     trace.252
     nop
+    swap.1
     push.56
     dup.3
     add
@@ -121,6 +122,8 @@ proc miden:base/authentication-component@1.0.0#auth-procedure(
     assertz
     u32divmod.4
     swap.1
+    movup.2
+    movdn.3
     trace.240
     nop
     exec.::intrinsics::mem::store_dw
@@ -143,6 +146,7 @@ proc miden:base/authentication-component@1.0.0#auth-procedure(
     exec.::intrinsics::mem::load_dw
     trace.252
     nop
+    swap.1
     push.48
     dup.3
     add
@@ -155,6 +159,8 @@ proc miden:base/authentication-component@1.0.0#auth-procedure(
     assertz
     u32divmod.4
     swap.1
+    movup.2
+    movdn.3
     trace.240
     nop
     exec.::intrinsics::mem::store_dw
@@ -194,6 +200,7 @@ proc miden:base/authentication-component@1.0.0#auth-procedure(
     exec.::intrinsics::mem::load_dw
     trace.252
     nop
+    swap.1
     push.56
     dup.3
     add
@@ -206,6 +213,8 @@ proc miden:base/authentication-component@1.0.0#auth-procedure(
     assertz
     u32divmod.4
     swap.1
+    movup.2
+    movdn.3
     trace.240
     nop
     exec.::intrinsics::mem::store_dw
@@ -228,6 +237,7 @@ proc miden:base/authentication-component@1.0.0#auth-procedure(
     exec.::intrinsics::mem::load_dw
     trace.252
     nop
+    swap.1
     push.48
     dup.3
     add
@@ -240,6 +250,8 @@ proc miden:base/authentication-component@1.0.0#auth-procedure(
     assertz
     u32divmod.4
     swap.1
+    movup.2
+    movdn.3
     trace.240
     nop
     exec.::intrinsics::mem::store_dw
@@ -599,6 +611,7 @@ proc miden_stdlib_sys::intrinsics::word::Word::reverse(
     exec.::intrinsics::mem::load_dw
     trace.252
     nop
+    swap.1
     push.8
     dup.3
     add
@@ -611,6 +624,8 @@ proc miden_stdlib_sys::intrinsics::word::Word::reverse(
     assertz
     u32divmod.4
     swap.1
+    movup.2
+    movdn.3
     trace.240
     nop
     exec.::intrinsics::mem::store_dw
@@ -630,6 +645,7 @@ proc miden_stdlib_sys::intrinsics::word::Word::reverse(
     exec.::intrinsics::mem::load_dw
     trace.252
     nop
+    swap.1
     dup.2
     push.4
     dup.1
@@ -639,6 +655,8 @@ proc miden_stdlib_sys::intrinsics::word::Word::reverse(
     assertz
     u32divmod.4
     swap.1
+    movup.2
+    movdn.3
     trace.240
     nop
     exec.::intrinsics::mem::store_dw
@@ -786,6 +804,7 @@ proc miden_stdlib_sys::intrinsics::word::Word::reverse(
     exec.::intrinsics::mem::load_dw
     trace.252
     nop
+    swap.1
     push.8
     dup.4
     add
@@ -798,6 +817,8 @@ proc miden_stdlib_sys::intrinsics::word::Word::reverse(
     assertz
     u32divmod.4
     swap.1
+    movup.2
+    movdn.3
     trace.240
     nop
     exec.::intrinsics::mem::store_dw
@@ -816,6 +837,7 @@ proc miden_stdlib_sys::intrinsics::word::Word::reverse(
     exec.::intrinsics::mem::load_dw
     trace.252
     nop
+    swap.1
     movup.2
     push.8
     dup.1
@@ -825,6 +847,8 @@ proc miden_stdlib_sys::intrinsics::word::Word::reverse(
     assertz
     u32divmod.4
     swap.1
+    movup.2
+    movdn.3
     trace.240
     nop
     exec.::intrinsics::mem::store_dw