E2E Tests using the VM (#59)

* begin register allocator

* begin reg alloc

* mutable virtual registers; basic allocation algorithm skeleton

* mutable registers in allocation

* pull in fuel-asm official ops

* switching laptops

* begin work on virtual registers and ops

* daily checkpoint

* add AllocatedOp abstraction

* template for parsing ops

* allocation algorithm progress

* change op parsing logic

* WIP parsing inline asm to new ops

* more op parsing

* finish parsing virtual ops from asm

* start registers method

* register allocation method

* convert virtual registers to allocated ones

* switch back to organizational labels for jumps

* realized ops

* fully allocate registers and resolve labels

* print allocated registers

* fill in todo!() errors in asm parsing

* resolve all todosudo apt-get install vlc in core_lang

* switch to ssh for fuel-asm

* resolve warnings

* fix git url

* rustfmt

* small self-code-review

* resolve module

* map the virtual opcodes to fuel_asm ops

* code review feedback

* factor finalized asm out into its own file

* realize data section and instructions to bits

* data section offset label

* initial bytecode generation

* add forc --asm command

* print out the loading of the data section op

* resolve warnings

* begin VM tests

* fix register allocater bug

* cleanup

* fix bad error message

* some tiny tweaks

* fix opcode encoding bug

* fix off-by-one error

* change all tests to run in the vm

* proper allocation size for structs

* code review feedback

* use ssh for deps

* undo rename

* fix doctest

* fix typo

* reference fuel_core for register constants

* new ssh key

* git change rust version

* wrong rust version

* update lockfile

* fix jump and enum instantiation bugs

* fix toml

* lockfile

* code review feedback

Co-authored-by: Alexander Hansen <[email protected]>

core_lang/src/asm_generation/expression/enum_instantiation.rs

@@ -56,7 +56,7 @@ pub(crate) fn convert_enum_instantiation_to_asm<'sc>(
 
     asm_buf.push(Op::unowned_stack_allocate_memory(
         VirtualImmediate24::new_unchecked(
-            size_of_enum,
+            size_of_enum * 8,
             "this size is manually checked to be lower than 2^24",
         ),
     ));

core_lang/src/asm_generation/expression/structs.rs

@@ -85,7 +85,7 @@ pub(crate) fn convert_struct_expression_to_asm<'sc>(
         // we call `new_unchecked` here because we have validated the size is okay above
         asm_buf.push(Op::unowned_stack_allocate_memory(
             VirtualImmediate24::new_unchecked(
-                this_allocation,
+                this_allocation * 8, // this_allocation is words but this op takes bytes
                 "struct size was checked manually to be within 12 bits",
             ),
         ));

core_lang/src/asm_generation/finalized_asm.rs

@@ -1,7 +1,7 @@
 use super::{DataSection, InstructionSet};
 use crate::error::*;
 use either::Either;
-use std::io::Write;
+use std::io::Read;
 /// Represents an ASM set which has had register allocation, jump elimination, and optimization
 /// applied to it
 pub enum FinalizedAsm<'sc> {
@@ -17,7 +17,6 @@ pub enum FinalizedAsm<'sc> {
     // Libraries do not generate any asm.
     Library,
 }
-
 impl<'sc> FinalizedAsm<'sc> {
     pub(crate) fn to_bytecode(&self) -> CompileResult<'sc, Vec<u8>> {
         use FinalizedAsm::*;
@@ -54,19 +53,22 @@ fn to_bytecode<'sc>(
     let offset_to_data_section = (program_section.ops.len() * 4) as u64;
 
     // each op is four bytes, so the length of the buf is then number of ops times four.
-    let mut buf = vec![0; program_section.ops.len() * 4];
+    let mut buf = vec![0; (program_section.ops.len() * 4) + 4];
 
-    for (ix, op) in program_section.ops.iter().enumerate() {
+    let mut half_word_ix = 0;
+    for op in program_section.ops.iter() {
         let op = op.to_fuel_asm(offset_to_data_section, data_section);
         match op {
             Either::Right(data) => {
                 for i in 0..data.len() {
-                    buf[ix + i] = data[i];
+                    buf[(half_word_ix * 4) + i] = data[i];
                 }
+                half_word_ix += 2;
             }
             Either::Left(mut op) => {
-                op.write(&buf[ix * 4..])
+                op.read(&mut buf[half_word_ix * 4..])
                     .expect("Failed to write to in-memory buffer.");
+                half_word_ix += 1;
             }
         }
     }

core_lang/src/asm_generation/mod.rs

@@ -158,6 +158,8 @@ impl<'sc> AbstractInstructionSet<'sc> {
             }
             buf.push(self.ops[i].clone());
         }
+        // the last item cannot sequentially jump by definition so we add it in here
+        self.ops.last().map(|x| buf.push(x.clone()));
 
         // scan through the jumps and remove any labels that are unused
         // this could of course be N instead of 2N if i did this in the above for loop.
@@ -346,20 +348,20 @@ pub struct DataSection<'sc> {
 
 impl<'sc> DataSection<'sc> {
     /// Given a [DataId], calculate the offset _from the beginning of the data section_ to the data
-    /// in words.
+    /// in bytes.
     pub(crate) fn offset_to_id(&self, id: &DataId) -> usize {
         self.value_pairs
             .iter()
             .take(id.0 as usize)
-            .map(|x| x.as_type().stack_size_of())
-            .sum::<u64>() as usize
+            .map(|x| x.to_bytes().len())
+            .sum()
     }
 
     pub(crate) fn serialize_to_bytes(&self) -> Vec<u8> {
         // not the exact right capacity but serves as a lower bound
         let mut buf = Vec::with_capacity(self.value_pairs.len());
         for val in &self.value_pairs {
-            buf.append(&mut val.to_bytes());
+            buf.append(&mut val.to_bytes().to_vec());
         }
         buf
     }
@@ -657,7 +659,6 @@ impl<'sc> HllAsmSet<'sc> {
 
 impl<'sc> JumpOptimizedAsmSet<'sc> {
     fn allocate_registers(self) -> RegisterAllocatedAsmSet<'sc> {
-        // TODO implement this -- noop for now
         match self {
             JumpOptimizedAsmSet::Library => RegisterAllocatedAsmSet::Library,
             JumpOptimizedAsmSet::ScriptMain {
@@ -811,6 +812,31 @@ fn convert_node_to_asm<'sc>(
                 errors,
             )
         }
+        TypedAstNodeContent::ReturnStatement(exp) => {
+            // if a return register was specified, we use it. If not, we generate a register but
+            // it is going to get thrown away later (in coalescing) as it is never read
+            let return_register = if let Some(return_register) = return_register {
+                return_register.clone()
+            } else {
+                register_sequencer.next()
+            };
+            let ops = type_check!(
+                convert_expression_to_asm(
+                    &exp.expr,
+                    namespace,
+                    &return_register,
+                    register_sequencer
+                ),
+                return err(warnings, errors),
+                warnings,
+                errors
+            );
+            ok(
+                NodeAsmResult::ReturnStatement { asm: ops },
+                warnings,
+                errors,
+            )
+        }
         _ => {
             errors.push(CompileError::Unimplemented(
                 "The ASM for this construct has not been written yet.",
@@ -849,6 +875,7 @@ fn build_preamble(register_sequencer: &mut RegisterSequencer) -> [Op<'static>; 6
             comment: "data section offset".into(),
             owning_span: None,
         },
+        Op::unowned_jump_label_comment(label, "end of metadata"),
         // word 3 -- load the data offset into $ds
         Op {
             opcode: Either::Left(VirtualOp::DataSectionRegisterLoadPlaceholder),
@@ -865,7 +892,5 @@ fn build_preamble(register_sequencer: &mut RegisterSequencer) -> [Op<'static>; 6
             comment: "".into(),
             owning_span: None,
         },
-        // word 3
-        Op::unowned_jump_label_comment(label, "end of metadata"),
     ]
 }

core_lang/src/asm_lang/allocated_ops.rs

@@ -340,7 +340,8 @@ impl<'sc> AllocatedOp<'sc> {
 
 fn realize_lw(dest: &AllocatedRegister, data_id: &DataId, data_section: &DataSection) -> VmOp {
     let dest = dest.to_register_id();
-    let offset = data_section.offset_to_id(data_id) as u64;
+    // all data is word-aligned right now, and `offset_to_id` returns the offset in bytes
+    let offset = (data_section.offset_to_id(data_id) / 8) as u64;
     let offset = match VirtualImmediate12::new(offset, Span::new(" ", 0, 0).unwrap()) {
         Ok ( value ) => value,
         Err  (_) => panic!("Unable to offset into the data section more than 2^12 bits. Unsupported data section length.")

core_lang/src/parse_tree/literal.rs

@@ -19,6 +19,7 @@ pub enum Literal<'sc> {
 }
 
 impl<'sc> Literal<'sc> {
+    #[allow(dead_code)]
     pub(crate) fn as_type(&self) -> ResolvedType<'sc> {
         use Literal::*;
         match self {
@@ -144,15 +145,24 @@ impl<'sc> Literal<'sc> {
             Err(compile_err) => err(Vec::new(), vec![compile_err]),
         }
     }
-    pub(crate) fn to_bytes(&self) -> Vec<u8> {
+    /// Converts a literal to a big-endian representation. This is padded to words.
+    pub(crate) fn to_bytes(&self) -> [u8; 8] {
         use Literal::*;
         match self {
-            // TODO are we big endian?
-            U8(val) => val.to_be_bytes().to_vec(),
-            U16(val) => val.to_be_bytes().to_vec(),
-            U32(val) => val.to_be_bytes().to_vec(),
-            U64(val) => val.to_be_bytes().to_vec(),
-            Boolean(b) => (if *b { 1u64 } else { 0u64 }).to_be_bytes().to_vec(),
+            U8(val) => [0, 0, 0, 0, 0, 0, 0, val.to_be_bytes()[0]],
+            U16(val) => {
+                let bytes = val.to_be_bytes();
+                [0, 0, 0, 0, 0, 0, bytes[0], bytes[1]]
+            }
+            U32(val) => {
+                let bytes = val.to_be_bytes();
+                [0, 0, 0, 0, bytes[0], bytes[1], bytes[2], bytes[3]]
+            }
+            U64(val) => val.to_be_bytes(),
+            Boolean(b) => {
+                let bytes = (if *b { 1u64 } else { 0u64 }).to_be_bytes();
+                [0, 0, 0, 0, 0, 0, 0, bytes[0]]
+            }
             a => todo!("{:?}", a),
         }
     }

example_project/fuel_project/src/main.sw

@@ -1,4 +1,4 @@
-script;
+predicate;
 struct Rgb {
   red: u64,
   green: u64,

forc/src/cli/commands/build.rs

@@ -14,6 +14,7 @@ pub(crate) fn exec(command: Command) -> Result<(), String> {
     if command.asm {
         forc_build::print_asm(command.path)
     } else {
-        forc_build::build(command.path)
+        forc_build::build(command.path)?;
+        Ok(())
     }
 }

forc/src/ops/forc_build.rs

@@ -46,7 +46,7 @@ pub fn print_asm(path: Option<String>) -> Result<(), String> {
     Ok(())
 }
 
-pub fn build(path: Option<String>) -> Result<(), String> {
+pub fn build(path: Option<String>) -> Result<Vec<u8>, String> {
     // find manifest directory, even if in subdirectory
     let this_dir = if let Some(path) = path {
         PathBuf::from(path)
@@ -82,7 +82,7 @@ pub fn build(path: Option<String>) -> Result<(), String> {
 
     println!("Bytecode size is {} bytes.", main.len());
 
-    Ok(())
+    Ok(main)
 }
 
 /// Continually go up in the file tree until a manifest (Forc.toml) is found.

stdlib/src/main.sw

@@ -7,6 +7,14 @@ pub trait Subtract {
   fn subtract(self, other: Self) -> Self;
 }
 
+pub trait Multiply {
+  fn multiply(self, other: Self) -> Self;
+}
+
+pub trait Divide {
+  fn divide(self, other: Self) -> Self;
+}
+
 impl Subtract for u64 {
   fn subtract(self, other: Self) -> Self {
     // TODO write asm

test_suite/Cargo.toml

@@ -8,3 +8,6 @@ edition = "2018"
 
 [dependencies]
 forc = { path = "../forc" }
+fuel-tx = { git = "ssh://[email protected]/FuelLabs/fuel-tx.git" }
+fuel-vm-rust = { git = "ssh://[email protected]/FuelLabs/fuel-core.git" }
+fuel-asm = { git = "ssh://[email protected]/FuelLabs/fuel-asm.git" }

test_suite/src/e2e_vm_tests/harness.rs

@@ -0,0 +1,50 @@
+use forc;
+
+use fuel_tx::Transaction;
+use fuel_vm_rust::interpreter::Interpreter;
+
+/// Very basic check that code does indeed run in the VM.
+/// `true` if it does, `false` if not.
+pub(crate) fn runs_in_vm(file_name: &str) {
+    let script = compile_to_bytes(file_name);
+    let gas_price = 10;
+    let gas_limit = 10000;
+    let maturity = 100;
+    let script_data = vec![];
+    let inputs = vec![];
+    let outputs = vec![];
+    let witness = vec![];
+    let tx = Transaction::script(
+        gas_price,
+        gas_limit,
+        maturity,
+        script,
+        script_data,
+        inputs,
+        outputs,
+        witness,
+    );
+    let block_height = (u32::MAX >> 1) as u64;
+    tx.validate(block_height).unwrap();
+    Interpreter::execute_tx(tx).unwrap();
+}
+
+/// Returns `true` if a file compiled without any errors or warnings,
+/// and `false` if it did not.
+pub(crate) fn compile_to_bytes(file_name: &str) -> Vec<u8> {
+    println!("Compiling {}", file_name);
+    let manifest_dir = env!("CARGO_MANIFEST_DIR");
+    let res = forc::ops::forc_build::build(Some(format!(
+        "{}/src/e2e_vm_tests/test_programs/{}",
+        manifest_dir, file_name
+    )));
+    match res {
+        Ok(bytes) => bytes,
+        Err(_) => {
+            panic!(
+                "TEST FAILURE: Project \"{}\" failed to compile. ",
+                file_name
+            );
+        }
+    }
+}

test_suite/src/e2e_vm_tests/mod.rs

@@ -0,0 +1,8 @@
+mod harness;
+
+pub fn run() {
+    let project_names = vec!["script_1", "script_2", "script_3"];
+    project_names.into_iter().for_each(|name| {
+        crate::e2e_vm_tests::harness::runs_in_vm(name);
+    });
+}

test_suite/src/basic_compilation_tests/test_programs/script_3/Forc.toml → test_suite/src/e2e_vm_tests/test_programs/script_2/Forc.toml

@@ -1,7 +1,7 @@
 [project]
 author  = "Alexander Hansen <[email protected]>"
 license = "MIT"
-name = "script_1"
+name = "script_2"
 entry = "main.sw"
 
 

test_suite/src/basic_compilation_tests/test_programs/script_2/Forc.toml → test_suite/src/e2e_vm_tests/test_programs/script_3/Forc.toml

@@ -1,7 +1,7 @@
 [project]
 author  = "Alexander Hansen <[email protected]>"
 license = "MIT"
-name = "script_1"
+name = "script_3"
 entry = "main.sw"
 
 

test_suite/src/basic_compilation_tests/test_programs/script_3/src/main.sw → test_suite/src/e2e_vm_tests/test_programs/script_3/src/main.sw

@@ -1,7 +1,7 @@
 script;
 // This test tests two-pass compilation and allowing usages before declarations.
 
-fn main() {
+fn main() -> bool {
   // fn before decl
   let x = the_number_five();
   // enum before decl
@@ -11,6 +11,7 @@ fn main() {
     a: true,
     b: false
   };
+  return true;
 }
 
 struct FuelStruct {