Skip to content

Commit 2091033

Browse files
JulianGCalderongabrielbosio
authored andcommitted
Gather compilation statistics (#1236)
* Add basic Statistics struct and its builder * Add Serialzie to Statistics * Generate basic sierra statistics * Add stats argument to compile * Save sierra to mlir and mlir passes time * Add stats argument to object_to_shared_lib * Save linking time * Add stats argument to module_to_object * Save llvm time * Save total compilation time * Save object size in statistics * Remove StatisticsBuilder * Add libfunc frequency * Remove work * Assert frequency map is not empty * Save mlir operation count * Save llvmir instructions and opcode frequency * Save llvmir virtual register count * Adapt trace_dump.rs * Don't rely on transmute * Move MLIR walking logic to walk_ir module * Move LLVMIR walking to walk_ir module * Improve comments * Pass statistics to compile stats * Save MLIR operations by libfunc * Add clone_option_mut macro * Validate statistics * Use clone_option_mut * Document statistics * Document walk_ir module * Add API to statistics * Add support for saving stats on `starknet-native-compile` * Add documentation * Fix clippy * Update starknet-blocks.yml ref * Rephrase docs Co-authored-by: Gabriel Bosio <[email protected]> * Update script for comparing state dumps --------- Co-authored-by: Gabriel Bosio <[email protected]>
1 parent fef79e6 commit 2091033

36 files changed

+577
-162
lines changed

.github/workflows/daily.yml

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -199,8 +199,7 @@ jobs:
199199
continue-on-error: true
200200

201201
- name: Compare states
202-
run: |
203-
./scripts/cmp_state_dumps.sh | tee output
202+
run: python ./scripts/cmp_state_dumps.py | tee output
204203

205204
- name: Upload Compare Results
206205
id: upload_compare_results

.github/workflows/starknet-blocks.yml

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ jobs:
3232
with:
3333
repository: lambdaclass/starknet-replay
3434
path: starknet-replay
35-
ref: 135ba7cd5b45fe137b11b7f75654dcd472363033
35+
ref: 1b8e2e0be21a8df9f5f6b8f8514d1a40b456ef58
3636
# We need native to use the linux deps ci action
3737
- name: Checkout Native
3838
uses: actions/checkout@v4
@@ -43,7 +43,7 @@ jobs:
4343
with:
4444
repository: lambdaclass/sequencer
4545
path: sequencer
46-
ref: b61262980394dab0e0a4c4cab85f12d31d0ce878
46+
ref: 40331042c1149f5cb84b27f9dd8d47994a010bbe
4747

4848
- name: Cache RPC Calls
4949
uses: actions/[email protected]
@@ -126,5 +126,4 @@ jobs:
126126
continue-on-error: true
127127

128128
- name: Compare states
129-
run: |
130-
./scripts/cmp_state_dumps.sh
129+
run: python ./scripts/cmp_state_dumps.py

benches/compile_time.rs

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ pub fn bench_compile_time(c: &mut Criterion) {
1515
b.iter(|| {
1616
let native_context = NativeContext::new();
1717
native_context
18-
.compile(program, false, Some(Default::default()))
18+
.compile(program, false, Some(Default::default()), None)
1919
.unwrap();
2020
// pass manager internally verifies the MLIR output is correct.
2121
})
@@ -32,7 +32,7 @@ pub fn bench_compile_time(c: &mut Criterion) {
3232
c.bench_with_input(BenchmarkId::new(filename, 1), &program, |b, program| {
3333
b.iter(|| {
3434
native_context
35-
.compile(program, false, Some(Default::default()))
35+
.compile(program, false, Some(Default::default()), None)
3636
.unwrap();
3737
// pass manager internally verifies the MLIR output is correct.
3838
})
@@ -48,9 +48,9 @@ pub fn bench_compile_time(c: &mut Criterion) {
4848
b.iter(|| {
4949
let native_context = NativeContext::new();
5050
let module = native_context
51-
.compile(black_box(program), false, Some(Default::default()))
51+
.compile(black_box(program), false, Some(Default::default()), None)
5252
.unwrap();
53-
let object = module_to_object(module.module(), OptLevel::None)
53+
let object = module_to_object(module.module(), OptLevel::None, None)
5454
.expect("to compile correctly to a object file");
5555
black_box(object)
5656
})
@@ -67,9 +67,9 @@ pub fn bench_compile_time(c: &mut Criterion) {
6767
c.bench_with_input(BenchmarkId::new(filename, 1), &program, |b, program| {
6868
b.iter(|| {
6969
let module = native_context
70-
.compile(black_box(program), false, Some(Default::default()))
70+
.compile(black_box(program), false, Some(Default::default()), None)
7171
.unwrap();
72-
let object = module_to_object(module.module(), OptLevel::None)
72+
let object = module_to_object(module.module(), OptLevel::None, None)
7373
.expect("to compile correctly to a object file");
7474
black_box(object)
7575
})
@@ -86,9 +86,9 @@ pub fn bench_compile_time(c: &mut Criterion) {
8686
c.bench_with_input(BenchmarkId::new(filename, 1), &program, |b, program| {
8787
b.iter(|| {
8888
let module = native_context
89-
.compile(black_box(program), false, Some(Default::default()))
89+
.compile(black_box(program), false, Some(Default::default()), None)
9090
.unwrap();
91-
let object = module_to_object(module.module(), OptLevel::Aggressive)
91+
let object = module_to_object(module.module(), OptLevel::Aggressive, None)
9292
.expect("to compile correctly to a object file");
9393
black_box(object)
9494
})

benches/libfuncs.rs

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ pub fn bench_libfuncs(c: &mut Criterion) {
5555
let native_context = NativeContext::new();
5656
b.iter(|| {
5757
let module = native_context
58-
.compile(program, false, Some(Default::default()))
58+
.compile(program, false, Some(Default::default()), None)
5959
.unwrap();
6060
// pass manager internally verifies the MLIR output is correct.
6161
let native_executor =
@@ -77,7 +77,7 @@ pub fn bench_libfuncs(c: &mut Criterion) {
7777
|b, program| {
7878
let native_context = NativeContext::new();
7979
let module = native_context
80-
.compile(program, false, Some(Default::default()))
80+
.compile(program, false, Some(Default::default()), None)
8181
.unwrap();
8282
// pass manager internally verifies the MLIR output is correct.
8383
let native_executor =
@@ -108,7 +108,7 @@ pub fn bench_libfuncs(c: &mut Criterion) {
108108
let native_context = NativeContext::new();
109109
b.iter(|| {
110110
let module = native_context
111-
.compile(program, false, Some(Default::default()))
111+
.compile(program, false, Some(Default::default()), None)
112112
.unwrap();
113113
// pass manager internally verifies the MLIR output is correct.
114114
let native_executor =
@@ -130,7 +130,7 @@ pub fn bench_libfuncs(c: &mut Criterion) {
130130
|b, program| {
131131
let native_context = NativeContext::new();
132132
let module = native_context
133-
.compile(program, false, Some(Default::default()))
133+
.compile(program, false, Some(Default::default()), None)
134134
.unwrap();
135135
// pass manager internally verifies the MLIR output is correct.
136136
let native_executor =

examples/easy_api.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ fn main() {
1616

1717
// Compile the sierra program into a MLIR module.
1818
let native_program = native_context
19-
.compile(&sierra_program, false, Some(Default::default()))
19+
.compile(&sierra_program, false, Some(Default::default()), None)
2020
.unwrap();
2121

2222
// The parameters of the entry point.

examples/erc20.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -322,7 +322,7 @@ fn main() {
322322
let native_context = NativeContext::new();
323323

324324
let native_program = native_context
325-
.compile(&sierra_program, false, Some(Default::default()))
325+
.compile(&sierra_program, false, Some(Default::default()), None)
326326
.unwrap();
327327

328328
let entry_point_fn =

examples/invoke.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ fn main() {
2121
let native_context = NativeContext::new();
2222

2323
let native_program = native_context
24-
.compile(&sierra_program, false, Some(Default::default()))
24+
.compile(&sierra_program, false, Some(Default::default()), None)
2525
.unwrap();
2626

2727
// Call the echo function from the contract using the generated wrapper.

examples/starknet.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -456,7 +456,7 @@ fn main() {
456456
let native_context = NativeContext::new();
457457

458458
let native_program = native_context
459-
.compile(&sierra_program, false, Some(Default::default()))
459+
.compile(&sierra_program, false, Some(Default::default()), None)
460460
.unwrap();
461461

462462
// Call the echo function from the contract using the generated wrapper.

scripts/cmp_state_dumps.py

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
#!/usr/bin/env python
2+
#
3+
# usage: cmp-state-dumps [-h] [-d]
4+
# Compare all files in the state_dumps directory and outputs a summary
5+
# options:
6+
# -h, --help show this help message and exit
7+
# -d, --delete removes matching files
8+
#
9+
# Uses a pool of worker threads that compare each state dump.
10+
# possible improvements: use a pool of workers for file removing.
11+
12+
import argparse
13+
import glob
14+
import re
15+
import multiprocessing as mp
16+
import os
17+
from collections import defaultdict
18+
19+
POOL_SIZE = 16
20+
21+
STATE_DUMPS_PATH = "state_dumps"
22+
VM_DIRECTORY = "vm"
23+
NATIVE_DIRECTORY = "native"
24+
25+
LOG_PATH = "state_dumps/matching.log"
26+
27+
28+
def compare(vm_dump_path: str):
29+
native_dump_path = re.sub(VM_DIRECTORY, NATIVE_DIRECTORY, vm_dump_path, count=1)
30+
31+
if not (m := re.findall(r"/(0x.*).json", vm_dump_path)):
32+
raise Exception("bad path")
33+
tx = m[0]
34+
35+
if not (m := re.findall(r"block(\d+)", vm_dump_path)):
36+
raise Exception("bad path")
37+
block = m[0]
38+
39+
try:
40+
with open(native_dump_path) as f:
41+
native_dump = f.read()
42+
with open(vm_dump_path) as f:
43+
vm_dump = f.read()
44+
except: # noqa: E722
45+
return ("MISS", block, tx)
46+
47+
native_dump = re.sub(r".*reverted.*", "", native_dump, count=1)
48+
vm_dump = re.sub(r".*reverted.*", "", vm_dump, count=1)
49+
50+
if native_dump == vm_dump:
51+
return ("MATCH", block, tx, vm_dump_path, native_dump_path)
52+
else:
53+
return ("DIFF", block, tx)
54+
55+
56+
if __name__ == "__main__":
57+
parser = argparse.ArgumentParser(
58+
prog="cmp-state-dumps",
59+
description="Compare all files in the state_dumps directory and outputs a summary",
60+
)
61+
parser.add_argument(
62+
"-d", "--delete", action="store_true", help="removes matching files"
63+
)
64+
config = parser.parse_args()
65+
66+
files = glob.glob(f"{STATE_DUMPS_PATH}/{VM_DIRECTORY}/*/*.json")
67+
files.sort(key=os.path.getmtime)
68+
69+
print(f"Starting comparison with {POOL_SIZE} workers")
70+
71+
stats = defaultdict(int)
72+
with mp.Pool(POOL_SIZE) as pool, open(LOG_PATH, mode="a") as log:
73+
for status, *info in pool.imap(compare, files):
74+
stats[status] += 1
75+
76+
if status != "MATCH":
77+
(block, tx) = info
78+
print(status, block, tx)
79+
80+
elif status == "MATCH" and config.delete:
81+
(block, tx, vm_dump_path, native_dump_path) = info
82+
83+
log.write(f"{block} {tx}\n")
84+
log.flush()
85+
os.remove(native_dump_path)
86+
os.remove(vm_dump_path)
87+
88+
print("Finished comparison")
89+
90+
print()
91+
for key, count in stats.items():
92+
print(key, count)
93+
94+
if stats["DIFF"] != 0 or stats["MISS"] != 0:
95+
exit(1)
96+
else:
97+
exit(0)

scripts/cmp_state_dumps.sh

Lines changed: 0 additions & 51 deletions
This file was deleted.

0 commit comments

Comments
 (0)