vlut.cpp/evaluation/scripts/plot/plot_utils.py at master · OpenBitSys/vlut.cpp · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115

TYPE_ORDER = {
    'Ours I2 (b2.00)': 0,
    'Ours I1 (b1.60)': 1,
    'T-MAC INT_N (b2.00)': 2,
    'llama.cpp Q4_0 (b4.50)': 3,
    'llama.cpp TQ2_0 (b2.06)': 4,
    'llama.cpp TQ1_0 (b1.69)': 5,
    'bitnet.cpp I2_S (b2.00)': 6,
    'bitnet.cpp TL1 (b2.00)': 7,
    'bitnet.cpp TL2 (b1.67)': 8,
}

DEVICE_MAP = {
    'pc_intel': 'Intel PC (x86)',
    'laptop_amd': 'Legion 5 Pro (x86)',
    'smartphone': 'Xiaomi 15 (ARM)',
    'orangepi': 'Orange Pi 5 Plus (ARM)',
    'aws_arm': 'AWS Graviton 3 (ARM)',
}

# Centralized evaluation root directory for outputs (figures, results, reports)
# This resolves to the `evaluation` folder regardless of the script's location.
import os
EVALUATION_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

def eval_path(*parts: str) -> str:
    """Join path parts under the evaluation root.

    Example:
        eval_path('figures', 'gemm_comparison_single_thread.pdf')
        eval_path('reports_gemm', 'single_thread')
    """
    return os.path.join(EVALUATION_ROOT, *parts)

GEMM_TYPE_VARIANTS = ['i2_v','i2_v_4','i2_v_8','i1_v','i1_v_2']
E2E_TYPE_VARIANTS = ['I2_V','I2_V_4','I2_V_8','I1_V','I1_V_2']

GEMM_TYPE_DEVICE_MAP = {
    'aws_arm': ['i2_v','i1_v'],
    'pc_intel': ['i2_v_8','i1_v_2'],
    'laptop_amd': ['i2_v_4','i1_v_2'],
    'laptop_intel': ['i2_v_4','i1_v_2'],
    'smartphone': ['i2_v_4','i1_v_2'],
    'orangepi': ['i2_v_4','i1_v_2'],
}

E2E_TYPE_DEVICE_MAP = {
    'aws_arm': ['I2_V','I1_V'],
    'pc_intel': ['I2_V_8','I1_V_2'],
    'laptop_amd': ['I2_V_4','I1_V_2'],
    'laptop_intel': ['I2_V_4','I1_V_2'],
    'smartphone': ['I2_V_4','I1_V_2'],
    'orangepi': ['I2_V_4','I1_V_2'],
}

GEMM_TYPE_MAP = {
    'i2_v': 'Ours I2 (b2.00)',
    'i2_v_4': 'Ours I2 (b2.00)',
    'i2_v_8': 'Ours I2 (b2.00)',
    'i1_v': 'Ours I1 (b1.60)',
    'i1_v_2': 'Ours I1 (b1.60)',
    'tmac': 'T-MAC INT_N (b2.00)',
    'q4_0': 'llama.cpp Q4_0 (b4.50)',
    'tq1_0': 'llama.cpp TQ1_0 (b1.69)',
    'tq2_0': 'llama.cpp TQ2_0 (b2.06)',
}

GEMM_TYPE_STYLES = {
    'Ours I2 (b2.00)': {'color': '#3274A1', 'hatch': ''},
    'Ours I1 (b1.60)': {'color': '#3274A1', 'hatch': '//'},
    'T-MAC INT_N (b2.00)': {'color': "#FF7F7D", 'hatch': ''},
    'llama.cpp Q4_0 (b4.50)': {'color': '#32A178', 'hatch': '..'},
    'llama.cpp TQ1_0 (b1.69)': {'color': '#32A178', 'hatch': '//'},
    'llama.cpp TQ2_0 (b2.06)': {'color': '#32A178', 'hatch': ''},
}

E2E_TYPE_MAP = {
    'I2_V': 'Ours I2 (b2.00)',
    'I2_V_4': 'Ours I2 (b2.00)',
    'I2_V_8': 'Ours I2 (b2.00)',
    'I1_V': 'Ours I1 (b1.60)',
    'I1_V_2': 'Ours I1 (b1.60)',
    'INT_N': 'T-MAC INT_N (b2.00)',
    'Q4_0': 'llama.cpp Q4_0 (b4.50)',
    'TQ2_0': 'llama.cpp TQ2_0 (b2.06)',
    'TQ1_0': 'llama.cpp TQ1_0 (b1.69)',
    'i2_s': 'bitnet.cpp I2_S (b2.00)',
    'tl1': 'bitnet.cpp TL1 (b2.00)',
    'tl2': 'bitnet.cpp TL2 (b1.67)',
}

E2E_TYPE_STYLES = {
    'Ours I2 (b2.00)': GEMM_TYPE_STYLES['Ours I2 (b2.00)'],
    'Ours I1 (b1.60)': GEMM_TYPE_STYLES['Ours I1 (b1.60)'],
    'T-MAC INT_N (b2.00)': GEMM_TYPE_STYLES['T-MAC INT_N (b2.00)'],
    'llama.cpp Q4_0 (b4.50)': GEMM_TYPE_STYLES['llama.cpp Q4_0 (b4.50)'],
    'llama.cpp TQ1_0 (b1.69)': GEMM_TYPE_STYLES['llama.cpp TQ1_0 (b1.69)'],
    'llama.cpp TQ2_0 (b2.06)': GEMM_TYPE_STYLES['llama.cpp TQ2_0 (b2.06)'],
    'bitnet.cpp I2_S (b2.00)': {'color': "#FCAD58", 'hatch': ''},
    'bitnet.cpp TL1 (b2.00)': {'color': "#FCAD58", 'hatch': '\\\\'},
    'bitnet.cpp TL2 (b1.67)': {'color': "#FCAD58", 'hatch': '//'},
}

GEMM_MODEL_MAP = {
    'bitnet_3b': 'BitNet 3B',
    'falcon_1b': 'Falcon 1B',
    'llama3_8b': 'LLaMA 3 8B',
}

E2E_MODEL_MAP = {
    'bitnet_b1_58-3B': 'BitNet 3B',
    'Falcon3-1B-Instruct-1.58bit': 'Falcon 1B',
    'Llama3-8B-1.58-100B-tokens': 'Llama3 8B',
}