diff --git a/speed-bench/m5_max_q2q4.csv b/speed-bench/m5_max_q2q4.csv new file mode 100644 index 000000000..270586549 --- /dev/null +++ b/speed-bench/m5_max_q2q4.csv @@ -0,0 +1,33 @@ +ctx_tokens,prefill_tokens,prefill_tps,gen_tokens,gen_tps,gen_first_ms,gen_steady_tokens,gen_steady_tps,kvcache_bytes +2048,2048,485.27,128,38.71,27.744,127,39.33,52184460 +4096,2048,414.22,128,31.86,31.731,127,32.29,80373132 +6144,2048,392.00,128,30.83,39.596,127,31.29,108561804 +8192,2048,371.88,128,29.62,41.437,127,30.03,136750476 +10240,2048,358.84,128,29.91,41.415,127,30.33,164939148 +12288,2048,345.78,128,30.28,35.960,127,30.67,193127820 +14336,2048,334.54,128,29.83,34.750,127,30.21,221316492 +16384,2048,328.57,128,29.46,35.633,127,29.83,249505164 +18432,2048,323.05,128,29.35,35.330,127,29.71,277693836 +20480,2048,322.23,128,29.32,35.552,127,29.69,305882508 +22528,2048,324.35,128,29.29,35.098,127,29.64,334071180 +24576,2048,330.26,128,29.07,35.436,127,29.42,362259852 +26624,2048,325.29,128,26.03,45.964,127,26.35,390448524 +28672,2048,322.47,128,25.92,45.968,127,26.24,418637196 +30720,2048,316.03,128,25.73,41.932,127,26.02,446825868 +32768,2048,312.71,128,25.58,46.120,127,25.88,475014540 +34816,2048,303.72,128,25.29,41.835,127,25.56,503203212 +36864,2048,297.84,128,24.77,44.588,127,25.04,531391884 +38912,2048,290.76,128,24.47,49.117,127,24.75,559580556 +40960,2048,278.88,128,23.85,49.927,127,24.11,587769228 +43008,2048,265.64,128,23.47,46.880,127,23.71,615957900 +45056,2048,263.86,128,22.99,49.739,127,23.23,644146572 +47104,2048,255.24,128,22.86,52.785,127,23.11,672335244 +49152,2048,254.56,128,22.57,50.878,127,22.81,700523916 +51200,2048,249.90,128,22.27,54.205,127,22.52,728712588 +53248,2048,249.21,128,21.90,59.040,127,22.15,756901260 +55296,2048,248.48,128,22.07,52.683,127,22.30,785089932 +57344,2048,244.16,128,21.86,52.282,127,22.08,813278604 +59392,2048,241.48,128,21.54,56.468,127,21.76,841467276 +61440,2048,242.71,128,21.69,52.280,127,21.90,869655948 +63488,2048,239.01,128,21.61,57.501,127,21.84,897844620 +65536,2048,237.88,128,21.30,53.420,127,21.50,0 diff --git a/speed-bench/m5_max_q2q4_ts.svg b/speed-bench/m5_max_q2q4_ts.svg new file mode 100644 index 000000000..6a236151f --- /dev/null +++ b/speed-bench/m5_max_q2q4_ts.svg @@ -0,0 +1,51 @@ + + + + +M5 Max Q2+Q4 t/s + +0 + +200 + +400 + +600 + +800 + +1k +0 +10 +20 +30 +40 +50 + +0 + +20k + +40k + +60k + + + +ctx size +prefill t/s +generation t/s + + + + +prefill + +generation +