diff --git a/benchmark/benchmark_config.json b/benchmark/benchmark_config.json new file mode 100644 index 0000000..9397319 --- /dev/null +++ b/benchmark/benchmark_config.json @@ -0,0 +1,59 @@ +{ + "schema_version": 1, + "purpose": "Reference benchmark dimensions for the real C++ and Python Quadtrix benchmark suites.", + "common": { + "runs": 10, + "warmup": 3, + "quick_runs": 2, + "quick_warmup": 1, + "generate_tokens": 32, + "quick_generate_tokens": 4, + "train_steps": 5, + "quick_train_steps": 1, + "max_data_chars": 1000000, + "quick_max_data_chars": 50000 + }, + "suites": { + "data": [ + "tokenizer_or_char_encode", + "batch_sample_to_device" + ], + "primitive": [ + "matmul_3d", + "attention_scores_or_softmax3d", + "layer_norm" + ], + "forward": [ + "batch1_seq8", + "batch1_full_context", + "configured_batch_full_context" + ], + "training": [ + "adamw_step_forward_backward_update" + ], + "generation": [ + "empty_prompt", + "short_prompt", + "long_prompt" + ] + }, + "metrics": [ + "avg_ms", + "median_ms", + "min_ms", + "max_ms", + "p90_ms", + "p95_ms", + "std_ms", + "tokens_per_sec", + "loss", + "parameter_mb_fp32", + "memory_mb" + ], + "outputs": { + "cpp_json": "benchmark/results/cpp_benchmark.json", + "cpp_csv": "benchmark/results/cpp_benchmark.csv", + "python_json": "benchmark/results/python_benchmark.json", + "python_csv": "benchmark/results/python_benchmark.csv" + } +}