Port Qwen row-map SwiGLU expert gathers
This commit is contained in:
Georg Bauer committed
1 parent
8965e77ea1
commit
7b894fbfee
42 files changed
+11489
-65
No files matched your search
@@ -0,0 +1,838 @@
|
||||
[
|
||||
{
|
||||
"configuration": "default-tail",
|
||||
"arm": "before",
|
||||
"records": [
|
||||
{
|
||||
"chunk": 2048,
|
||||
"rep": 0,
|
||||
"phase": "cold",
|
||||
"prompt": 4096,
|
||||
"cached": 0,
|
||||
"ttft_seconds": 3.7007944999999998,
|
||||
"prefill_seconds": 3.6969075010000005,
|
||||
"prefill_tokens_per_second": 1107.9530658779117,
|
||||
"elapsed_seconds": 3.721188
|
||||
},
|
||||
{
|
||||
"chunk": 2048,
|
||||
"rep": 0,
|
||||
"phase": "warm",
|
||||
"prompt": 6145,
|
||||
"cached": 4097,
|
||||
"ttft_seconds": 1.7724733750000001,
|
||||
"prefill_seconds": 1.769735542,
|
||||
"prefill_tokens_per_second": 1157.23505088536,
|
||||
"elapsed_seconds": 1.793335125
|
||||
},
|
||||
{
|
||||
"chunk": 2048,
|
||||
"rep": 1,
|
||||
"phase": "cold",
|
||||
"prompt": 4096,
|
||||
"cached": 0,
|
||||
"ttft_seconds": 2.820473209,
|
||||
"prefill_seconds": 2.81689246,
|
||||
"prefill_tokens_per_second": 1454.0846191906098,
|
||||
"elapsed_seconds": 2.841268834
|
||||
},
|
||||
{
|
||||
"chunk": 2048,
|
||||
"rep": 1,
|
||||
"phase": "warm",
|
||||
"prompt": 6145,
|
||||
"cached": 4097,
|
||||
"ttft_seconds": 1.838782375,
|
||||
"prefill_seconds": 1.836070168,
|
||||
"prefill_tokens_per_second": 1115.4257803942492,
|
||||
"elapsed_seconds": 1.8597940419999999
|
||||
},
|
||||
{
|
||||
"chunk": 2048,
|
||||
"rep": 2,
|
||||
"phase": "cold",
|
||||
"prompt": 4096,
|
||||
"cached": 0,
|
||||
"ttft_seconds": 2.959423792,
|
||||
"prefill_seconds": 2.9556270000000002,
|
||||
"prefill_tokens_per_second": 1385.831162051233,
|
||||
"elapsed_seconds": 2.98031925
|
||||
},
|
||||
{
|
||||
"chunk": 2048,
|
||||
"rep": 2,
|
||||
"phase": "warm",
|
||||
"prompt": 6145,
|
||||
"cached": 4097,
|
||||
"ttft_seconds": 1.922527375,
|
||||
"prefill_seconds": 1.919643709,
|
||||
"prefill_tokens_per_second": 1066.8646428492007,
|
||||
"elapsed_seconds": 1.9441567499999999
|
||||
},
|
||||
{
|
||||
"chunk": 4096,
|
||||
"rep": 0,
|
||||
"phase": "cold",
|
||||
"prompt": 4096,
|
||||
"cached": 0,
|
||||
"ttft_seconds": 3.626180458,
|
||||
"prefill_seconds": 3.6201352510000007,
|
||||
"prefill_tokens_per_second": 1131.4494393181994,
|
||||
"elapsed_seconds": 3.64758525
|
||||
},
|
||||
{
|
||||
"chunk": 4096,
|
||||
"rep": 0,
|
||||
"phase": "warm",
|
||||
"prompt": 6145,
|
||||
"cached": 4097,
|
||||
"ttft_seconds": 1.994229042,
|
||||
"prefill_seconds": 1.9914031639999998,
|
||||
"prefill_tokens_per_second": 1028.4205815392588,
|
||||
"elapsed_seconds": 2.016047875
|
||||
},
|
||||
{
|
||||
"chunk": 4096,
|
||||
"rep": 1,
|
||||
"phase": "cold",
|
||||
"prompt": 4096,
|
||||
"cached": 0,
|
||||
"ttft_seconds": 3.6903119589999998,
|
||||
"prefill_seconds": 3.6843170420000004,
|
||||
"prefill_tokens_per_second": 1111.7392866322168,
|
||||
"elapsed_seconds": 3.711886
|
||||
},
|
||||
{
|
||||
"chunk": 4096,
|
||||
"rep": 1,
|
||||
"phase": "warm",
|
||||
"prompt": 6145,
|
||||
"cached": 4097,
|
||||
"ttft_seconds": 2.011648875,
|
||||
"prefill_seconds": 2.0087712079999998,
|
||||
"prefill_tokens_per_second": 1019.5287506331085,
|
||||
"elapsed_seconds": 2.033881333
|
||||
},
|
||||
{
|
||||
"chunk": 4096,
|
||||
"rep": 2,
|
||||
"phase": "cold",
|
||||
"prompt": 4096,
|
||||
"cached": 0,
|
||||
"ttft_seconds": 3.752376958,
|
||||
"prefill_seconds": 3.7459722089999996,
|
||||
"prefill_tokens_per_second": 1093.4411072669013,
|
||||
"elapsed_seconds": 3.774499291
|
||||
},
|
||||
{
|
||||
"chunk": 4096,
|
||||
"rep": 2,
|
||||
"phase": "warm",
|
||||
"prompt": 6145,
|
||||
"cached": 4097,
|
||||
"ttft_seconds": 2.020803542,
|
||||
"prefill_seconds": 2.017895623,
|
||||
"prefill_tokens_per_second": 1014.918698795354,
|
||||
"elapsed_seconds": 2.043202292
|
||||
}
|
||||
],
|
||||
"summary": [
|
||||
{
|
||||
"chunk": 2048,
|
||||
"phase": "cold",
|
||||
"ttft_seconds": {
|
||||
"mean": 3.1602305003333333,
|
||||
"median": 2.959423792,
|
||||
"sample_stddev": 0.47326936772912714
|
||||
},
|
||||
"prefill_tokens_per_second": {
|
||||
"mean": 1315.9562823732515,
|
||||
"median": 1385.831162051233,
|
||||
"sample_stddev": 183.340222420435
|
||||
}
|
||||
},
|
||||
{
|
||||
"chunk": 2048,
|
||||
"phase": "warm",
|
||||
"ttft_seconds": {
|
||||
"mean": 1.844594375,
|
||||
"median": 1.838782375,
|
||||
"sample_stddev": 0.07519564639658331
|
||||
},
|
||||
"prefill_tokens_per_second": {
|
||||
"mean": 1113.1751580429366,
|
||||
"median": 1115.4257803942492,
|
||||
"sample_stddev": 45.227222310038556
|
||||
}
|
||||
},
|
||||
{
|
||||
"chunk": 4096,
|
||||
"phase": "cold",
|
||||
"ttft_seconds": {
|
||||
"mean": 3.689623125,
|
||||
"median": 3.6903119589999998,
|
||||
"sample_stddev": 0.06310106989799923
|
||||
},
|
||||
"prefill_tokens_per_second": {
|
||||
"mean": 1112.2099444057726,
|
||||
"median": 1111.7392866322168,
|
||||
"sample_stddev": 19.00853664502584
|
||||
}
|
||||
},
|
||||
{
|
||||
"chunk": 4096,
|
||||
"phase": "warm",
|
||||
"ttft_seconds": {
|
||||
"mean": 2.0088938196666666,
|
||||
"median": 2.011648875,
|
||||
"sample_stddev": 0.013499768886162316
|
||||
},
|
||||
"prefill_tokens_per_second": {
|
||||
"mean": 1020.9560103225738,
|
||||
"median": 1019.5287506331085,
|
||||
"sample_stddev": 6.863163415905101
|
||||
}
|
||||
}
|
||||
],
|
||||
"resources": {
|
||||
"event": "test_resource_summary",
|
||||
"elapsed_ms": 41097,
|
||||
"peak": {
|
||||
"cpu_time_ns": 19599062416,
|
||||
"resident_bytes": 25489604608,
|
||||
"physical_bytes": 84724073856,
|
||||
"peak_physical_bytes": 84724139512,
|
||||
"disk_read_bytes": 64097386496,
|
||||
"disk_write_bytes": 442368
|
||||
},
|
||||
"error": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"configuration": "default-tail",
|
||||
"arm": "after",
|
||||
"records": [
|
||||
{
|
||||
"chunk": 2048,
|
||||
"rep": 0,
|
||||
"phase": "cold",
|
||||
"prompt": 4096,
|
||||
"cached": 0,
|
||||
"ttft_seconds": 3.699041667,
|
||||
"prefill_seconds": 3.6953074989999997,
|
||||
"prefill_tokens_per_second": 1108.4327897227588,
|
||||
"elapsed_seconds": 3.719328292
|
||||
},
|
||||
{
|
||||
"chunk": 2048,
|
||||
"rep": 0,
|
||||
"phase": "warm",
|
||||
"prompt": 6145,
|
||||
"cached": 4097,
|
||||
"ttft_seconds": 1.776398,
|
||||
"prefill_seconds": 1.7737045,
|
||||
"prefill_tokens_per_second": 1154.6455455235075,
|
||||
"elapsed_seconds": 1.797202167
|
||||
},
|
||||
{
|
||||
"chunk": 2048,
|
||||
"rep": 1,
|
||||
"phase": "cold",
|
||||
"prompt": 4096,
|
||||
"cached": 0,
|
||||
"ttft_seconds": 2.80434375,
|
||||
"prefill_seconds": 2.8008109150000005,
|
||||
"prefill_tokens_per_second": 1462.4336038050606,
|
||||
"elapsed_seconds": 2.825135334
|
||||
},
|
||||
{
|
||||
"chunk": 2048,
|
||||
"rep": 1,
|
||||
"phase": "warm",
|
||||
"prompt": 6145,
|
||||
"cached": 4097,
|
||||
"ttft_seconds": 1.900156542,
|
||||
"prefill_seconds": 1.897415875,
|
||||
"prefill_tokens_per_second": 1079.3627411808177,
|
||||
"elapsed_seconds": 1.9216478750000001
|
||||
},
|
||||
{
|
||||
"chunk": 2048,
|
||||
"rep": 2,
|
||||
"phase": "cold",
|
||||
"prompt": 4096,
|
||||
"cached": 0,
|
||||
"ttft_seconds": 2.952652333,
|
||||
"prefill_seconds": 2.948979292,
|
||||
"prefill_tokens_per_second": 1388.9551585226932,
|
||||
"elapsed_seconds": 2.973752041
|
||||
},
|
||||
{
|
||||
"chunk": 2048,
|
||||
"rep": 2,
|
||||
"phase": "warm",
|
||||
"prompt": 6145,
|
||||
"cached": 4097,
|
||||
"ttft_seconds": 1.947212708,
|
||||
"prefill_seconds": 1.944330169,
|
||||
"prefill_tokens_per_second": 1053.3190466582735,
|
||||
"elapsed_seconds": 1.9688718330000001
|
||||
},
|
||||
{
|
||||
"chunk": 4096,
|
||||
"rep": 0,
|
||||
"phase": "cold",
|
||||
"prompt": 4096,
|
||||
"cached": 0,
|
||||
"ttft_seconds": 3.637618958,
|
||||
"prefill_seconds": 3.631528458,
|
||||
"prefill_tokens_per_second": 1127.8997390139687,
|
||||
"elapsed_seconds": 3.659774833
|
||||
},
|
||||
{
|
||||
"chunk": 4096,
|
||||
"rep": 0,
|
||||
"phase": "warm",
|
||||
"prompt": 6145,
|
||||
"cached": 4097,
|
||||
"ttft_seconds": 1.9611875410000001,
|
||||
"prefill_seconds": 1.958409373,
|
||||
"prefill_tokens_per_second": 1045.7466289914453,
|
||||
"elapsed_seconds": 1.982932208
|
||||
},
|
||||
{
|
||||
"chunk": 4096,
|
||||
"rep": 1,
|
||||
"phase": "cold",
|
||||
"prompt": 4096,
|
||||
"cached": 0,
|
||||
"ttft_seconds": 3.6904271250000003,
|
||||
"prefill_seconds": 3.684310291,
|
||||
"prefill_tokens_per_second": 1111.7413237440048,
|
||||
"elapsed_seconds": 3.71194925
|
||||
},
|
||||
{
|
||||
"chunk": 4096,
|
||||
"rep": 1,
|
||||
"phase": "warm",
|
||||
"prompt": 6145,
|
||||
"cached": 4097,
|
||||
"ttft_seconds": 1.988992875,
|
||||
"prefill_seconds": 1.98611425,
|
||||
"prefill_tokens_per_second": 1031.159209496634,
|
||||
"elapsed_seconds": 2.010877
|
||||
},
|
||||
{
|
||||
"chunk": 4096,
|
||||
"rep": 2,
|
||||
"phase": "cold",
|
||||
"prompt": 4096,
|
||||
"cached": 0,
|
||||
"ttft_seconds": 3.684088083,
|
||||
"prefill_seconds": 3.6781829150000003,
|
||||
"prefill_tokens_per_second": 1113.5933406944227,
|
||||
"elapsed_seconds": 3.705752166
|
||||
},
|
||||
{
|
||||
"chunk": 4096,
|
||||
"rep": 2,
|
||||
"phase": "warm",
|
||||
"prompt": 6145,
|
||||
"cached": 4097,
|
||||
"ttft_seconds": 1.998740333,
|
||||
"prefill_seconds": 1.9958163330000003,
|
||||
"prefill_tokens_per_second": 1026.1465276825147,
|
||||
"elapsed_seconds": 2.021001667
|
||||
}
|
||||
],
|
||||
"summary": [
|
||||
{
|
||||
"chunk": 2048,
|
||||
"phase": "cold",
|
||||
"ttft_seconds": {
|
||||
"mean": 3.1520125833333332,
|
||||
"median": 2.952652333,
|
||||
"sample_stddev": 0.4795096169325091
|
||||
},
|
||||
"prefill_tokens_per_second": {
|
||||
"mean": 1319.940517350171,
|
||||
"median": 1388.9551585226932,
|
||||
"sample_stddev": 186.81918963228082
|
||||
}
|
||||
},
|
||||
{
|
||||
"chunk": 2048,
|
||||
"phase": "warm",
|
||||
"ttft_seconds": {
|
||||
"mean": 1.8745890833333332,
|
||||
"median": 1.900156542,
|
||||
"sample_stddev": 0.0882308751191163
|
||||
},
|
||||
"prefill_tokens_per_second": {
|
||||
"mean": 1095.775777787533,
|
||||
"median": 1079.3627411808177,
|
||||
"sample_stddev": 52.619441949355554
|
||||
}
|
||||
},
|
||||
{
|
||||
"chunk": 4096,
|
||||
"phase": "cold",
|
||||
"ttft_seconds": {
|
||||
"mean": 3.6707113886666667,
|
||||
"median": 3.684088083,
|
||||
"sample_stddev": 0.02883361907462447
|
||||
},
|
||||
"prefill_tokens_per_second": {
|
||||
"mean": 1117.7448011507988,
|
||||
"median": 1113.5933406944227,
|
||||
"sample_stddev": 8.843051732845977
|
||||
}
|
||||
},
|
||||
{
|
||||
"chunk": 4096,
|
||||
"phase": "warm",
|
||||
"ttft_seconds": {
|
||||
"mean": 1.9829735830000002,
|
||||
"median": 1.988992875,
|
||||
"sample_stddev": 0.019486583946006573
|
||||
},
|
||||
"prefill_tokens_per_second": {
|
||||
"mean": 1034.3507887235314,
|
||||
"median": 1031.159209496634,
|
||||
"sample_stddev": 10.18236840333609
|
||||
}
|
||||
}
|
||||
],
|
||||
"resources": {
|
||||
"event": "test_resource_summary",
|
||||
"elapsed_ms": 40871,
|
||||
"peak": {
|
||||
"cpu_time_ns": 19458778041,
|
||||
"resident_bytes": 25328418816,
|
||||
"physical_bytes": 84668646808,
|
||||
"peak_physical_bytes": 84668761376,
|
||||
"disk_read_bytes": 69973549056,
|
||||
"disk_write_bytes": 18446744073709289472
|
||||
},
|
||||
"error": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"configuration": "full-chunks",
|
||||
"arm": "before",
|
||||
"records": [
|
||||
{
|
||||
"chunk": 2048,
|
||||
"rep": 0,
|
||||
"phase": "cold",
|
||||
"prompt": 4096,
|
||||
"cached": 0,
|
||||
"ttft_seconds": 3.182152,
|
||||
"prefill_seconds": 3.17999279,
|
||||
"prefill_tokens_per_second": 1288.0532348628376,
|
||||
"elapsed_seconds": 3.202144875
|
||||
},
|
||||
{
|
||||
"chunk": 2048,
|
||||
"rep": 0,
|
||||
"phase": "warm",
|
||||
"prompt": 6145,
|
||||
"cached": 4097,
|
||||
"ttft_seconds": 1.219195625,
|
||||
"prefill_seconds": 1.21829175,
|
||||
"prefill_tokens_per_second": 1681.042328325707,
|
||||
"elapsed_seconds": 1.240087208
|
||||
},
|
||||
{
|
||||
"chunk": 2048,
|
||||
"rep": 1,
|
||||
"phase": "cold",
|
||||
"prompt": 4096,
|
||||
"cached": 0,
|
||||
"ttft_seconds": 2.317122125,
|
||||
"prefill_seconds": 2.3159108330000002,
|
||||
"prefill_tokens_per_second": 1768.6345871503593,
|
||||
"elapsed_seconds": 2.338011208
|
||||
},
|
||||
{
|
||||
"chunk": 2048,
|
||||
"rep": 1,
|
||||
"phase": "warm",
|
||||
"prompt": 6145,
|
||||
"cached": 4097,
|
||||
"ttft_seconds": 1.26216725,
|
||||
"prefill_seconds": 1.261251084,
|
||||
"prefill_tokens_per_second": 1623.7845310743853,
|
||||
"elapsed_seconds": 1.283571459
|
||||
},
|
||||
{
|
||||
"chunk": 2048,
|
||||
"rep": 2,
|
||||
"phase": "cold",
|
||||
"prompt": 4096,
|
||||
"cached": 0,
|
||||
"ttft_seconds": 2.411353459,
|
||||
"prefill_seconds": 2.410105791,
|
||||
"prefill_tokens_per_second": 1699.510459373026,
|
||||
"elapsed_seconds": 2.431755125
|
||||
},
|
||||
{
|
||||
"chunk": 2048,
|
||||
"rep": 2,
|
||||
"phase": "warm",
|
||||
"prompt": 6145,
|
||||
"cached": 4097,
|
||||
"ttft_seconds": 1.34331275,
|
||||
"prefill_seconds": 1.34238875,
|
||||
"prefill_tokens_per_second": 1525.6385305672443,
|
||||
"elapsed_seconds": 1.364779
|
||||
},
|
||||
{
|
||||
"chunk": 4096,
|
||||
"rep": 0,
|
||||
"phase": "cold",
|
||||
"prompt": 4096,
|
||||
"cached": 0,
|
||||
"ttft_seconds": 3.2133175,
|
||||
"prefill_seconds": 3.212275166,
|
||||
"prefill_tokens_per_second": 1275.1086965879188,
|
||||
"elapsed_seconds": 3.233724875
|
||||
},
|
||||
{
|
||||
"chunk": 4096,
|
||||
"rep": 0,
|
||||
"phase": "warm",
|
||||
"prompt": 6145,
|
||||
"cached": 4097,
|
||||
"ttft_seconds": 1.7273878329999999,
|
||||
"prefill_seconds": 1.7264041659999998,
|
||||
"prefill_tokens_per_second": 1186.2807332915115,
|
||||
"elapsed_seconds": 1.7481919160000001
|
||||
},
|
||||
{
|
||||
"chunk": 4096,
|
||||
"rep": 1,
|
||||
"phase": "cold",
|
||||
"prompt": 4096,
|
||||
"cached": 0,
|
||||
"ttft_seconds": 2.980801042,
|
||||
"prefill_seconds": 2.979894292,
|
||||
"prefill_tokens_per_second": 1374.5454028340412,
|
||||
"elapsed_seconds": 3.001205167
|
||||
},
|
||||
{
|
||||
"chunk": 4096,
|
||||
"rep": 1,
|
||||
"phase": "warm",
|
||||
"prompt": 6145,
|
||||
"cached": 4097,
|
||||
"ttft_seconds": 1.7668409999999999,
|
||||
"prefill_seconds": 1.765948334,
|
||||
"prefill_tokens_per_second": 1159.7168278197203,
|
||||
"elapsed_seconds": 1.787861709
|
||||
},
|
||||
{
|
||||
"chunk": 4096,
|
||||
"rep": 2,
|
||||
"phase": "cold",
|
||||
"prompt": 4096,
|
||||
"cached": 0,
|
||||
"ttft_seconds": 3.100441792,
|
||||
"prefill_seconds": 3.0994622499999998,
|
||||
"prefill_tokens_per_second": 1321.5195635952657,
|
||||
"elapsed_seconds": 3.121023875
|
||||
},
|
||||
{
|
||||
"chunk": 4096,
|
||||
"rep": 2,
|
||||
"phase": "warm",
|
||||
"prompt": 6145,
|
||||
"cached": 4097,
|
||||
"ttft_seconds": 1.838995291,
|
||||
"prefill_seconds": 1.838062583,
|
||||
"prefill_tokens_per_second": 1114.216686059378,
|
||||
"elapsed_seconds": 1.860065208
|
||||
}
|
||||
],
|
||||
"summary": [
|
||||
{
|
||||
"chunk": 2048,
|
||||
"phase": "cold",
|
||||
"ttft_seconds": {
|
||||
"mean": 2.6368758613333334,
|
||||
"median": 2.411353459,
|
||||
"sample_stddev": 0.4745676312458291
|
||||
},
|
||||
"prefill_tokens_per_second": {
|
||||
"mean": 1585.3994271287409,
|
||||
"median": 1699.510459373026,
|
||||
"sample_stddev": 259.81840735698063
|
||||
}
|
||||
},
|
||||
{
|
||||
"chunk": 2048,
|
||||
"phase": "warm",
|
||||
"ttft_seconds": {
|
||||
"mean": 1.274891875,
|
||||
"median": 1.26216725,
|
||||
"sample_stddev": 0.06302937601316841
|
||||
},
|
||||
"prefill_tokens_per_second": {
|
||||
"mean": 1610.1551299891123,
|
||||
"median": 1623.7845310743853,
|
||||
"sample_stddev": 78.59329182503896
|
||||
}
|
||||
},
|
||||
{
|
||||
"chunk": 4096,
|
||||
"phase": "cold",
|
||||
"ttft_seconds": {
|
||||
"mean": 3.098186778,
|
||||
"median": 3.100441792,
|
||||
"sample_stddev": 0.11627463019215153
|
||||
},
|
||||
"prefill_tokens_per_second": {
|
||||
"mean": 1323.7245543390752,
|
||||
"median": 1321.5195635952657,
|
||||
"sample_stddev": 49.755011058230345
|
||||
}
|
||||
},
|
||||
{
|
||||
"chunk": 4096,
|
||||
"phase": "warm",
|
||||
"ttft_seconds": {
|
||||
"mean": 1.7777413746666666,
|
||||
"median": 1.7668409999999999,
|
||||
"sample_stddev": 0.05659655286508827
|
||||
},
|
||||
"prefill_tokens_per_second": {
|
||||
"mean": 1153.4047490568698,
|
||||
"median": 1159.7168278197203,
|
||||
"sample_stddev": 36.444320265304924
|
||||
}
|
||||
}
|
||||
],
|
||||
"resources": {
|
||||
"event": "test_resource_summary",
|
||||
"elapsed_ms": 35856,
|
||||
"peak": {
|
||||
"cpu_time_ns": 21540742125,
|
||||
"resident_bytes": 25113673728,
|
||||
"physical_bytes": 88578784936,
|
||||
"peak_physical_bytes": 88578866928,
|
||||
"disk_read_bytes": 69525413888,
|
||||
"disk_write_bytes": 524288
|
||||
},
|
||||
"error": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"configuration": "full-chunks",
|
||||
"arm": "after",
|
||||
"records": [
|
||||
{
|
||||
"chunk": 2048,
|
||||
"rep": 0,
|
||||
"phase": "cold",
|
||||
"prompt": 4096,
|
||||
"cached": 0,
|
||||
"ttft_seconds": 3.126132542,
|
||||
"prefill_seconds": 3.1225067920000003,
|
||||
"prefill_tokens_per_second": 1311.766562203846,
|
||||
"elapsed_seconds": 3.147092209
|
||||
},
|
||||
{
|
||||
"chunk": 2048,
|
||||
"rep": 0,
|
||||
"phase": "warm",
|
||||
"prompt": 6145,
|
||||
"cached": 4097,
|
||||
"ttft_seconds": 1.262224125,
|
||||
"prefill_seconds": 1.258582084,
|
||||
"prefill_tokens_per_second": 1627.2279941337542,
|
||||
"elapsed_seconds": 1.282883709
|
||||
},
|
||||
{
|
||||
"chunk": 2048,
|
||||
"rep": 1,
|
||||
"phase": "cold",
|
||||
"prompt": 4096,
|
||||
"cached": 0,
|
||||
"ttft_seconds": 2.304937333,
|
||||
"prefill_seconds": 2.303636208,
|
||||
"prefill_tokens_per_second": 1778.058525810426,
|
||||
"elapsed_seconds": 2.325319833
|
||||
},
|
||||
{
|
||||
"chunk": 2048,
|
||||
"rep": 1,
|
||||
"phase": "warm",
|
||||
"prompt": 6145,
|
||||
"cached": 4097,
|
||||
"ttft_seconds": 1.333947917,
|
||||
"prefill_seconds": 1.3330325,
|
||||
"prefill_tokens_per_second": 1536.346638210246,
|
||||
"elapsed_seconds": 1.354474625
|
||||
},
|
||||
{
|
||||
"chunk": 2048,
|
||||
"rep": 2,
|
||||
"phase": "cold",
|
||||
"prompt": 4096,
|
||||
"cached": 0,
|
||||
"ttft_seconds": 2.504060958,
|
||||
"prefill_seconds": 2.502749457,
|
||||
"prefill_tokens_per_second": 1636.6000953646394,
|
||||
"elapsed_seconds": 2.524487958
|
||||
},
|
||||
{
|
||||
"chunk": 2048,
|
||||
"rep": 2,
|
||||
"phase": "warm",
|
||||
"prompt": 6145,
|
||||
"cached": 4097,
|
||||
"ttft_seconds": 1.363442375,
|
||||
"prefill_seconds": 1.362460209,
|
||||
"prefill_tokens_per_second": 1503.1631650388993,
|
||||
"elapsed_seconds": 1.3840412500000001
|
||||
},
|
||||
{
|
||||
"chunk": 4096,
|
||||
"rep": 0,
|
||||
"phase": "cold",
|
||||
"prompt": 4096,
|
||||
"cached": 0,
|
||||
"ttft_seconds": 2.955218083,
|
||||
"prefill_seconds": 2.9541866239999997,
|
||||
"prefill_tokens_per_second": 1386.50685326507,
|
||||
"elapsed_seconds": 2.9757388750000002
|
||||
},
|
||||
{
|
||||
"chunk": 4096,
|
||||
"rep": 0,
|
||||
"phase": "warm",
|
||||
"prompt": 6145,
|
||||
"cached": 4097,
|
||||
"ttft_seconds": 1.606924,
|
||||
"prefill_seconds": 1.6059842079999997,
|
||||
"prefill_tokens_per_second": 1275.2304722538095,
|
||||
"elapsed_seconds": 1.62792175
|
||||
},
|
||||
{
|
||||
"chunk": 4096,
|
||||
"rep": 1,
|
||||
"phase": "cold",
|
||||
"prompt": 4096,
|
||||
"cached": 0,
|
||||
"ttft_seconds": 3.209116167,
|
||||
"prefill_seconds": 3.2081535000000003,
|
||||
"prefill_tokens_per_second": 1276.7468888256126,
|
||||
"elapsed_seconds": 3.230131833
|
||||
},
|
||||
{
|
||||
"chunk": 4096,
|
||||
"rep": 1,
|
||||
"phase": "warm",
|
||||
"prompt": 6145,
|
||||
"cached": 4097,
|
||||
"ttft_seconds": 1.767990417,
|
||||
"prefill_seconds": 1.767019791,
|
||||
"prefill_tokens_per_second": 1159.0136174088839,
|
||||
"elapsed_seconds": 1.789304875
|
||||
},
|
||||
{
|
||||
"chunk": 4096,
|
||||
"rep": 2,
|
||||
"phase": "cold",
|
||||
"prompt": 4096,
|
||||
"cached": 0,
|
||||
"ttft_seconds": 2.998665042,
|
||||
"prefill_seconds": 2.997745792,
|
||||
"prefill_tokens_per_second": 1366.3600198959098,
|
||||
"elapsed_seconds": 3.019893375
|
||||
},
|
||||
{
|
||||
"chunk": 4096,
|
||||
"rep": 2,
|
||||
"phase": "warm",
|
||||
"prompt": 6145,
|
||||
"cached": 4097,
|
||||
"ttft_seconds": 1.633976833,
|
||||
"prefill_seconds": 1.633039209,
|
||||
"prefill_tokens_per_second": 1254.1033850951462,
|
||||
"elapsed_seconds": 1.656772416
|
||||
}
|
||||
],
|
||||
"summary": [
|
||||
{
|
||||
"chunk": 2048,
|
||||
"phase": "cold",
|
||||
"ttft_seconds": {
|
||||
"mean": 2.645043611,
|
||||
"median": 2.504060958,
|
||||
"sample_stddev": 0.4283660515917751
|
||||
},
|
||||
"prefill_tokens_per_second": {
|
||||
"mean": 1575.475061126304,
|
||||
"median": 1636.6000953646394,
|
||||
"sample_stddev": 239.08001001547228
|
||||
}
|
||||
},
|
||||
{
|
||||
"chunk": 2048,
|
||||
"phase": "warm",
|
||||
"ttft_seconds": {
|
||||
"mean": 1.3198714723333334,
|
||||
"median": 1.333947917,
|
||||
"sample_stddev": 0.052056635063226324
|
||||
},
|
||||
"prefill_tokens_per_second": {
|
||||
"mean": 1555.5792657942998,
|
||||
"median": 1536.346638210246,
|
||||
"sample_stddev": 64.22959541694213
|
||||
}
|
||||
},
|
||||
{
|
||||
"chunk": 4096,
|
||||
"phase": "cold",
|
||||
"ttft_seconds": {
|
||||
"mean": 3.054333097333333,
|
||||
"median": 2.998665042,
|
||||
"sample_stddev": 0.1357949135761868
|
||||
},
|
||||
"prefill_tokens_per_second": {
|
||||
"mean": 1343.2045873288641,
|
||||
"median": 1366.3600198959098,
|
||||
"sample_stddev": 58.42895678910406
|
||||
}
|
||||
},
|
||||
{
|
||||
"chunk": 4096,
|
||||
"phase": "warm",
|
||||
"ttft_seconds": {
|
||||
"mean": 1.6696304166666667,
|
||||
"median": 1.633976833,
|
||||
"sample_stddev": 0.0862495286509525
|
||||
},
|
||||
"prefill_tokens_per_second": {
|
||||
"mean": 1229.4491582526132,
|
||||
"median": 1254.1033850951462,
|
||||
"sample_stddev": 61.90688583198103
|
||||
}
|
||||
}
|
||||
],
|
||||
"resources": {
|
||||
"event": "test_resource_summary",
|
||||
"elapsed_ms": 34614,
|
||||
"peak": {
|
||||
"cpu_time_ns": 20995139625,
|
||||
"resident_bytes": 25576030208,
|
||||
"physical_bytes": 87952375368,
|
||||
"peak_physical_bytes": 87952457288,
|
||||
"disk_read_bytes": 64869085184,
|
||||
"disk_write_bytes": 442368
|
||||
},
|
||||
"error": null
|
||||
}
|
||||
}
|
||||
]
|
||||
Reference in new issue
Block a user