{
 "llamacpp": {
  "small": {
   "metric": "llama.cpp llama-bench, Llama-2 7B Q4_0, tg128 (text generation tok/s); best of FA on/off as posted",
   "short": "Small model: Llama-2 7B Q4_0",
   "chips": {
    "strix-halo": {
     "label": "AMD Ryzen AI Max+ 395 (Radeon 8060S)",
     "tps": 50.95,
     "backend": "ROCm",
     "author": "kresle",
     "date": "2026-02-02",
     "url": "https://github.com/ggml-org/llama.cpp/discussions/15013#discussioncomment-15671492",
     "pp": 554.48,
     "pp_metric": "pp512",
     "source": "llamacpp",
     "quant": "Q4_0"
    },
    "gb10": {
     "label": "NVIDIA GB10 (DGX Spark)",
     "tps": 57.21,
     "backend": "CUDA",
     "author": "ggerganov",
     "date": "2025-10-15",
     "url": "https://github.com/ggml-org/llama.cpp/discussions/15013#discussioncomment-14687095",
     "pp": 3062.31,
     "pp_metric": "pp512",
     "source": "llamacpp",
     "quant": "Q4_0"
    },
    "m5-ultra-80": {
     "label": "Apple M5 Ultra 36/80 (256GB)",
     "tps": 179.1,
     "backend": "Metal",
     "author": "see thread",
     "date": "2026-09-30",
     "url": "https://github.com/ggml-org/llama.cpp/discussions/4167#discussioncomment-18683605",
     "pp": 4944.66,
     "pp_metric": "pp512",
     "source": "llamacpp",
     "quant": "Q4_0"
    },
    "m5-max-40": {
     "label": "Apple M5 Max 40-core GPU (MacBook Pro 16, 128GB)",
     "tps": 119.08,
     "backend": "Metal",
     "author": "see thread",
     "date": "2026-06-22",
     "url": "https://github.com/ggml-org/llama.cpp/discussions/4167#discussioncomment-17386908",
     "pp": 3347.44,
     "pp_metric": "pp512",
     "source": "llamacpp",
     "quant": "Q4_0"
    },
    "rtx-5090": {
     "label": "NVIDIA RTX 5090",
     "tps": 314.0,
     "backend": "CUDA",
     "author": "totaldev",
     "date": "2025-12-04",
     "url": "https://github.com/ggml-org/llama.cpp/discussions/15013#discussioncomment-15160999",
     "pp": 14805.32,
     "pp_metric": "pp512",
     "source": "llamacpp",
     "quant": "Q4_0"
    },
    "rtx-4090": {
     "label": "NVIDIA RTX 4090",
     "tps": 188.96,
     "backend": "CUDA",
     "author": "lhl",
     "date": "2025-08-06",
     "url": "https://github.com/ggml-org/llama.cpp/discussions/15013#discussioncomment-14016393",
     "pp": 14770.63,
     "pp_metric": "pp512",
     "source": "llamacpp",
     "quant": "Q4_0"
    },
    "rtx-3090": {
     "label": "NVIDIA RTX 3090",
     "tps": 177.79,
     "backend": "CUDA",
     "author": "ygafarov",
     "date": "2026-05-03",
     "url": "https://github.com/ggml-org/llama.cpp/discussions/15013#discussioncomment-16797292",
     "pp": 5992.1,
     "pp_metric": "pp512",
     "source": "llamacpp",
     "quant": "Q4_0"
    },
    "m5-pro-16": {
     "label": "Apple M5 Pro 16-core GPU (Mac mini, 64GB)",
     "tps": 67.59,
     "backend": "Metal",
     "author": "see thread",
     "date": "2026-09-24",
     "url": "https://github.com/ggml-org/llama.cpp/discussions/4167#discussioncomment-18583406",
     "pp": 1340.44,
     "pp_metric": "pp512",
     "source": "llamacpp",
     "quant": "Q4_0"
    },
    "rtx-5080": {
     "label": "NVIDIA RTX 5080",
     "tps": 184.68,
     "backend": "CUDA",
     "author": "Hedede",
     "date": "2025-08-28",
     "url": "https://github.com/ggml-org/llama.cpp/discussions/15013#discussioncomment-14244491",
     "pp": 9487.7,
     "pp_metric": "pp512",
     "source": "llamacpp",
     "quant": "Q4_0"
    },
    "rtx-5070-ti": {
     "label": "NVIDIA RTX 5070 Ti",
     "tps": 182.43,
     "backend": "CUDA",
     "author": "TinyServal",
     "date": "2025-12-07",
     "url": "https://github.com/ggml-org/llama.cpp/discussions/15013#discussioncomment-15183261",
     "pp": 8419.56,
     "pp_metric": "pp512",
     "source": "llamacpp",
     "quant": "Q4_0"
    },
    "rtx-pro-6000-maxq": {
     "label": "NVIDIA RTX PRO 6000 Blackwell Max-Q",
     "tps": 287.29,
     "backend": "CUDA",
     "author": "RexBytes",
     "date": "2026-05-08",
     "url": "https://github.com/ggml-org/llama.cpp/discussions/15013#discussioncomment-16854411",
     "pp": 13403.7,
     "pp_metric": "pp512",
     "source": "llamacpp",
     "quant": "Q4_0"
    }
   },
   "pp_note": "pp512 from the same llama-bench run and flash-attention setting as the tg128 figure, same source link."
  },
  "large": {
   "metric": "gpt-oss-120b MXFP4 (59 GiB weights), text generation tok/s at empty context, single request",
   "short": "Large model: gpt-oss-120b",
   "min_mem_gb": 96,
   "fit_rule": "Needs about 59 GiB for weights plus context and OS, so devices with 64GB or less are marked 'doesn't fit'.",
   "chips": {
    "gb10": {
     "label": "NVIDIA GB10 (DGX Spark)",
     "tps": 58.72,
     "engine": "llama.cpp CUDA, llama-bench tg32, b7946",
     "date": "2026-02-05",
     "url": "https://github.com/ggml-org/llama.cpp/blob/master/benches/dgx-spark/dgx-spark.md",
     "pp": 2443.91,
     "pp_metric": "pp2048",
     "source": "llamacpp",
     "quant": "MXFP4"
    },
    "strix-halo": {
     "label": "Ryzen AI Max+ 395, 128GB (Beelink GTR9 Pro)",
     "tps": 55.57,
     "engine": "llama.cpp Vulkan, llama-bench tg128, b9049, 20 repeats",
     "date": "2026-05-07",
     "url": "https://github.com/hogeheer499-commits/strix-halo-guide/blob/HEAD/data/benchmarks.csv",
     "pp": 728.6,
     "pp_metric": "pp2048",
     "source": "shg",
     "quant": "MXFP4"
    },
    "m5-max-40": {
     "label": "Apple M5 Max 40-core GPU (run on a 128GB machine)",
     "tps": 92.8,
     "engine": "oMLX, 1k context, median of 5 community runs (not llama-bench)",
     "date": "2026-09-28",
     "url": "https://omlx.ai/benchmarks/performance/6p90fycd",
     "pp": 1619.0,
     "pp_metric": "prefill at 1k context",
     "needs_mem_gb": 128,
     "source": "omlx",
     "quant": "MLX 4-bit (MXFP4-Q8)",
     "runs": 5,
     "range": [
      81.2,
      94.2
     ]
    },
    "m5-ultra-64": {
     "label": "Apple M5 Ultra 64-core GPU (run on a 96GB machine)",
     "tps": 97.2,
     "engine": "oMLX, 1k context, median of 2 community runs (not llama-bench)",
     "date": "2026-10-08",
     "url": "https://omlx.ai/benchmarks/performance/xiuzsnqb",
     "pp": 858.1,
     "pp_metric": "prefill at 1k context",
     "source": "omlx",
     "quant": "MLX 4-bit (MXFP4-Q8)",
     "runs": 2,
     "range": [
      97.2,
      104.3
     ]
    }
   }
  },
  "mid": {
   "short": "Qwen3-Coder-30B-A3B",
   "model": "qwen3-coder-30b-a3b",
   "metric": "Qwen3-Coder-30B-A3B (MoE, 3B active), text generation tok/s at short context, single request. Engine and quantization differ per source: llama.cpp Q8_0 on GB10, llama.cpp UD-Q4_K_XL on Strix Halo, MLX 4-bit via oMLX on Apple, so compare with care.",
   "why": "Chosen over gpt-oss-20b because it has public results for more of our machines (Strix Halo incl. 64GB boxes, and GB10). gpt-oss-20b has results only for GB10 and M5 Max.",
   "min_mem_gb": 32,
   "chips": {
    "gb10": {
     "label": "NVIDIA GB10 (DGX Spark)",
     "tps": 61.06,
     "pp": 2986.97,
     "pp_metric": "pp2048",
     "engine": "llama.cpp CUDA llama-bench tg32",
     "date": "2026-02-05",
     "url": "https://github.com/ggml-org/llama.cpp/blob/master/benches/dgx-spark/dgx-spark.md",
     "source": "llamacpp",
     "quant": "Q8_0"
    },
    "strix-halo": {
     "label": "AMD Ryzen AI Max+ 395 (Beelink GTR9 Pro)",
     "tps": 94.64,
     "pp": 1264.16,
     "pp_metric": "pp512",
     "engine": "llama.cpp Vulkan llama-bench tg128, b10687, 20 repeats",
     "date": "2026-08-30",
     "url": "https://github.com/hogeheer499-commits/strix-halo-guide/blob/HEAD/data/benchmarks.csv",
     "source": "shg",
     "quant": "UD-Q4_K_XL"
    },
    "m5-max-40": {
     "label": "Apple M5 Max 40-core GPU (run on a 64GB machine)",
     "tps": 120.8,
     "pp": 3963.0,
     "pp_metric": "prefill at 1k context",
     "engine": "oMLX, 1k context, median of 4 community runs",
     "date": "2026-08-26",
     "url": "https://omlx.ai/benchmarks/performance/wni9l2ia",
     "source": "omlx",
     "quant": "MLX 4-bit",
     "runs": 4,
     "range": [
      119.5,
      125
     ]
    },
    "m5-pro-20": {
     "label": "Apple M5 Pro 20-core GPU (run on a 48GB machine)",
     "tps": 88.9,
     "pp": 1181.0,
     "pp_metric": "prefill at 1k context",
     "engine": "oMLX, 1k context, median of 6 community runs",
     "date": "2026-10-04",
     "url": "https://omlx.ai/benchmarks/performance/0l7gbufd",
     "source": "omlx",
     "quant": "MLX 4-bit",
     "runs": 6,
     "range": [
      87.8,
      95
     ]
    },
    "m5-pro-16": {
     "label": "Apple M5 Pro 16-core GPU (run on a 48GB machine)",
     "tps": 92.0,
     "pp": 2056.0,
     "pp_metric": "prefill at 1k context",
     "engine": "oMLX, 1k context, median of 2 community runs",
     "date": "2026-10-09",
     "url": "https://omlx.ai/benchmarks/performance/0yoseyxv",
     "source": "omlx",
     "quant": "MLX 4-bit",
     "runs": 2,
     "range": [
      92,
      94.1
     ]
    }
   }
  }
 },
 "llm_speed": {
  "source": "llm-speed.com",
  "url": "https://llm-speed.com/data",
  "license": "CC BY 4.0",
  "license_url": "https://creativecommons.org/licenses/by/4.0/",
  "changes": "Aggregated to median top_decode_tps per accelerator; mixed models, runtimes and workloads.",
  "imported": "2026-10-16",
  "groups": [
   {
    "accelerator": "RTX 5090",
    "runs": 165,
    "median_decode_tps": 71.3,
    "models": [
     "Codestral-22B-v0.1",
     "DeepSeek-Coder-V2-Lite-Instruct",
     "Llama-3.1-8B-Instruct",
     "Qwen2.5-14B-Instruct",
     "Qwen2.5-32B-Instruct",
     "Qwen2.5-7B-Instruct",
     "Qwen2.5-Coder-14B-Instruct",
     "Qwen2.5-Coder-32B-Instruct"
    ],
    "example_permalink": "https://llm-speed.com/r/r_bfpto9so2o1"
   },
   {
    "accelerator": "M3 Ultra (60-core GPU)",
    "runs": 21,
    "median_decode_tps": 80.3,
    "models": [
     "lmstudio-community-Qwen3-Next-80B-A3B-Instruct-MLX-4bit",
     "mlx-community-Qwen2.5-14B-Instruct-4bit",
     "mlx-community-Qwen2.5-32B-Instruct-4bit",
     "mlx-community-Qwen2.5-7B-Instruct-4bit",
     "mlx-community/Codestral-22B-v0.1-4bit",
     "mlx-community/DeepSeek-Coder-V2-Lite-Instruct-4bit",
     "mlx-community/Llama-3.1-8B-Instruct-4bit",
     "mlx-community/Qwen2.5-Coder-14B-Instruct-4bit"
    ],
    "example_permalink": "https://llm-speed.com/r/r_1pl79r50ofy"
   },
   {
    "accelerator": "RTX 4090",
    "runs": 14,
    "median_decode_tps": 131.1,
    "models": [
     "deepseek-r1",
     "gemma3",
     "glm-4.7-flash",
     "gpt-oss",
     "llama3.1",
     "qwen2.5-coder",
     "qwen3-coder",
     "qwen3.6"
    ],
    "example_permalink": "https://llm-speed.com/r/r_o636l3cc-rr"
   },
   {
    "accelerator": "M3 Pro (18-core GPU)",
    "runs": 11,
    "median_decode_tps": 19.4,
    "models": [
     "mlx-community-Qwen2.5-0.5B-Instruct-4bit",
     "mlx-community-Qwen2.5-14B-Instruct-4bit",
     "mlx-community-Qwen2.5-7B-Instruct-4bit",
     "mlx-community/Llama-3.1-8B-Instruct-4bit",
     "mlx-community/Qwen2.5-32B-Instruct-4bit",
     "mlx-community/Qwen3-32B-4bit",
     "mlx-community/gemma-2-9b-it-4bit",
     "mlx-community/phi-4-4bit"
    ],
    "example_permalink": "https://llm-speed.com/r/r_pnrrpcdqfo4"
   },
   {
    "accelerator": "M4 Max (40-core GPU)",
    "runs": 5,
    "median_decode_tps": 109.1,
    "models": [
     "qwen3-coder",
     "qwen3-coder-bench-32k"
    ],
    "example_permalink": "https://llm-speed.com/r/r_roktphpc--8"
   },
   {
    "accelerator": "RTX 3090",
    "runs": 4,
    "median_decode_tps": 137.7,
    "models": [
     "deepseek-coder-v2",
     "llama3.1",
     "qwen2.5-coder"
    ],
    "example_permalink": "https://llm-speed.com/r/r_o2-1w665rtq"
   }
  ]
 }
}