You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 

1 lines
82 KiB

<!doctypehtml><html lang=zh-CN><meta charset=UTF-8><meta content=width=device-width name=viewport><meta media="(prefers-color-scheme: light)" content=#222 name=theme-color><meta media="(prefers-color-scheme: dark)" content=#222 name=theme-color><meta content="Hexo 7.3.0" name=generator><link href=https://cdnjs.cloudflare.com rel=preconnect><link href=/images/apple-touch-icon-next.png rel=apple-touch-icon sizes=180x180><link href=/images/favicon-32x32-next.png rel=icon sizes=32x32 type=image/png><link href=/images/favicon-16x16-next.png rel=icon sizes=16x16 type=image/png><link color=#222 href=/images/logo.svg rel=mask-icon><meta content=2X6S9P7CAjXjVvw8YyQR8pCu-B0oEu7O9quLgxXuWyA name=google-site-verification><meta content=dV8JGNzi0c name=baidu-site-verification><script>!function(){var t=!1;function e(){var e;t||(t=!0,(e=document.createElement("script")).setAttribute("data-ad-client","ca-pub-7480618470784058"),e.async=!0,e.src="https://pagead2.googlesyndication.com/pagead/js/adsbygoogle.js",document.head.appendChild(e))}function a(){"requestIdleCallback"in window?window.requestIdleCallback(e,{timeout:5e3}):window.setTimeout(e,5e3)}"complete"===document.readyState?a():window.addEventListener("load",a,{once:!0})}()</script><link href=/css/main.css rel=stylesheet><link crossorigin=anonymous href=https://cdnjs.cloudflare.com/ajax/libs/font-awesome/7.0.0/css/all.min.css integrity=sha256-VHqXKFhhMxcpubYf9xiWdCiojEbY9NexQ4jh8AxbvcM= rel=stylesheet><link crossorigin=anonymous href=https://cdnjs.cloudflare.com/ajax/libs/fancyapps-ui/6.1.14/fancybox/fancybox.css integrity=sha256-mdaMxrItEW6hMIVJJNPKuX0B7KCSJQVEv3mVY4N2we4= rel=stylesheet><script class=next-config data-name=main type=application/json>{"hostname":"nicksxs.me","root":"/","images":"/images","scheme":"Pisces","darkmode":true,"version":"8.29.0","exturl":false,"sidebar":{"position":"left","width_expanded":320,"width_dual_column":240,"display":"post","padding":18,"offset":12},"hljswrap":true,"codeblock":{"theme":{"light":"default","dark":"stackoverflow-dark"},"prism":{"light":"prism","dark":"prism-dark"},"copy_button":{"enable":true,"style":"default"},"fold":{"enable":false,"height":500},"language":false},"bookmark":{"enable":false,"color":"#222","save":"auto"},"mediumzoom":false,"pangu":false,"comments":{"style":"tabs","active":null,"storage":true,"lazyload":true,"nav":null},"stickytabs":false,"motion":{"enable":false,"async":false,"duration":200,"transition":{"menu_item":"fadeInDown","post_block":"fadeIn","post_header":"fadeInDown","post_body":"fadeInDown","coll_header":"fadeInLeft","sidebar":"fadeInUp"}},"prism":false,"i18n":{"placeholder":"搜索...","empty":"没有找到任何搜索结果:${query}","hits_time":"找到 ${hits} 个搜索结果(用时 ${time} 毫秒)","hits":"找到 ${hits} 个搜索结果"},"algolia":{"appID":"663Q99OQQD","apiKey":"e461a1371d12cec1491c1153b288a9ed","indexName":"nicksxs","hits":{"per_page":10}}}</script><script defer src=/js/config.js></script><meta content=6GB显存可以尝试运行Qwen3.6-35B-A3B-NVFP4,但代价是更小的专家缓存和KV容量,以及对32GB以上内存、CPU与PCIe带宽的更高依赖。本文给出已验证配置、调优顺序、压测脚本和明确限制。 name=description><meta content=article property=og:type><meta content="6GB显存能跑35B MoE吗:FreeToken极限配置、性能压榨与实操教程" property=og:title><meta content=https://nicksxs.me/2026/09/05/6GB%E6%98%BE%E5%AD%98%E8%B7%9135B-MoE-FreeToken%E6%9E%81%E9%99%90%E9%85%8D%E7%BD%AE%E4%B8%8E%E5%AE%9E%E6%93%8D%E6%95%99%E7%A8%8B/index.html property=og:url><meta content="Nicksxs's Blog" property=og:site_name><meta content=6GB显存可以尝试运行Qwen3.6-35B-A3B-NVFP4,但代价是更小的专家缓存和KV容量,以及对32GB以上内存、CPU与PCIe带宽的更高依赖。本文给出已验证配置、调优顺序、压测脚本和明确限制。 property=og:description><meta content=zh_CN property=og:locale><meta content=https://nicksxs.me/uploads/freetoken-6gb/vram-budget.png property=og:image><meta content=2026-09-05T02:00:00.000Z property=article:published_time><meta content=2026-09-05T02:00:00.000Z property=article:modified_time><meta content=Nicksxs property=article:author><meta content=LLM property=article:tag><meta content=FreeToken property=article:tag><meta content=CUDA property=article:tag><meta content=本地大模型 property=article:tag><meta content=summary name=twitter:card><meta content=https://nicksxs.me/uploads/freetoken-6gb/vram-budget.png name=twitter:image><link href=https://nicksxs.me/2026/09/05/6GB%E6%98%BE%E5%AD%98%E8%B7%9135B-MoE-FreeToken%E6%9E%81%E9%99%90%E9%85%8D%E7%BD%AE%E4%B8%8E%E5%AE%9E%E6%93%8D%E6%95%99%E7%A8%8B/ rel=canonical><script class=next-config data-name=page type=application/json>{"sidebar":"","isHome":false,"isPost":true,"lang":"zh-CN","comments":true,"permalink":"https://nicksxs.me/2026/09/05/6GB%E6%98%BE%E5%AD%98%E8%B7%9135B-MoE-FreeToken%E6%9E%81%E9%99%90%E9%85%8D%E7%BD%AE%E4%B8%8E%E5%AE%9E%E6%93%8D%E6%95%99%E7%A8%8B/","path":"2026/09/05/6GB显存跑35B-MoE-FreeToken极限配置与实操教程/","title":"6GB显存能跑35B MoE吗:FreeToken极限配置、性能压榨与实操教程"}</script><script class=next-config data-name=calendar type=application/json>""</script><title>6GB显存能跑35B MoE吗:FreeToken极限配置、性能压榨与实操教程 | Nicksxs's Blog</title><script async src=https://www.googletagmanager.com/gtag/js?id=UA-61358619-1></script><script class=next-config data-name=google_analytics type=application/json>{"tracking_id":"UA-61358619-1","only_pageview":false,"measure_protocol_api_secret":null}</script><script defer src=/js/third-party/analytics/google-analytics.js></script><script defer src=/js/third-party/analytics/baidu-analytics.js></script><script async src=https://hm.baidu.com/hm.js?20f33b3c0c0eff9b1522999c0015646d></script><script crossorigin=anonymous defer integrity=sha256-QnJV4HT5r8i89ovZGRD86iynZqHerEbT2xt69vDM36I= src=https://cdnjs.cloudflare.com/ajax/libs/fancyapps-ui/6.1.14/fancybox/fancybox.umd.js></script><script defer src=/js/utils.js></script><script defer src=/js/sidebar.js></script><script defer src=/js/next-boot.js></script><script data-algolia-integrity=sha256-o49zoHLDPBm9WLg6AiliAVi3axCRTqe3ltTxLvhzAVg= data-algolia-src=https://cdnjs.cloudflare.com/ajax/libs/algoliasearch/5.36.0/lite/builds/browser.umd.js data-search-src=/js/third-party/search/algolia-search.js src=/js/third-party/search/algolia-search-loader.js></script><script defer src=/js/third-party/fancybox.js></script><noscript><link href=/css/noscript.css rel=stylesheet></noscript><link title="Nicksxs's Blog" href=/atom.xml rel=alternate type=application/atom+xml><body itemscope itemtype=http://schema.org/WebPage><div class=headband></div><main class=main><div class=column><header class=header itemscope itemtype=http://schema.org/WPHeader><div class=site-brand-container><div class=site-nav-toggle><div aria-label=切换导航栏 class=toggle role=button><span class=toggle-line></span><span class=toggle-line></span><span class=toggle-line></span></div></div><div class=site-meta><a class=brand href=/ rel=start><i class=logo-line></i><p class=site-title>Nicksxs's Blog</p><i class=logo-line></i></a><p class=site-subtitle itemprop=description>Java 运行时、生产诊断与 AI 工程实践</div><div class=site-nav-right><div class="toggle popup-trigger" aria-label=搜索 role=button><i class="fa fa-search fa-fw fa-lg"></i></div></div></div><nav class=site-nav><ul class="main-menu menu"><li class="menu-item menu-item-home"><a href=/ rel=section><i class="fa fa-home fa-fw"></i>首页</a><li class="menu-item menu-item-专题"><a href=/topics/ rel=section><i class="fa fa-layer-group fa-fw"></i>专题</a><li class="menu-item menu-item-工具"><a href=/tools/ rel=section><i class="fa fa-toolbox fa-fw"></i>工具</a><li class="menu-item menu-item-about"><a href=/about/ rel=section><i class="fa fa-user fa-fw"></i>关于我</a><li class="menu-item menu-item-tags"><a href=/tags/ rel=section><i class="fa fa-tags fa-fw"></i>标签</a><li class="menu-item menu-item-categories"><a href=/categories/ rel=section><i class="fa fa-th fa-fw"></i>分类</a><li class="menu-item menu-item-archives"><a href=/archives/ rel=section><i class="fa fa-archive fa-fw"></i>归档</a><li class="menu-item menu-item-search"><a class=popup-trigger role=button><i class="fa fa-search fa-fw"></i>搜索</a></ul></nav><div class=search-pop-overlay><div class="popup search-popup"><div class=search-header><span class=search-icon><i class="fa fa-search"></i></span><div class=search-input-container><input autocapitalize=off autocomplete=off class=search-input maxlength=80 placeholder=搜索... spellcheck=false type=search></div><span class=popup-btn-close role=button><i class="fa fa-times-circle"></i></span></div><div class=search-result-container><div class=search-result-icon><i class="fab fa-algolia fa-5x"></i></div></div></div></div></header><aside class=sidebar><div class="sidebar-inner sidebar-nav-active sidebar-toc-active"><ul class=sidebar-nav><li class=sidebar-nav-toc>文章目录<li class=sidebar-nav-overview>站点概览</ul><div class=sidebar-panel-container><div class="post-toc-wrap sidebar-panel"><div class="post-toc animated"><ol class=nav><li class="nav-item nav-level-2"><a class=nav-link href=#%E5%85%88%E7%9C%8B%E7%BB%93%E8%AE%BA%E8%A1%A8><span class=nav-number>1.</span> <span class=nav-text>先看结论表</span></a><li class="nav-item nav-level-2"><a class=nav-link href=#6GB-%E7%9A%84%E7%9B%B4%E6%8E%A5%E8%AF%81%E6%8D%AE%E6%98%AF%E4%BB%80%E4%B9%88><span class=nav-number>2.</span> <span class=nav-text>6GB 的直接证据是什么</span></a><li class="nav-item nav-level-2"><a class=nav-link href=#%E4%B8%BA%E4%BB%80%E4%B9%88%E4%BB%8E-8GB-%E5%87%8F%E5%88%B0-6GB-%E4%BC%9A%E6%98%8E%E6%98%BE%E5%8F%98%E9%9A%BE><span class=nav-number>3.</span> <span class=nav-text>为什么从 8GB 减到 6GB 会明显变难</span></a><ol class=nav-child><li class="nav-item nav-level-3"><a class=nav-link href=#%E9%9D%9E%E4%B8%93%E5%AE%B6%E6%9D%83%E9%87%8D%E6%98%AF%E7%A1%AC%E6%88%90%E6%9C%AC><span class=nav-number>3.1.</span> <span class=nav-text>非专家权重是硬成本</span></a><li class="nav-item nav-level-3"><a class=nav-link href=#%E4%B8%93%E5%AE%B6%E7%BC%93%E5%AD%98%E5%86%B3%E5%AE%9A-Decode-%E5%91%BD%E4%B8%AD%E7%8E%87><span class=nav-number>3.2.</span> <span class=nav-text>专家缓存决定 Decode 命中率</span></a><li class="nav-item nav-level-3"><a class=nav-link href=#KV-Cache-%E4%B8%8E%E4%B8%93%E5%AE%B6%E7%BC%93%E5%AD%98%E4%BA%89%E5%90%8C%E4%B8%80%E5%9D%97%E6%98%BE%E5%AD%98><span class=nav-number>3.3.</span> <span class=nav-text>KV Cache 与专家缓存争同一块显存</span></a><li class="nav-item nav-level-3"><a class=nav-link href=#%E8%BF%90%E8%A1%8C%E6%97%B6%E5%B7%A5%E4%BD%9C%E5%8C%BA%E4%B8%8D%E4%BC%9A%E6%98%BE%E7%A4%BA%E5%9C%A8%E6%A8%A1%E5%9E%8B%E6%96%87%E4%BB%B6%E5%A4%A7%E5%B0%8F%E9%87%8C><span class=nav-number>3.4.</span> <span class=nav-text>运行时工作区不会显示在模型文件大小里</span></a></ol><li class="nav-item nav-level-2"><a class=nav-link href=#%E5%86%85%E5%AD%98%E5%88%B0%E5%BA%95%E8%A6%81%E5%A4%9A%E5%A4%A7><span class=nav-number>4.</span> <span class=nav-text>内存到底要多大</span></a><li class="nav-item nav-level-2"><a class=nav-link href=#%E7%A1%AC%E4%BB%B6%E4%B8%8E%E7%B3%BB%E7%BB%9F%E6%A3%80%E6%9F%A5><span class=nav-number>5.</span> <span class=nav-text>硬件与系统检查</span></a><ol class=nav-child><li class="nav-item nav-level-3"><a class=nav-link href=#%E6%98%BE%E5%8D%A1%E4%B8%8D%E8%83%BD%E5%8F%AA%E7%9C%8B-6GB-%E8%BF%99%E4%B8%AA%E6%95%B0%E5%AD%97><span class=nav-number>5.1.</span> <span class=nav-text>显卡不能只看 6GB 这个数字</span></a><li class="nav-item nav-level-3"><a class=nav-link href=#%E5%BC%80%E5%A7%8B%E5%89%8D%E8%BF%90%E8%A1%8C%E8%BF%99%E4%BA%9B%E6%A3%80%E6%9F%A5><span class=nav-number>5.2.</span> <span class=nav-text>开始前运行这些检查</span></a></ol><li class="nav-item nav-level-2"><a class=nav-link href=#%E4%BB%8E%E9%9B%B6%E5%AE%89%E8%A3%85><span class=nav-number>6.</span> <span class=nav-text>从零安装</span></a><li class="nav-item nav-level-2"><a class=nav-link href=#%E6%A8%A1%E5%9E%8B%E5%BA%94%E8%AF%A5%E4%B8%8B%E8%BD%BD%E5%93%AA%E4%B8%80%E4%B8%AA><span class=nav-number>7.</span> <span class=nav-text>模型应该下载哪一个</span></a><ol class=nav-child><li class="nav-item nav-level-3"><a class=nav-link href=#%E6%96%B9%E6%B3%95%E4%B8%80%EF%BC%9A%E8%AE%A9-FreeToken-%E8%87%AA%E5%8A%A8%E4%B8%8B%E8%BD%BD><span class=nav-number>7.1.</span> <span class=nav-text>方法一:让 FreeToken 自动下载</span></a><li class="nav-item nav-level-3"><a class=nav-link href=#%E6%96%B9%E6%B3%95%E4%BA%8C%EF%BC%9A%E5%85%88%E5%AE%8C%E6%95%B4%E4%B8%8B%E8%BD%BD%EF%BC%8C%E5%86%8D%E4%BB%8E%E6%9C%AC%E5%9C%B0%E7%9B%AE%E5%BD%95%E5%90%AF%E5%8A%A8><span class=nav-number>7.2.</span> <span class=nav-text>方法二:先完整下载,再从本地目录启动</span></a></ol><li class="nav-item nav-level-2"><a class=nav-link href=#%E7%AC%AC%E4%B8%80%E6%AD%A5%EF%BC%9A%E5%85%88%E6%B5%8B%E8%BF%99%E5%8F%B0%E6%9C%BA%E5%99%A8%E7%9A%84%E5%B8%A6%E5%AE%BD><span class=nav-number>8.</span> <span class=nav-text>第一步:先测这台机器的带宽</span></a><li class="nav-item nav-level-2"><a class=nav-link href=#%E7%AC%AC%E4%BA%8C%E6%AD%A5%EF%BC%9A%E4%BD%BF%E7%94%A8-6GB-%E4%BF%9D%E5%AE%88%E9%85%8D%E7%BD%AE%E5%90%AF%E5%8A%A8><span class=nav-number>9.</span> <span class=nav-text>第二步:使用 6GB 保守配置启动</span></a><li class="nav-item nav-level-2"><a class=nav-link href=#%E7%AC%AC%E4%B8%89%E6%AD%A5%EF%BC%9A%E6%A3%80%E6%9F%A5%E6%98%AF%E5%90%A6%E7%9C%9F%E7%9A%84%E5%90%AF%E5%8A%A8%E6%AD%A3%E7%A1%AE><span class=nav-number>10.</span> <span class=nav-text>第三步:检查是否真的启动正确</span></a><li class="nav-item nav-level-2"><a class=nav-link href=#%E7%AC%AC%E5%9B%9B%E6%AD%A5%EF%BC%9A%E5%85%88%E5%81%9A%E6%9C%80%E5%B0%8F%E5%86%92%E7%83%9F%E6%B5%8B%E8%AF%95><span class=nav-number>11.</span> <span class=nav-text>第四步:先做最小冒烟测试</span></a><li class="nav-item nav-level-2"><a class=nav-link href=#%E7%AC%AC%E4%BA%94%E6%AD%A5%EF%BC%9A%E9%80%90%E7%BA%A7%E5%A2%9E%E5%8A%A0%E4%B8%93%E5%AE%B6%E7%BC%93%E5%AD%98><span class=nav-number>12.</span> <span class=nav-text>第五步:逐级增加专家缓存</span></a><li class="nav-item nav-level-2"><a class=nav-link href=#%E7%AC%AC%E5%85%AD%E6%AD%A5%EF%BC%9A%E6%8C%89%E5%B7%A5%E4%BD%9C%E8%B4%9F%E8%BD%BD%E5%8F%8D%E5%90%91%E5%88%86%E9%85%8D-KV><span class=nav-number>13.</span> <span class=nav-text>第六步:按工作负载反向分配 KV</span></a><ol class=nav-child><li class="nav-item nav-level-3"><a class=nav-link href=#%E5%8F%AA%E5%81%9A%E7%9F%AD%E5%AF%B9%E8%AF%9D%E6%88%96%E4%BB%A3%E7%A0%81%E8%A1%A5%E5%85%A8><span class=nav-number>13.1.</span> <span class=nav-text>只做短对话或代码补全</span></a><li class="nav-item nav-level-3"><a class=nav-link href=#%E5%81%9A-8K-%E5%B7%A6%E5%8F%B3%E7%9A%84-Coding-Agent><span class=nav-number>13.2.</span> <span class=nav-text>做 8K 左右的 Coding Agent</span></a><li class="nav-item nav-level-3"><a class=nav-link href=#%E4%B8%8D%E8%A6%81%E4%BB%8E-32K-%E6%88%96-128K-%E5%BC%80%E5%A7%8B><span class=nav-number>13.3.</span> <span class=nav-text>不要从 32K 或 128K 开始</span></a></ol><li class="nav-item nav-level-2"><a class=nav-link href=#%E6%80%8E%E6%A0%B7%E5%88%A4%E6%96%AD%E8%BF%98%E5%BA%94%E8%AF%A5%E7%BB%A7%E7%BB%AD%E5%8A%A0-expert-slots><span class=nav-number>14.</span> <span class=nav-text>怎样判断还应该继续加 expert slots</span></a><li class="nav-item nav-level-2"><a class=nav-link href=#%E6%AF%94%E8%BE%83-auto%E3%80%81offload%E3%80%81cpu-%E5%92%8C-hybrid><span class=nav-number>15.</span> <span class=nav-text>比较 auto、offload、cpu 和 hybrid</span></a><li class="nav-item nav-level-2"><a class=nav-link href=#%E4%B8%80%E9%94%AE%E5%AE%9E%E9%AA%8C%E8%84%9A%E6%9C%AC><span class=nav-number>16.</span> <span class=nav-text>一键实验脚本</span></a><li class="nav-item nav-level-2"><a class=nav-link href=#%E5%B8%B8%E8%A7%81%E9%97%AE%E9%A2%98%E4%B8%8E%E6%8E%92%E6%9F%A5%E9%A1%BA%E5%BA%8F><span class=nav-number>17.</span> <span class=nav-text>常见问题与排查顺序</span></a><ol class=nav-child><li class="nav-item nav-level-3"><a class=nav-link href=#%E5%90%AF%E5%8A%A8%E6%97%B6%E7%94%B3%E8%AF%B7-256MiB-%E5%A4%B1%E8%B4%A5><span class=nav-number>17.1.</span> <span class=nav-text>启动时申请 256MiB 失败</span></a><li class="nav-item nav-level-3"><a class=nav-link href=#%E6%A8%A1%E5%9E%8B%E5%8A%A0%E8%BD%BD%E6%97%B6%E7%B3%BB%E7%BB%9F%E5%86%85%E5%AD%98%E4%B8%8D%E5%A4%9F><span class=nav-number>17.2.</span> <span class=nav-text>模型加载时系统内存不够</span></a><li class="nav-item nav-level-3"><a class=nav-link href=#%E8%83%BD%E5%90%AF%E5%8A%A8%EF%BC%8C%E4%BD%86%E9%95%BF-Prompt-%E5%8D%A1%E4%BD%8F><span class=nav-number>17.3.</span> <span class=nav-text>能启动,但长 Prompt 卡住</span></a><li class="nav-item nav-level-3"><a class=nav-link href=#Decode-%E5%BE%88%E6%85%A2%EF%BC%8CCPU-%E6%BB%A1%E8%BD%BD><span class=nav-number>17.4.</span> <span class=nav-text>Decode 很慢,CPU 满载</span></a><li class="nav-item nav-level-3"><a class=nav-link href=#%E6%98%BE%E5%AD%98%E7%9C%8B%E8%B5%B7%E6%9D%A5%E8%BF%98%E6%9C%89%E5%87%A0%E7%99%BE-MiB%EF%BC%8C%E4%BB%8D%E7%84%B6-OOM><span class=nav-number>17.5.</span> <span class=nav-text>显存看起来还有几百 MiB,仍然 OOM</span></a></ol><li class="nav-item nav-level-2"><a class=nav-link href=#6GB-%E7%9A%84%E6%98%8E%E7%A1%AE%E9%99%90%E5%88%B6><span class=nav-number>18.</span> <span class=nav-text>6GB 的明确限制</span></a><ol class=nav-child><li class="nav-item nav-level-3"><a class=nav-link href=#1-%E5%AE%83%E4%BB%8D%E7%84%B6%E9%9C%80%E8%A6%81%E5%A4%A7%E5%86%85%E5%AD%98><span class=nav-number>18.1.</span> <span class=nav-text>1. 它仍然需要大内存</span></a><li class="nav-item nav-level-3"><a class=nav-link href=#2-6GB-%E4%B8%8D%E6%98%AF%E6%89%80%E6%9C%89-6GB-%E6%98%BE%E5%8D%A1><span class=nav-number>18.2.</span> <span class=nav-text>2. 6GB 不是所有 6GB 显卡</span></a><li class="nav-item nav-level-3"><a class=nav-link href=#3-%E4%B8%8D%E8%83%BD%E6%8A%8A-39-3-tok-s-%E6%90%AC%E8%BF%87%E6%9D%A5><span class=nav-number>18.3.</span> <span class=nav-text>3. 不能把 39.3 tok/s 搬过来</span></a><li class="nav-item nav-level-3"><a class=nav-link href=#4-%E9%95%BF%E4%B8%8A%E4%B8%8B%E6%96%87%E5%92%8C%E4%B8%93%E5%AE%B6%E5%91%BD%E4%B8%AD%E7%8E%87%E7%9B%B4%E6%8E%A5%E7%AB%9E%E4%BA%89><span class=nav-number>18.4.</span> <span class=nav-text>4. 长上下文和专家命中率直接竞争</span></a><li class="nav-item nav-level-3"><a class=nav-link href=#5-Prefill-%E5%92%8C-Decode-%E6%98%AF%E4%B8%A4%E7%A7%8D%E7%93%B6%E9%A2%88><span class=nav-number>18.5.</span> <span class=nav-text>5. Prefill 和 Decode 是两种瓶颈</span></a><li class="nav-item nav-level-3"><a class=nav-link href=#6-%E6%95%B4%E6%9C%BA%E5%86%B3%E5%AE%9A%E7%BB%93%E6%9E%9C><span class=nav-number>18.6.</span> <span class=nav-text>6. 整机决定结果</span></a><li class="nav-item nav-level-3"><a class=nav-link href=#7-%E5%BD%93%E5%89%8D%E8%BD%AF%E4%BB%B6%E4%BB%8D%E5%9C%A8%E5%BF%AB%E9%80%9F%E5%8F%98%E5%8C%96><span class=nav-number>18.7.</span> <span class=nav-text>7. 当前软件仍在快速变化</span></a></ol><li class="nav-item nav-level-2"><a class=nav-link href=#%E6%88%91%E4%BC%9A%E6%80%8E%E6%A0%B7%E9%80%89%E6%8B%A9%E7%A1%AC%E4%BB%B6><span class=nav-number>19.</span> <span class=nav-text>我会怎样选择硬件</span></a><li class="nav-item nav-level-2"><a class=nav-link href=#%E6%9C%80%E5%90%8E%E6%80%BB%E7%BB%93><span class=nav-number>20.</span> <span class=nav-text>最后总结</span></a><li class="nav-item nav-level-2"><a class=nav-link href=#%E5%8F%82%E8%80%83%E8%B5%84%E6%96%99><span class=nav-number>21.</span> <span class=nav-text>参考资料</span></a></ol></div></div><div class="site-overview-wrap sidebar-panel"><div class="site-author animated" itemprop=author itemscope itemtype=http://schema.org/Person><img alt=Nicksxs class=site-author-image itemprop=image src=/uploads/avatar.jpg><p class=site-author-name itemprop=name>Nicksxs<div class=site-description itemprop=description>Nicksxs 的技术博客,深入 Java 运行时与生产诊断、低成本大模型推理和 Agent 工程实践。</div></div><div class="site-state-wrap animated"><nav class=site-state><div class="site-state-item site-state-posts"><a href=/archives/><span class=site-state-item-count>394</span> <span class=site-state-item-name>日志</span></a></div><div class="site-state-item site-state-categories"><a href=/categories/><span class=site-state-item-count>181</span> <span class=site-state-item-name>分类</span></a></div><div class="site-state-item site-state-tags"><a href=/tags/><span class=site-state-item-count>347</span> <span class=site-state-item-name>标签</span></a></div></nav></div><div class="links-of-author animated"><span class=links-of-author-item><a rel="noopener me" title="GitHub → https://github.com/nicksxs" href=https://github.com/nicksxs target=_blank><i class="fab fa-github fa-fw"></i>GitHub</a> </span><span class=links-of-author-item><a rel="noopener me" title="E-Mail → mailto:nicksxs1202@gmail.com" href=mailto:nicksxs1202@gmail.com target=_blank><i class="fa fa-envelope fa-fw"></i>E-Mail</a></span></div><div class="cc-license animated" itemprop=license><a class=cc-opacity href=https://creativecommons.org/licenses/by-nc-sa/4.0/ rel=noopener target=_blank><img alt="Creative Commons" src=https://cdnjs.cloudflare.com/ajax/libs/creativecommons-vocabulary/2020.11.3/assets/license_badges/small/by_nc_sa.svg></a></div><div class="widget-wrap sidebar-tag-cloud"><div class=tag-cloud-tags><a class=tag-cloud-10 href=/tags/Java/ style=font-size:16px>Java</a><a class=tag-cloud-9 href=/tags/leetcode/ style=font-size:15.62px>leetcode</a><a class=tag-cloud-8 href=/tags/LLM/ style=font-size:15.23px>LLM</a><a class=tag-cloud-8 href=/tags/%E7%94%9F%E6%B4%BB/ style=font-size:14.85px>生活</a><a class=tag-cloud-7 href=/tags/%E9%A2%98%E8%A7%A3/ style=font-size:14.46px>题解</a><a class=tag-cloud-6 href=/tags/SpringBoot/ style=font-size:14.08px>SpringBoot</a><a class=tag-cloud-6 href=/tags/MySQL/ style=font-size:14.08px>MySQL</a><a class=tag-cloud-5 href=/tags/C/ style=font-size:13.69px>C++</a><a class=tag-cloud-5 href=/tags/Redis/ style=font-size:13.31px>Redis</a><a class=tag-cloud-4 href=/tags/Tomcat/ style=font-size:12.92px>Tomcat</a><a class=tag-cloud-4 href=/tags/Mybatis/ style=font-size:12.92px>Mybatis</a><a class=tag-cloud-3 href=/tags/%E5%B0%8F%E6%8A%80%E5%B7%A7/ style=font-size:12.54px>小技巧</a><a class=tag-cloud-2 href=/tags/%E6%95%B0%E6%8D%AE%E7%BB%93%E6%9E%84/ style=font-size:12.15px>数据结构</a><a class=tag-cloud-2 href=/tags/%E6%BA%90%E7%A0%81/ style=font-size:12.15px>源码</a><a class=tag-cloud-2 href=/tags/MQ/ style=font-size:11.77px>MQ</a><a class=tag-cloud-2 href=/tags/%E6%B6%88%E6%81%AF%E9%98%9F%E5%88%97/ style=font-size:11.77px>消息队列</a><a class=tag-cloud-2 href=/tags/RocketMQ/ style=font-size:11.77px>RocketMQ</a><a class=tag-cloud-2 href=/tags/%E8%BF%90%E5%8A%A8/ style=font-size:11.77px>运动</a><a class=tag-cloud-1 href=/tags/php/ style=font-size:11.38px>php</a><a class=tag-cloud-0 href=/tags/Spring/ style=font-size:11px>Spring</a><a class=tag-cloud-0 href=/tags/linux/ style=font-size:11px>linux</a><a class=tag-cloud-0 href=/tags/Docker/ style=font-size:11px>Docker</a><a class=tag-cloud-0 href=/tags/%E5%87%8F%E8%82%A5/ style=font-size:11px>减肥</a><a class=tag-cloud-0 href=/tags/%E8%B7%91%E6%AD%A5/ style=font-size:11px>跑步</a></div></div><div class="affiliate-creative affiliate-creative--sidebar" data-affiliate-id=tencent-cloud-server><a rel="sponsored noopener" aria-label=腾讯云服务器优惠 href=https://cloud.tencent.com/act/cps/redirect?redirect=6544&cps_key=2e33a1eedb380997d03618b4a5ba6ca7&from=console target=_blank><img alt="腾讯云 AI 焕新云服务器优惠" decoding=async height=241 loading=lazy src=/uploads/affiliate/tencent-cloud-server.webp width=690> </a><span class=affiliate-disclosure>推广</span></div></div></div></div><div class="sidebar-inner sidebar-blogroll"><div class="links-of-blogroll animated"><div class=links-of-blogroll-title><i class="fa fa-globe fa-fw"></i> 链接</div><ul class=links-of-blogroll-list><li class=links-of-blogroll-item><a href=https://covermusic.cn/ rel=noopener target=_blank title=https://covermusic.cn>69伙伴</a></ul></div></div></aside></div><div class="main-inner post posts-expand"><div class=post-block><article class=post-content itemscope itemtype=http://schema.org/Article lang=zh-CN><link href=https://nicksxs.me/2026/09/05/6GB%E6%98%BE%E5%AD%98%E8%B7%9135B-MoE-FreeToken%E6%9E%81%E9%99%90%E9%85%8D%E7%BD%AE%E4%B8%8E%E5%AE%9E%E6%93%8D%E6%95%99%E7%A8%8B/ itemprop=mainEntityOfPage><span hidden itemprop=author itemscope itemtype=http://schema.org/Person><meta content=/uploads/avatar.jpg itemprop=image><meta content=Nicksxs itemprop=name></span><span hidden itemprop=publisher itemscope itemtype=http://schema.org/Organization><meta content="Nicksxs's Blog" itemprop=name><meta content="Nicksxs 的技术博客,深入 Java 运行时与生产诊断、低成本大模型推理和 Agent 工程实践。" itemprop=description></span><span hidden itemprop=post itemscope itemtype=http://schema.org/CreativeWork><meta content="6GB显存能跑35B MoE吗:FreeToken极限配置、性能压榨与实操教程 | Nicksxs's Blog" itemprop=name><meta content=6GB显存可以尝试运行Qwen3.6-35B-A3B-NVFP4,但代价是更小的专家缓存和KV容量,以及对32GB以上内存、CPU与PCIe带宽的更高依赖。本文给出已验证配置、调优顺序、压测脚本和明确限制。 itemprop=description></span><header class=post-header><h1 itemprop="name headline" class=post-title>6GB显存能跑35B MoE吗:FreeToken极限配置、性能压榨与实操教程</h1><div class=post-meta-container><div class=post-meta><span class=post-meta-item><span class=post-meta-item-icon><i class="far fa-calendar"></i> </span><span class=post-meta-item-text>发表于</span> <time itemprop="dateCreated datePublished" title="创建时间:2026-09-05 10:00:00" datetime=2026-09-05T10:00:00+08:00>2026-09-05</time> </span><span class=post-meta-item><span class=post-meta-item-icon><i class="far fa-folder"></i> </span><span class=post-meta-item-text>分类于</span> <span itemprop=about itemscope itemtype=http://schema.org/Thing><a href=/categories/LLM/ itemprop=url rel=index><span itemprop=name>LLM</span></a> </span></span><span class="post-meta-item cloudflare-page-counter" data-title="6GB显存能跑35B MoE吗:FreeToken极限配置、性能压榨与实操教程" data-increment=true data-path=/2026/09/05/6GB%E6%98%BE%E5%AD%98%E8%B7%9135B-MoE-FreeToken%E6%9E%81%E9%99%90%E9%85%8D%E7%BD%AE%E4%B8%8E%E5%AE%9E%E6%93%8D%E6%95%99%E7%A8%8B/ title=阅读次数><span class=post-meta-item-icon><i class="far fa-eye"></i> </span><span class=post-meta-item-text>阅读次数:</span> <span class=cloudflare-page-counter-count></span></span></div><div class=post-description>6GB显存可以尝试运行Qwen3.6-35B-A3B-NVFP4,但代价是更小的专家缓存和KV容量,以及对32GB以上内存、CPU与PCIe带宽的更高依赖。本文给出已验证配置、调优顺序、压测脚本和明确限制。</div></div></header><div class=post-body itemprop=articleBody><p>之前在<a href=/2026/08/23/8GB%E6%98%BE%E5%AD%98%E5%A6%82%E4%BD%95%E8%B7%9135B-%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3FreeToken%E7%9A%84%E4%B8%93%E5%AE%B6%E7%BC%93%E5%AD%98%E4%B8%8ECPU-GPU%E5%8D%8F%E5%90%8C/>8GB 显存运行 35B 的原理文章</a>里,我重点解释了 FreeToken 的三级存储、全局 LRU 专家缓存和 CPU-GPU 混合执行。<p>继续往下压一个档位,会遇到更实际的问题:<blockquote><p>只有 6GB 显存,能不能运行同一个 Qwen3.6-35B-A3B-NVFP4?怎样配置才不会一启动就 OOM?</blockquote><p>先说结论:<strong>可以尝试,而且已经有 RTX 3060 Laptop 6GB 的成功案例;但 6GB 不在论文公开基准的范围内,目前能引用的是 FreeToken 官方仓库中的社区复现,不是官方性能承诺。</strong><p>成功启动也不等于能复现 8GB RTX 4060 Laptop 的 <code>39.3 tok/s</code>。6GB 会进一步压缩专家缓存和 KV Cache,更多专家 miss 会落到 PCIe 或 CPU,最终性能更依赖整机内存带宽、CPU、PCIe 和工作负载。<h2 id=先看结论表><a class=headerlink href=#先看结论表 title=先看结论表></a>先看结论表</h2><table><thead><tr><th>问题<th>结论<tbody><tr><td>6GB 能否启动 35B MoE<td>RTX 3060 Laptop 6GB 已有成功复现<tr><td>6GB 是否有论文跑分<td>没有;论文最低公开硬件是 8GB Laptop GPU<tr><td>能否复现 39.3 tok/s<td>没有证据,不能这样承诺<tr><td>最低显卡架构<td>当前官方要求 Ampere,也就是 RTX 30 系列起<tr><td>RTX 2060 / GTX 1660 6GB<td>显存容量够写“6GB”,但架构不在当前官方支持范围<tr><td>系统内存<td>32GB 是紧张下限,建议 48GB,开发或多任务建议 64GB<tr><td>推荐模型<td><code>nvidia/Qwen3.6-35B-A3B-NVFP4</code><tr><td>推荐起步配置<td>Triton attention、<code>memory-ratio=0.92</code>、2048 Prefill chunk、512 expert slots<tr><td>最适合的用途<td>单用户、短到中等上下文、连续 Decode、Coding Agent 实验<tr><td>不适合的用途<td>16GB 内存、长上下文高并发、Dense 35B、只看显卡不看整机</table><p>这里的“32GB 是下限、48GB 更推荐”是根据 30GB 内存环境的成功日志、<a href=https://github.com/FlashML-org/FreeToken/issues/87 rel=noopener target=_blank>约 22GB 的 NVFP4 checkpoint</a>、Host expert bank、JIT 编译和操作系统余量做出的工程建议,不是 FreeToken 官方写死的最低内存规格。<h2 id=6GB-的直接证据是什么><a title="6GB 的直接证据是什么" class=headerlink href=#6GB-的直接证据是什么></a>6GB 的直接证据是什么</h2><p>FreeToken 官方仓库的 <a href=https://github.com/FlashML-org/FreeToken/issues/303 rel=noopener target=_blank>issue #303</a> 给出了完整环境:<figure class="highlight text"><table><tr><td class=gutter><pre><span class=line>1</span><br><span class=line>2</span><br><span class=line>3</span><br><span class=line>4</span><br><span class=line>5</span><br><span class=line>6</span><br><span class=line>7</span><br><span class=line>8</span><br></pre><td class=code><pre><span class=line>GPU: RTX 3060 Laptop 6GB,实际容量 5.67GiB</span><br><span class=line>CPU: Ryzen 9 5900HX</span><br><span class=line>RAM: 30GB</span><br><span class=line>OS: Linux Mint 22.3</span><br><span class=line>FreeToken: 0.1.2 + 当时的源码版本</span><br><span class=line>CUDA toolkit: 13.0.2</span><br><span class=line>Driver: 595.84</span><br><span class=line>Model: nvidia/Qwen3.6-35B-A3B-NVFP4</span><br></pre></table></figure><p>失败配置是:<figure class="highlight bash"><table><tr><td class=gutter><pre><span class=line>1</span><br><span class=line>2</span><br><span class=line>3</span><br><span class=line>4</span><br></pre><td class=code><pre><span class=line>ft serve \</span><br><span class=line> --model nvidia/Qwen3.6-35B-A3B-NVFP4 \</span><br><span class=line> --memory-ratio 0.95 \</span><br><span class=line> --kv-reserve-tokens 4096</span><br></pre></table></figure><p>它在启动阶段还要申请 FlashInfer 的 <code>256MiB float_workspace_buffer</code>,此时只剩 <code>238.12MiB</code>,最终 CUDA OOM。<p>社区报告给出的稳定配置是:<figure class="highlight text"><table><tr><td class=gutter><pre><span class=line>1</span><br><span class=line>2</span><br><span class=line>3</span><br><span class=line>4</span><br></pre><td class=code><pre><span class=line>--attn triton</span><br><span class=line>--memory-ratio 0.92</span><br><span class=line>--max-prefill-length 2048</span><br><span class=line>--moe-cache-size 512</span><br></pre></table></figure><p>启动后还可以通过:<figure class="highlight bash"><table><tr><td class=gutter><pre><span class=line>1</span><br></pre><td class=code><pre><span class=line>ft ctl cache --moe 940</span><br></pre></table></figure><p>在线扩大专家缓存。<p>这组证据能证明“6GB 可以启动并服务”,但 issue 没有给出可与论文严格对齐的吞吐表,所以本文不会编造一个 6GB token/s。<h2 id=为什么从-8GB-减到-6GB-会明显变难><a title="为什么从 8GB 减到 6GB 会明显变难" class=headerlink href=#为什么从-8GB-减到-6GB-会明显变难></a>为什么从 8GB 减到 6GB 会明显变难</h2><p>35B MoE 的专家可以留在 Host RAM,但显存里仍然有几块不能随便消失的内容:<p><img alt="6GB 显存中的常驻权重、专家缓存、KV Cache 与运行时余量" loading=lazy src=/uploads/freetoken-6gb/vram-budget.png><h3 id=非专家权重是硬成本><a class=headerlink href=#非专家权重是硬成本 title=非专家权重是硬成本></a>非专家权重是硬成本</h3><p>Attention、Router、Embedding、Norm、GDN 等非专家模块仍要在 GPU 上执行。专家能 offload,不代表模型其余部分也能全部移走。<h3 id=专家缓存决定-Decode-命中率><a title="专家缓存决定 Decode 命中率" class=headerlink href=#专家缓存决定-Decode-命中率></a>专家缓存决定 Decode 命中率</h3><p>显存越少,能保留的 expert slots 越少。缓存 miss 变多以后,有两条补救路径:<figure class="highlight text"><table><tr><td class=gutter><pre><span class=line>1</span><br><span class=line>2</span><br><span class=line>3</span><br></pre><td class=code><pre><span class=line>PCIe 搬到 GPU 再算</span><br><span class=line>或者</span><br><span class=line>CPU 直接读取 Host RAM 中的专家权重计算</span><br></pre></table></figure><p>在 issue #303 的那台机器上,带宽 profile 让 hybrid 模式把约 <code>20.1%</code> 的 Decode miss 通过 PCIe 搬给 GPU,其余交给 CPU。这个比例只属于那台机器,换 CPU、内存或 PCIe 后会重新计算。<h3 id=KV-Cache-与专家缓存争同一块显存><a title="KV Cache 与专家缓存争同一块显存" class=headerlink href=#KV-Cache-与专家缓存争同一块显存></a>KV Cache 与专家缓存争同一块显存</h3><p>更多 expert slots 通常能降低 Decode miss,但会挤压 KV Cache;KV 更多则能容纳更长上下文,却可能让 Decode 更依赖 CPU 和 PCIe。<p>所以 6GB 调优不是“把专家缓存拉到最大”,而是找到:<figure class="highlight text"><table><tr><td class=gutter><pre><span class=line>1</span><br><span class=line>2</span><br><span class=line>3</span><br><span class=line>4</span><br><span class=line>5</span><br></pre><td class=code><pre><span class=line>满足目标上下文的最小 KV</span><br><span class=line>+</span><br><span class=line>不会触发运行时 OOM 的余量</span><br><span class=line>+</span><br><span class=line>剩余空间尽可能给专家缓存</span><br></pre></table></figure><h3 id=运行时工作区不会显示在模型文件大小里><a class=headerlink href=#运行时工作区不会显示在模型文件大小里 title=运行时工作区不会显示在模型文件大小里></a>运行时工作区不会显示在模型文件大小里</h3><p>FlashInfer 的 256MiB workspace 正是典型例子。CUDA Graph、JIT kernel、临时 tensor 和内存碎片也需要余量。6GB 卡上把 <code>--memory-ratio</code> 从 <code>0.92</code> 拉到 <code>0.95</code>,看起来只多用了 3%,实际可能正好吃掉最后的启动空间。<h2 id=内存到底要多大><a class=headerlink href=#内存到底要多大 title=内存到底要多大></a>内存到底要多大</h2><p>FreeToken 官方 FAQ 的表述是:MoE 专家位于 Host RAM,因此需要大致相当于专家权重大小的<strong>可用内存</strong>。Qwen3.6-35B-A3B 的 BF16 版本约需 70GB,而 NVFP4 小得多。<p>这里要区分“物理内存”和“真正空闲内存”。<table><thead><tr><th>物理内存<th>对 35B NVFP4 的判断<th>实际建议<tbody><tr><td>16GB<td>不建议<td>权重、expert bank 与系统无法同时稳定容纳,进入 swap 后性能会崩<tr><td>24GB<td>极限实验<td>可能在加载或构建 expert bank 时失败,不适合作为教程基线<tr><td>32GB<td>紧张下限<td>关闭浏览器、IDE 和容器;让 expert bank 使用串行构建<tr><td>48GB<td>推荐<td>给模型、Pinned RAM、JIT、文件缓存和系统留出实际余量<tr><td>64GB<td>舒适<td>适合同时跑 Agent、索引、IDE 和监控工具</table><p>issue #303 的 30GB 环境能够完成加载,但日志已经出现:<figure class="highlight text"><table><tr><td class=gutter><pre><span class=line>1</span><br></pre><td class=code><pre><span class=line>expert banks: low free RAM -> serial build</span><br></pre></table></figure><p>这说明 32GB 更像“认真清理后台后可以做实验”,而不是完全没有压力。不要手动强制 <code>--expert-load parallel</code>;低内存时让自动策略选择串行构建,虽然启动慢一些,但峰值更低。<p>Swap 可以防止进程立刻被 OOM killer 杀掉,却不能替代物理内存。一旦推理热路径频繁访问 swap,交互性能通常会失去意义。<h2 id=硬件与系统检查><a class=headerlink href=#硬件与系统检查 title=硬件与系统检查></a>硬件与系统检查</h2><h3 id=显卡不能只看-6GB-这个数字><a title="显卡不能只看 6GB 这个数字" class=headerlink href=#显卡不能只看-6GB-这个数字></a>显卡不能只看 6GB 这个数字</h3><p>当前官方要求是:<figure class="highlight text"><table><tr><td class=gutter><pre><span class=line>1</span><br><span class=line>2</span><br><span class=line>3</span><br><span class=line>4</span><br><span class=line>5</span><br></pre><td class=code><pre><span class=line>Linux x86_64</span><br><span class=line>NVIDIA Ampere 或更新架构</span><br><span class=line>driver r580+</span><br><span class=line>CUDA 13 toolkit,nvcc 在 PATH 中</span><br><span class=line>Python 3.10+</span><br></pre></table></figure><p>因此:<ul><li>RTX 3060 Laptop 6GB:架构满足,并已有社区成功案例<li>RTX 3050 6GB:架构满足,但没有本文引用的同模型实测,需要自己验证性能<li>RTX 2060 6GB:Turing,不在当前官方支持范围<li>GTX 1660 6GB:Turing,不在当前官方支持范围<li>AMD 6GB:当前官方 CLI 不支持<li>macOS:当前不支持 FreeToken CUDA 路径,继续使用 llama.cpp 或 MLX</ul><p>Windows 和 Linux 可以使用桌面应用;下面的 CLI 教程以 Linux 为准。<h3 id=开始前运行这些检查><a class=headerlink href=#开始前运行这些检查 title=开始前运行这些检查></a>开始前运行这些检查</h3><figure class="highlight bash"><table><tr><td class=gutter><pre><span class=line>1</span><br><span class=line>2</span><br><span class=line>3</span><br><span class=line>4</span><br><span class=line>5</span><br><span class=line>6</span><br><span class=line>7</span><br><span class=line>8</span><br><span class=line>9</span><br></pre><td class=code><pre><span class=line>nvidia-smi --query-gpu=name,memory.total,memory.free,driver_version,pci.bus_id \</span><br><span class=line> --format=csv,noheader</span><br><span class=line></span><br><span class=line>nvcc --version</span><br><span class=line>python3 --version</span><br><span class=line>free -h</span><br><span class=line>swapon --show</span><br><span class=line><span class=built_in>df</span> -h <span class=string>"<span class=variable>${HF_HOME:-<span class=variable>$HOME</span>/.cache/huggingface}</span>"</span></span><br><span class=line>lscpu | grep -E <span class=string>'Model name|Socket|Core|Thread|Flags'</span></span><br></pre></table></figure><p>重点不是看到 <code>6144 MiB</code> 就结束,而是确认:<ol><li>启动前实际空闲显存最好接近整卡容量;桌面环境和浏览器 GPU 进程会吃掉宝贵余量。<li>Driver 与 CUDA toolkit 满足版本要求,<code>nvcc</code> 确实可用,因为 kernel 首次运行会 JIT 编译。<li>系统有至少约 30GB 可用内存,而不是只看“已安装 32GB”。<li>Hugging Face 缓存位于空间充足的 NVMe;模型、虚拟环境和编译缓存建议预留 40GB 以上。<li>CPU 支持 AVX2,并尽量使用双通道内存。6GB 下 CPU 与内存带宽比 8GB 更重要。</ol><h2 id=从零安装><a class=headerlink href=#从零安装 title=从零安装></a>从零安装</h2><p>建议把 Hugging Face 缓存放到容量足够的 NVMe:<figure class="highlight bash"><table><tr><td class=gutter><pre><span class=line>1</span><br><span class=line>2</span><br></pre><td class=code><pre><span class=line><span class=built_in>export</span> HF_HOME=/mnt/nvme/huggingface</span><br><span class=line><span class=built_in>mkdir</span> -p <span class=string>"<span class=variable>$HF_HOME</span>"</span></span><br></pre></table></figure><p>安装 <code>uv</code> 后创建独立环境:<figure class="highlight bash"><table><tr><td class=gutter><pre><span class=line>1</span><br><span class=line>2</span><br><span class=line>3</span><br><span class=line>4</span><br><span class=line>5</span><br><span class=line>6</span><br><span class=line>7</span><br><span class=line>8</span><br><span class=line>9</span><br></pre><td class=code><pre><span class=line><span class=built_in>mkdir</span> -p ~/lab/freetoken-6gb</span><br><span class=line><span class=built_in>cd</span> ~/lab/freetoken-6gb</span><br><span class=line></span><br><span class=line>uv venv --python 3.12</span><br><span class=line><span class=built_in>source</span> .venv/bin/activate</span><br><span class=line>uv pip install <span class=string>"freetoken[accel]"</span></span><br><span class=line></span><br><span class=line>ft --version</span><br><span class=line>nvcc --version</span><br></pre></table></figure><p>也可以从源码安装:<figure class="highlight bash"><table><tr><td class=gutter><pre><span class=line>1</span><br><span class=line>2</span><br><span class=line>3</span><br><span class=line>4</span><br><span class=line>5</span><br></pre><td class=code><pre><span class=line>git <span class=built_in>clone</span> https://github.com/FlashML-org/FreeToken.git</span><br><span class=line><span class=built_in>cd</span> FreeToken</span><br><span class=line>uv venv --python 3.12</span><br><span class=line><span class=built_in>source</span> .venv/bin/activate</span><br><span class=line>uv pip install -e <span class=string>".[accel]"</span></span><br></pre></table></figure><p>第一次运行会编译 CUDA kernel,时间明显比后续启动长。不要把首次 JIT 时间误认为模型每次都需要这么久。<h2 id=模型应该下载哪一个><a class=headerlink href=#模型应该下载哪一个 title=模型应该下载哪一个></a>模型应该下载哪一个</h2><p>这里最容易踩坑:<strong>Hugging Face 是模型托管平台,不代表里面的文件都是 FreeToken 支持的格式。</strong> 当前这套 Qwen3.6 实验应使用 NVIDIA 官方仓库:<p><a href=https://huggingface.co/nvidia/Qwen3.6-35B-A3B-NVFP4 rel=noopener target=_blank>nvidia/Qwen3.6-35B-A3B-NVFP4</a><table><thead><tr><th>可以使用<th>不要用于当前实验<tbody><tr><td>仓库 <code>nvidia/Qwen3.6-35B-A3B-NVFP4</code><td>名称带 <code>GGUF</code> 的量化仓库<tr><td><code>model-00001-of-00003.safetensors</code> 等三段权重<td>单个或多个 <code>*.gguf</code> 文件<tr><td>NVFP4 checkpoint,完整下载约 23.5GB<td><code>Qwen/Qwen3.6-35B-A3B</code> 的 BF16 原始权重</table><p>FreeToken 可以直接加载 Hugging Face 的 <code>safetensors</code> checkpoint,不需要先转成 FTW。它当前对 GGUF 的原生支持只覆盖特定架构,不能用 Qwen3.6 GGUF 替代这个 NVFP4 checkpoint。<h3 id=方法一:让-FreeToken-自动下载><a title="方法一:让 FreeToken 自动下载" class=headerlink href=#方法一:让-FreeToken-自动下载></a>方法一:让 FreeToken 自动下载</h3><p><code>--model</code> 可以直接接 Hugging Face repo id:<figure class="highlight bash"><table><tr><td class=gutter><pre><span class=line>1</span><br><span class=line>2</span><br><span class=line>3</span><br><span class=line>4</span><br><span class=line>5</span><br><span class=line>6</span><br><span class=line>7</span><br><span class=line>8</span><br><span class=line>9</span><br><span class=line>10</span><br><span class=line>11</span><br><span class=line>12</span><br></pre><td class=code><pre><span class=line><span class=built_in>export</span> HF_HOME=/mnt/nvme/huggingface</span><br><span class=line></span><br><span class=line>ft serve \</span><br><span class=line> --model nvidia/Qwen3.6-35B-A3B-NVFP4 \</span><br><span class=line> --gpu 0 \</span><br><span class=line> --attn triton \</span><br><span class=line> --moe-strategy auto \</span><br><span class=line> --memory-ratio 0.92 \</span><br><span class=line> --max-running-requests 1 \</span><br><span class=line> --max-prefill-length 2048 \</span><br><span class=line> --moe-cache-size 512 \</span><br><span class=line> --max-output-tokens 2048</span><br></pre></table></figure><p>这种方式最省事,模型会进入 <code>$HF_HOME</code>。中断后重新执行会复用已经下载的文件。<h3 id=方法二:先完整下载,再从本地目录启动><a class=headerlink href=#方法二:先完整下载,再从本地目录启动 title=方法二:先完整下载,再从本地目录启动></a>方法二:先完整下载,再从本地目录启动</h3><p>这种方式更容易检查文件,也便于把模型固定在容量充足的 NVMe 上:<figure class="highlight bash"><table><tr><td class=gutter><pre><span class=line>1</span><br><span class=line>2</span><br><span class=line>3</span><br><span class=line>4</span><br><span class=line>5</span><br><span class=line>6</span><br></pre><td class=code><pre><span class=line><span class=built_in>source</span> ~/lab/freetoken-6gb/.venv/bin/activate</span><br><span class=line>uv pip install -U huggingface_hub</span><br><span class=line></span><br><span class=line><span class=built_in>mkdir</span> -p /mnt/nvme/models</span><br><span class=line>hf download nvidia/Qwen3.6-35B-A3B-NVFP4 \</span><br><span class=line> --local-dir /mnt/nvme/models/Qwen3.6-35B-A3B-NVFP4</span><br></pre></table></figure><p>不要只下载权重分片;<code>config.json</code>、tokenizer、chat template 和 <code>model.safetensors.index.json</code> 也需要保留。完整下载后检查:<figure class="highlight bash"><table><tr><td class=gutter><pre><span class=line>1</span><br><span class=line>2</span><br><span class=line>3</span><br><span class=line>4</span><br><span class=line>5</span><br><span class=line>6</span><br></pre><td class=code><pre><span class=line>find /mnt/nvme/models/Qwen3.6-35B-A3B-NVFP4 \</span><br><span class=line> -maxdepth 1 -<span class=built_in>type</span> f \</span><br><span class=line> \( -name <span class=string>'*.safetensors'</span> -o -name <span class=string>'*.json'</span> -o -name <span class=string>'*.jinja'</span> \) \</span><br><span class=line> -<span class=built_in>printf</span> <span class=string>'%f\n'</span> | <span class=built_in>sort</span></span><br><span class=line></span><br><span class=line><span class=built_in>du</span> -sh /mnt/nvme/models/Qwen3.6-35B-A3B-NVFP4</span><br></pre></table></figure><p>应当至少看到下面三段权重,总目录约 23.5GB:<figure class="highlight text"><table><tr><td class=gutter><pre><span class=line>1</span><br><span class=line>2</span><br><span class=line>3</span><br></pre><td class=code><pre><span class=line>model-00001-of-00003.safetensors</span><br><span class=line>model-00002-of-00003.safetensors</span><br><span class=line>model-00003-of-00003.safetensors</span><br></pre></table></figure><p>然后把本地目录交给 FreeToken:<figure class="highlight bash"><table><tr><td class=gutter><pre><span class=line>1</span><br><span class=line>2</span><br><span class=line>3</span><br><span class=line>4</span><br><span class=line>5</span><br><span class=line>6</span><br><span class=line>7</span><br><span class=line>8</span><br><span class=line>9</span><br><span class=line>10</span><br></pre><td class=code><pre><span class=line>ft serve \</span><br><span class=line> --model /mnt/nvme/models/Qwen3.6-35B-A3B-NVFP4 \</span><br><span class=line> --gpu 0 \</span><br><span class=line> --attn triton \</span><br><span class=line> --moe-strategy auto \</span><br><span class=line> --memory-ratio 0.92 \</span><br><span class=line> --max-running-requests 1 \</span><br><span class=line> --max-prefill-length 2048 \</span><br><span class=line> --moe-cache-size 512 \</span><br><span class=line> --max-output-tokens 2048</span><br></pre></table></figure><p>建议下载前至少留出 35GB 磁盘空间。不要先用自动缓存下载一份,再手动复制一份到其他目录,否则很容易无意中占用双倍空间。<h2 id=第一步:先测这台机器的带宽><a class=headerlink href=#第一步:先测这台机器的带宽 title=第一步:先测这台机器的带宽></a>第一步:先测这台机器的带宽</h2><figure class="highlight bash"><table><tr><td class=gutter><pre><span class=line>1</span><br></pre><td class=code><pre><span class=line>ft bench bw --dtype nvfp4 --gpu 0</span><br></pre></table></figure><p>这个测试使用真实的 CPU/offload MoE kernel,结果会写入:<figure class="highlight text"><table><tr><td class=gutter><pre><span class=line>1</span><br></pre><td class=code><pre><span class=line>~/.cache/freetoken/benchbw/&LTgpu-uuid>.json</span><br></pre></table></figure><p><code>--moe-strategy auto</code> 会读取它,决定继续使用 offload,还是切换到 hybrid,并推导 <code>--moe-hybrid-max-fetch</code>。旧版本里的 <code>--moe-backend</code> 目前仍可兼容,但已经不是官方文档推荐的参数名。<p>不要复制别人的 <code>q*</code> 或 fetch 数量。FreeToken 会按 GPU UUID 和 expert format 区分 profile,其他机器的结果不会直接套用。<h2 id=第二步:使用-6GB-保守配置启动><a title="第二步:使用 6GB 保守配置启动" class=headerlink href=#第二步:使用-6GB-保守配置启动></a>第二步:使用 6GB 保守配置启动</h2><p>先关闭其他 CUDA 程序,再确认一次空闲显存:<figure class="highlight bash"><table><tr><td class=gutter><pre><span class=line>1</span><br></pre><td class=code><pre><span class=line>nvidia-smi</span><br></pre></table></figure><p>然后使用保守配置:<figure class="highlight bash"><table><tr><td class=gutter><pre><span class=line>1</span><br><span class=line>2</span><br><span class=line>3</span><br><span class=line>4</span><br><span class=line>5</span><br><span class=line>6</span><br><span class=line>7</span><br><span class=line>8</span><br><span class=line>9</span><br><span class=line>10</span><br><span class=line>11</span><br><span class=line>12</span><br><span class=line>13</span><br></pre><td class=code><pre><span class=line><span class=built_in>export</span> PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True</span><br><span class=line></span><br><span class=line>ft serve \</span><br><span class=line> --model nvidia/Qwen3.6-35B-A3B-NVFP4 \</span><br><span class=line> --gpu 0 \</span><br><span class=line> --attn triton \</span><br><span class=line> --moe-strategy auto \</span><br><span class=line> --memory-ratio 0.92 \</span><br><span class=line> --max-running-requests 1 \</span><br><span class=line> --max-prefill-length 2048 \</span><br><span class=line> --moe-cache-size 512 \</span><br><span class=line> --max-output-tokens 2048 \</span><br><span class=line> --decode-log-interval 20</span><br></pre></table></figure><p>其中真正针对 6GB 的关键项是:<table><thead><tr><th>参数<th>为什么这样设<tbody><tr><td><code>--attn triton</code><td>避开 issue #303 中 FlashInfer 固定 256MiB workspace 导致的启动 OOM<tr><td><code>--memory-ratio 0.92</code><td>给 warmup、graph capture、JIT 与临时 tensor 留余量<tr><td><code>--max-running-requests 1</code><td>单用户先不为并发预留额外资源<tr><td><code>--max-prefill-length 2048</code><td>降低单个 Prefill chunk 的峰值;它不是最大上下文长度<tr><td><code>--moe-cache-size 512</code><td>从已验证的保守专家槽位数起步<tr><td><code>--max-output-tokens 2048</code><td>避免客户端没有指定限制时默认请求很大的输出预算</table><p><code>--max-prefill-length 2048</code> 只控制分块 Prefill 每块处理多少 token,不代表模型只能使用 2048 上下文。真正可容纳的上下文由 KV pool 决定。<h2 id=第三步:检查是否真的启动正确><a class=headerlink href=#第三步:检查是否真的启动正确 title=第三步:检查是否真的启动正确></a>第三步:检查是否真的启动正确</h2><p>另开一个终端:<figure class="highlight bash"><table><tr><td class=gutter><pre><span class=line>1</span><br><span class=line>2</span><br><span class=line>3</span><br><span class=line>4</span><br><span class=line>5</span><br></pre><td class=code><pre><span class=line><span class=built_in>source</span> ~/lab/freetoken-6gb/.venv/bin/activate</span><br><span class=line></span><br><span class=line>ft ctl health</span><br><span class=line>ft ctl cache</span><br><span class=line>ft ctl stats</span><br></pre></table></figure><p>重点记录:<ul><li>实际选择的 <code>attention_backend</code><li>实际选择的 <code>moe_backend</code><li><code>moe_cache_size</code><li>KV pool 能容纳多少 token<li>GPU 显存占用和剩余量<li>CPU executor 的线程数与 ISA<li>hybrid 模式每步有多少 miss 走 PCIe</ul><p>issue #303 的成功日志中,512 起步配置对应的运行时曾显示 610 个自动 slots、4163 个 KV tokens;这只用于理解量级,不应该预设你的机器也会得到相同结果。<h2 id=第四步:先做最小冒烟测试><a class=headerlink href=#第四步:先做最小冒烟测试 title=第四步:先做最小冒烟测试></a>第四步:先做最小冒烟测试</h2><figure class="highlight bash"><table><tr><td class=gutter><pre><span class=line>1</span><br><span class=line>2</span><br><span class=line>3</span><br></pre><td class=code><pre><span class=line>ft ctl generate \</span><br><span class=line> <span class=string>"用三句话解释MoE中总参数和激活参数的区别"</span> \</span><br><span class=line> --max-tokens 128</span><br></pre></table></figure><p>再测试 OpenAI 兼容接口:<figure class="highlight bash"><table><tr><td class=gutter><pre><span class=line>1</span><br><span class=line>2</span><br><span class=line>3</span><br><span class=line>4</span><br><span class=line>5</span><br><span class=line>6</span><br><span class=line>7</span><br><span class=line>8</span><br><span class=line>9</span><br><span class=line>10</span><br></pre><td class=code><pre><span class=line>curl http://127.0.0.1:1919/v1/chat/completions \</span><br><span class=line> -H <span class=string>'Content-Type: application/json'</span> \</span><br><span class=line> -d <span class=string>'{</span></span><br><span class=line><span class=string> "model": "Qwen3.6-35B-A3B-NVFP4",</span></span><br><span class=line><span class=string> "messages": [</span></span><br><span class=line><span class=string> {"role": "user", "content": "写一个带单元测试的Python LRU缓存"}</span></span><br><span class=line><span class=string> ],</span></span><br><span class=line><span class=string> "max_tokens": 256,</span></span><br><span class=line><span class=string> "temperature": 0</span></span><br><span class=line><span class=string> }'</span></span><br></pre></table></figure><p>冒烟测试通过后再压测,不要第一次请求就塞入几十 K 上下文。<h2 id=第五步:逐级增加专家缓存><a class=headerlink href=#第五步:逐级增加专家缓存 title=第五步:逐级增加专家缓存></a>第五步:逐级增加专家缓存</h2><p>先保存基线:<figure class="highlight bash"><table><tr><td class=gutter><pre><span class=line>1</span><br><span class=line>2</span><br><span class=line>3</span><br></pre><td class=code><pre><span class=line><span class=built_in>mkdir</span> -p logs</span><br><span class=line>ft ctl --json cache > logs/cache-512.json</span><br><span class=line>ft ctl --json stats > logs/stats-512-before.json</span><br></pre></table></figure><p>然后一次只改一个变量:<figure class="highlight bash"><table><tr><td class=gutter><pre><span class=line>1</span><br><span class=line>2</span><br><span class=line>3</span><br><span class=line>4</span><br><span class=line>5</span><br><span class=line>6</span><br><span class=line>7</span><br><span class=line>8</span><br><span class=line>9</span><br><span class=line>10</span><br><span class=line>11</span><br><span class=line>12</span><br></pre><td class=code><pre><span class=line>ft ctl cache --moe 640 --<span class=built_in>wait</span> 300</span><br><span class=line>ft ctl cache</span><br><span class=line></span><br><span class=line>ft ctl cache --moe 768 --<span class=built_in>wait</span> 300</span><br><span class=line>ft ctl cache</span><br><span class=line></span><br><span class=line>ft ctl cache --moe 896 --<span class=built_in>wait</span> 300</span><br><span class=line>ft ctl cache</span><br><span class=line></span><br><span class=line><span class=comment># issue #303 使用过的目标值;并不保证你的机器也安全</span></span><br><span class=line>ft ctl cache --moe 940 --<span class=built_in>wait</span> 300</span><br><span class=line>ft ctl cache</span><br></pre></table></figure><p>每升一级,都用同一个 Prompt 连续跑至少三轮:<figure class="highlight bash"><table><tr><td class=gutter><pre><span class=line>1</span><br><span class=line>2</span><br><span class=line>3</span><br><span class=line>4</span><br><span class=line>5</span><br><span class=line>6</span><br><span class=line>7</span><br><span class=line>8</span><br><span class=line>9</span><br></pre><td class=code><pre><span class=line><span class=keyword>for</span> i <span class=keyword>in</span> 1 2 3; <span class=keyword>do</span></span><br><span class=line> ft ctl generate \</span><br><span class=line> <span class=string>"实现一个线程安全的LRU缓存并解释锁粒度"</span> \</span><br><span class=line> --max-tokens 256 \</span><br><span class=line> --ignore-eos</span><br><span class=line><span class=keyword>done</span></span><br><span class=line></span><br><span class=line>ft ctl --json stats</span><br><span class=line>ft ctl --json requests --<span class=built_in>limit</span> 20</span><br></pre></table></figure><p>第一轮更接近冷缓存,后两轮才能观察路由局部性被 LRU 利用后的状态。<p>增加 slots 时同时观察 KV 容量。如果 Decode 变快,但 KV 已经低于实际上下文需求,这个配置仍然不能用于 Agent。<h2 id=第六步:按工作负载反向分配-KV><a title="第六步:按工作负载反向分配 KV" class=headerlink href=#第六步:按工作负载反向分配-KV></a>第六步:按工作负载反向分配 KV</h2><h3 id=只做短对话或代码补全><a class=headerlink href=#只做短对话或代码补全 title=只做短对话或代码补全></a>只做短对话或代码补全</h3><p>目标上下文只有 2K 到 4K 时,可以把更多剩余显存给专家缓存。先验证 512,再向 640、768、896 增长。<h3 id=做-8K-左右的-Coding-Agent><a title="做 8K 左右的 Coding Agent" class=headerlink href=#做-8K-左右的-Coding-Agent></a>做 8K 左右的 Coding Agent</h3><p>先保证 KV,而不是先追求 expert hit rate:<figure class="highlight bash"><table><tr><td class=gutter><pre><span class=line>1</span><br><span class=line>2</span><br></pre><td class=code><pre><span class=line>ft ctl cache --kv 8k --<span class=built_in>wait</span> 300</span><br><span class=line>ft ctl cache</span><br></pre></table></figure><p>如果 8K KV 让专家缓存缩得过小、Decode 明显变慢,就说明 6GB 已经不适合这个工作集。此时更合理的选择通常是:<ol><li>降到 4K 左右上下文。<li>减少同时注入的仓库文件。<li>利用 radix/prefix cache 减少重复 Prefill。<li>换 8GB 或更大显卡,而不是继续挤掉运行时余量。</ol><h3 id=不要从-32K-或-128K-开始><a title="不要从 32K 或 128K 开始" class=headerlink href=#不要从-32K-或-128K-开始></a>不要从 32K 或 128K 开始</h3><p>模型声明支持长上下文,不等于 6GB 能为它分配足够 KV。<code>--max-seq-len-override</code> 是上限约束,不会凭空制造 KV 显存。<h2 id=怎样判断还应该继续加-expert-slots><a title="怎样判断还应该继续加 expert slots" class=headerlink href=#怎样判断还应该继续加-expert-slots></a>怎样判断还应该继续加 expert slots</h2><p>可以使用这组决策:<figure class="highlight text"><table><tr><td class=gutter><pre><span class=line>1</span><br><span class=line>2</span><br><span class=line>3</span><br><span class=line>4</span><br><span class=line>5</span><br><span class=line>6</span><br><span class=line>7</span><br><span class=line>8</span><br><span class=line>9</span><br><span class=line>10</span><br><span class=line>11</span><br><span class=line>12</span><br><span class=line>13</span><br><span class=line>14</span><br></pre><td class=code><pre><span class=line>KV 不够目标上下文</span><br><span class=line> -> 先加 KV,减少专家缓存</span><br><span class=line></span><br><span class=line>KV 足够,但 Decode miss 很高,GPU 仍有安全余量</span><br><span class=line> -> 分级增加 expert slots</span><br><span class=line></span><br><span class=line>CPU 已满载,Decode 仍慢</span><br><span class=line> -> 增加 expert slots,或比较 offload</span><br><span class=line></span><br><span class=line>PCIe 已满、CPU 还有余量</span><br><span class=line> -> hybrid 让更多 miss 在 CPU 计算</span><br><span class=line></span><br><span class=line>启动或 warmup OOM</span><br><span class=line> -> 降 memory-ratio,保持 Triton attention,不要先扩大 pool</span><br></pre></table></figure><p>专家缓存不是越大越快的单调函数。缓存增长可能降低 miss,却同时压缩 KV 和运行时余量;发生重建、长 Prompt 或工作集变化时,整体延迟仍可能更差。<h2 id=比较-auto、offload、cpu-和-hybrid><a title="比较 auto、offload、cpu 和 hybrid" class=headerlink href=#比较-auto、offload、cpu-和-hybrid></a>比较 auto、offload、cpu 和 hybrid</h2><p>不要只相信带宽微基准。官方仓库已有案例显示,某些硬件上 <code>ft bench bw</code> 推荐 hybrid,但真实模型中 offload 更快。原因包括每个 expert activation 的小 GEMV、同步和 CPU kernel 开销没有被简单带宽比完全描述。<p>使用同一套 Prompt,分别重启测试:<figure class="highlight bash"><table><tr><td class=gutter><pre><span class=line>1</span><br><span class=line>2</span><br><span class=line>3</span><br><span class=line>4</span><br><span class=line>5</span><br><span class=line>6</span><br><span class=line>7</span><br><span class=line>8</span><br><span class=line>9</span><br><span class=line>10</span><br><span class=line>11</span><br></pre><td class=code><pre><span class=line><span class=comment># 自动选择,完成 bench bw 后通常会在 offload/hybrid 中决定</span></span><br><span class=line>ft serve ... --moe-strategy auto</span><br><span class=line></span><br><span class=line><span class=comment># miss 全部通过 PCIe 搬到 GPU</span></span><br><span class=line>ft serve ... --moe-strategy offload</span><br><span class=line></span><br><span class=line><span class=comment># miss 由 CPU 计算</span></span><br><span class=line>ft serve ... --moe-strategy cpu</span><br><span class=line></span><br><span class=line><span class=comment># PCIe fetch 与 CPU 计算并行</span></span><br><span class=line>ft serve ... --moe-strategy hybrid</span><br></pre></table></figure><p>比较时固定:<ul><li>模型和 FreeToken commit<li><code>memory-ratio</code><li>expert slots<li>KV tokens<li>Prompt<li>输出 token 数<li>冷缓存还是热缓存</ul><p>只有这样,结果才能回答“哪种 backend 更适合我的机器”。<h2 id=一键实验脚本><a class=headerlink href=#一键实验脚本 title=一键实验脚本></a>一键实验脚本</h2><p>我把检查、启动、状态采集和逐级调 cache 整理成了脚本:<p><a href=/downloads/freetoken-6gb-lab.sh>下载 freetoken-6gb-lab.sh</a><p>用法:<figure class="highlight bash"><table><tr><td class=gutter><pre><span class=line>1</span><br><span class=line>2</span><br><span class=line>3</span><br><span class=line>4</span><br><span class=line>5</span><br><span class=line>6</span><br><span class=line>7</span><br><span class=line>8</span><br><span class=line>9</span><br><span class=line>10</span><br><span class=line>11</span><br><span class=line>12</span><br><span class=line>13</span><br><span class=line>14</span><br><span class=line>15</span><br><span class=line>16</span><br><span class=line>17</span><br><span class=line>18</span><br><span class=line>19</span><br><span class=line>20</span><br><span class=line>21</span><br><span class=line>22</span><br><span class=line>23</span><br></pre><td class=code><pre><span class=line><span class=built_in>chmod</span> +x freetoken-6gb-lab.sh</span><br><span class=line></span><br><span class=line><span class=comment># 检查环境</span></span><br><span class=line>./freetoken-6gb-lab.sh preflight</span><br><span class=line></span><br><span class=line><span class=comment># 下载正确的 NVFP4 safetensors checkpoint</span></span><br><span class=line>./freetoken-6gb-lab.sh download</span><br><span class=line></span><br><span class=line><span class=comment># 校验本地模型目录</span></span><br><span class=line>./freetoken-6gb-lab.sh verify-model</span><br><span class=line></span><br><span class=line><span class=comment># 按保守配置启动</span></span><br><span class=line>FT_MODEL=<span class=string>"<span class=variable>$HOME</span>/models/Qwen3.6-35B-A3B-NVFP4"</span> \</span><br><span class=line> ./freetoken-6gb-lab.sh run</span><br><span class=line></span><br><span class=line><span class=comment># 另开终端查看状态</span></span><br><span class=line>./freetoken-6gb-lab.sh status</span><br><span class=line></span><br><span class=line><span class=comment># 冒烟测试</span></span><br><span class=line>./freetoken-6gb-lab.sh smoke</span><br><span class=line></span><br><span class=line><span class=comment># 明确知道会在线重分配缓存后,再执行逐级实验</span></span><br><span class=line>./freetoken-6gb-lab.sh sweep</span><br></pre></table></figure><p>通过环境变量可以覆盖默认值:<figure class="highlight bash"><table><tr><td class=gutter><pre><span class=line>1</span><br><span class=line>2</span><br><span class=line>3</span><br><span class=line>4</span><br><span class=line>5</span><br></pre><td class=code><pre><span class=line>FT_GPU=0 \</span><br><span class=line>FT_MEMORY_RATIO=0.90 \</span><br><span class=line>FT_MOE_CACHE_SIZE=512 \</span><br><span class=line>FT_MAX_PREFILL=2048 \</span><br><span class=line>./freetoken-6gb-lab.sh run</span><br></pre></table></figure><h2 id=常见问题与排查顺序><a class=headerlink href=#常见问题与排查顺序 title=常见问题与排查顺序></a>常见问题与排查顺序</h2><h3 id=启动时申请-256MiB-失败><a title="启动时申请 256MiB 失败" class=headerlink href=#启动时申请-256MiB-失败></a>启动时申请 256MiB 失败</h3><p>症状通常指向 FlashInfer workspace。<p>处理顺序:<figure class="highlight text"><table><tr><td class=gutter><pre><span class=line>1</span><br><span class=line>2</span><br><span class=line>3</span><br><span class=line>4</span><br><span class=line>5</span><br></pre><td class=code><pre><span class=line>确认 --attn triton</span><br><span class=line>确认 memory-ratio 不高于 0.92</span><br><span class=line>关闭其他 GPU 进程</span><br><span class=line>必要时继续降到 0.90 或 0.88</span><br><span class=line>保持 moe-cache-size 512,不要先扩大缓存</span><br></pre></table></figure><p>issue #303 截至本文日期仍为 open,所以不要假设所有版本都已经自动为 attention workspace 预留空间。<h3 id=模型加载时系统内存不够><a class=headerlink href=#模型加载时系统内存不够 title=模型加载时系统内存不够></a>模型加载时系统内存不够</h3><figure class="highlight text"><table><tr><td class=gutter><pre><span class=line>1</span><br><span class=line>2</span><br><span class=line>3</span><br><span class=line>4</span><br></pre><td class=code><pre><span class=line>关闭浏览器、IDE、Docker 和其他模型服务</span><br><span class=line>让 expert-load 保持 auto,接受 serial build</span><br><span class=line>检查是否正在大量 swap</span><br><span class=line>仍然失败就增加物理内存</span><br></pre></table></figure><p>不要用 <code>--expert-load parallel</code> 强行换取启动速度,它会提高加载峰值。<h3 id=能启动,但长-Prompt-卡住><a title="能启动,但长 Prompt 卡住" class=headerlink href=#能启动,但长-Prompt-卡住></a>能启动,但长 Prompt 卡住</h3><p>先运行:<figure class="highlight bash"><table><tr><td class=gutter><pre><span class=line>1</span><br><span class=line>2</span><br></pre><td class=code><pre><span class=line>ft ctl cache</span><br><span class=line>ft ctl requests --<span class=built_in>limit</span> 20</span><br></pre></table></figure><p>确认请求长度没有超过 KV pool。FreeToken 仓库中已有“请求超过 KV pool 后排队而没有清晰错误”的问题报告,因此应用层最好主动限制上下文,而不是只依赖模型的最大长度声明。<h3 id=Decode-很慢,CPU-满载><a title="Decode 很慢,CPU 满载" class=headerlink href=#Decode-很慢,CPU-满载></a>Decode 很慢,CPU 满载</h3><ol><li>查看 expert cache 是否过小。<li>在 KV 足够的前提下逐步增加 slots。<li>比较 <code>offload</code> 与 <code>hybrid</code>,不要只看 auto。<li>检查内存是否双通道、频率是否正常。<li>检查 CPU 是否降频或笔记本处于省电模式。</ol><h3 id=显存看起来还有几百-MiB,仍然-OOM><a title="显存看起来还有几百 MiB,仍然 OOM" class=headerlink href=#显存看起来还有几百-MiB,仍然-OOM></a>显存看起来还有几百 MiB,仍然 OOM</h3><p><code>nvidia-smi</code> 的空闲值不等于一个足够大的连续可分配块,也没有替你计算后续 workspace、graph capture 和临时 tensor。6GB 卡必须保留保守余量。<h2 id=6GB-的明确限制><a title="6GB 的明确限制" class=headerlink href=#6GB-的明确限制></a>6GB 的明确限制</h2><h3 id=1-它仍然需要大内存><a title="1. 它仍然需要大内存" class=headerlink href=#1-它仍然需要大内存></a>1. 它仍然需要大内存</h3><p>FreeToken 降低的是 VRAM 门槛,不是模型总内存需求。35B NVFP4 的专家权重仍要驻留 Host RAM。16GB 内存不是“速度慢一点”,而是很可能无法稳定加载。<h3 id=2-6GB-不是所有-6GB-显卡><a title="2. 6GB 不是所有 6GB 显卡" class=headerlink href=#2-6GB-不是所有-6GB-显卡></a>2. 6GB 不是所有 6GB 显卡</h3><p>当前官方要求 Ampere 或更新架构。RTX 2060 和 GTX 1660 即使也是 6GB,也不能直接套用 RTX 3060 的结论。<h3 id=3-不能把-39-3-tok-s-搬过来><a title="3. 不能把 39.3 tok/s 搬过来" class=headerlink href=#3-不能把-39-3-tok-s-搬过来></a>3. 不能把 39.3 tok/s 搬过来</h3><p>39.3 tok/s 属于论文中的 8GB RTX 4060 Laptop、NVFP4 模型和 coding-agent workload。6GB 社区案例只证明能启动和服务,没有同条件吞吐数据。<h3 id=4-长上下文和专家命中率直接竞争><a title="4. 长上下文和专家命中率直接竞争" class=headerlink href=#4-长上下文和专家命中率直接竞争></a>4. 长上下文和专家命中率直接竞争</h3><p>6GB 下增加上下文通常意味着减少 expert slots。长 Agent 会话即使不 OOM,也可能因为专家 miss 增加而逐渐变慢。<h3 id=5-Prefill-和-Decode-是两种瓶颈><a title="5. Prefill 和 Decode 是两种瓶颈" class=headerlink href=#5-Prefill-和-Decode-是两种瓶颈></a>5. Prefill 和 Decode 是两种瓶颈</h3><p>缩小 <code>max-prefill-length</code> 能降低峰值,却可能增加分块开销。Decode 速度好看也不代表 TTFT 好看。<h3 id=6-整机决定结果><a title="6. 整机决定结果" class=headerlink href=#6-整机决定结果></a>6. 整机决定结果</h3><p>同一张 RTX 3060 Laptop 6GB,在 DDR4 单通道、DDR4 双通道和高带宽 DDR5 机器上,CPU fallback 的表现会完全不同。PCIe 是否降到 x4、笔记本功耗墙和 CPU 温度也会改变结果。<h3 id=7-当前软件仍在快速变化><a title="7. 当前软件仍在快速变化" class=headerlink href=#7-当前软件仍在快速变化></a>7. 当前软件仍在快速变化</h3><p>6GB 暴露的是显存预算边界问题。attention workspace、cache planner、请求长度校验和 backend 自动选择都仍有活跃 issue。复现实验必须记录 FreeToken 版本或 Git commit。<h2 id=我会怎样选择硬件><a class=headerlink href=#我会怎样选择硬件 title=我会怎样选择硬件></a>我会怎样选择硬件</h2><p>如果目标是“验证系统思路、偶尔运行 35B MoE”,已有 RTX 3060 Laptop 6GB 和 32GB 内存可以先试,成本最低。<p>如果目标是“每天把它当 Coding Agent 使用”,更合理的最低配置是:<figure class="highlight text"><table><tr><td class=gutter><pre><span class=line>1</span><br><span class=line>2</span><br><span class=line>3</span><br><span class=line>4</span><br><span class=line>5</span><br></pre><td class=code><pre><span class=line>RTX 3060/4060 8GB 或更大</span><br><span class=line>48GB 以上系统内存</span><br><span class=line>双通道高带宽内存</span><br><span class=line>NVMe</span><br><span class=line>Linux + CUDA 13</span><br></pre></table></figure><p>多出来的 2GB 显存不仅是多放一点权重,更重要的是给 KV、expert slots 和运行时 workspace 留出同时存在的空间,调优难度会明显下降。<h2 id=最后总结><a class=headerlink href=#最后总结 title=最后总结></a>最后总结</h2><p>6GB 显存运行 35B MoE 的正确理解是:<figure class="highlight text"><table><tr><td class=gutter><pre><span class=line>1</span><br><span class=line>2</span><br><span class=line>3</span><br><span class=line>4</span><br></pre><td class=code><pre><span class=line>不是把 35B 放进 6GB</span><br><span class=line>而是只把非专家模块、KV、工作区和少量热专家放进 6GB</span><br><span class=line>完整专家池仍然放在大内存里</span><br><span class=line>缓存 miss 再由 PCIe 和 CPU 兜底</span><br></pre></table></figure><p>6GB 能做,但已经进入系统工程的边缘:<ol><li>使用 Ampere 或更新的 NVIDIA GPU。<li>准备至少 32GB、最好 48GB 系统内存。<li>从 Triton attention、0.92 memory ratio、2048 Prefill chunk 和 512 slots 起步。<li>先保证启动与目标 KV,再逐级增加专家缓存。<li>实测 auto、offload 与 hybrid,不照搬别人的带宽比例。<li>不把 8GB 论文跑分包装成 6GB 性能承诺。</ol><p>对 6GB 用户来说,最有价值的结果不是跑出一个孤立的最高 token/s,而是找到一个在自己的 Prompt、上下文长度和散热条件下可以连续运行的配置。<h2 id=参考资料><a class=headerlink href=#参考资料 title=参考资料></a>参考资料</h2><ul><li><a href=https://arxiv.org/abs/2608.16157 rel=noopener target=_blank>FreeToken 论文</a><li><a href=https://github.com/FlashML-org/FreeToken/blob/main/docs/install.md rel=noopener target=_blank>FreeToken 安装文档</a><li><a href=https://github.com/FlashML-org/FreeToken/blob/main/docs/cli.md rel=noopener target=_blank>FreeToken CLI 参数</a><li><a href=https://github.com/FlashML-org/FreeToken/blob/main/docs/models.md rel=noopener target=_blank>FreeToken 支持模型与 MoE backend</a><li><a href=https://huggingface.co/nvidia/Qwen3.6-35B-A3B-NVFP4 rel=noopener target=_blank>NVIDIA Qwen3.6-35B-A3B-NVFP4 模型仓库</a><li><a href=https://github.com/FlashML-org/FreeToken/issues/34 rel=noopener target=_blank>FreeToken GGUF 支持范围与功能请求</a><li><a href=https://github.com/FlashML-org/FreeToken/issues/84 rel=noopener target=_blank>FreeToken FAQ:平台与 Host RAM 要求</a><li><a href=https://github.com/FlashML-org/FreeToken/issues/303 rel=noopener target=_blank>6GB RTX 3060 Laptop 启动 OOM 与稳定配置</a><li><a href=https://github.com/FlashML-org/FreeToken/issues/87 rel=noopener target=_blank>Qwen3.6-35B-A3B-NVFP4 checkpoint 大小与长 Prefill 问题</a><li><a href=https://github.com/FlashML-org/FreeToken/issues/111 rel=noopener target=_blank>KV pool 不足时的请求排队问题</a><li><a href=https://github.com/FlashML-org/FreeToken/issues/151 rel=noopener target=_blank>真实模型可能与带宽微基准结论不同</a></ul></div><footer class=post-footer><aside aria-label=文章所属专题 class=post-series><span>所属专题</span><a href=/topics/#local-llm>低成本大模型推理</a><p>从本地部署、量化和 KV Cache 参数入手,逐步理解显存占用、GPU Offload 与 MoE 专家缓存。</aside><div class="affiliate-creative affiliate-creative--post-end" data-affiliate-id=tencent-cloud-server><a rel="sponsored noopener" aria-label=腾讯云服务器优惠 href=https://cloud.tencent.com/act/cps/redirect?redirect=6544&cps_key=2e33a1eedb380997d03618b4a5ba6ca7&from=console target=_blank><img alt="腾讯云 AI 焕新云服务器优惠" decoding=async height=241 loading=lazy src=/uploads/affiliate/tencent-cloud-server.webp width=690> </a><span class=affiliate-disclosure>推广</span></div><div class=reward-container><div>请我一杯咖啡吧!</div><button>赞赏</button><div class=post-reward><div><img alt="Nicksxs 微信" src=https://i.loli.net/2020/01/12/NUlWanT31E8LQGx.png><span>微信</span></div><div><img alt="Nicksxs 支付宝" src=https://ooo.0o0.ooo/2016/08/17/57b421e773057.jpg><span>支付宝</span></div></div></div><div class=post-copyright><ul><li class=post-copyright-author><strong>本文作者: </strong>Nicksxs<li class=post-copyright-link><strong>本文链接:</strong> <a title="6GB显存能跑35B MoE吗:FreeToken极限配置、性能压榨与实操教程" href=https://nicksxs.me/2026/09/05/6GB%E6%98%BE%E5%AD%98%E8%B7%9135B-MoE-FreeToken%E6%9E%81%E9%99%90%E9%85%8D%E7%BD%AE%E4%B8%8E%E5%AE%9E%E6%93%8D%E6%95%99%E7%A8%8B/>https://nicksxs.me/2026/09/05/6GB显存跑35B-MoE-FreeToken极限配置与实操教程/</a><li class=post-copyright-license><strong>版权声明: </strong>本博客所有文章除特别声明外,均采用 <a href=https://creativecommons.org/licenses/by-nc-sa/4.0/ rel=noopener target=_blank><i class="fab fa-fw fa-creative-commons"></i>BY-NC-SA</a> 许可协议。转载请注明出处!</ul></div><div class=post-tags><a href=/tags/LLM/ rel=tag># LLM</a><a href=/tags/FreeToken/ rel=tag># FreeToken</a><a href=/tags/CUDA/ rel=tag># CUDA</a><a href=/tags/%E6%9C%AC%E5%9C%B0%E5%A4%A7%E6%A8%A1%E5%9E%8B/ rel=tag># 本地大模型</a></div><div class=post-nav><div class=post-nav-item><a href=/2026/08/30/%E4%BB%8EASM%E5%88%B0ByteKit%E5%86%8D%E5%88%B0Arthas-%E4%B8%80%E6%9D%A1Java%E5%AD%97%E8%8A%82%E7%A0%81%E5%A2%9E%E5%BC%BA%E9%93%BE%E8%B7%AF/ rel=prev title=从ASM到ByteKit再到Arthas:一条Java字节码增强链路><i class="fa fa-angle-left"></i> 从ASM到ByteKit再到Arthas:一条Java字节码增强链路</a></div><div class=post-nav-item><a title="Pi Agent 源码解析(一):从调试环境到双层主循环" href=/2026/09/13/Pi-Agent-%E6%BA%90%E7%A0%81%E8%A7%A3%E6%9E%90-%E4%BB%8E%E8%B0%83%E8%AF%95%E7%8E%AF%E5%A2%83%E5%88%B0%E5%8F%8C%E5%B1%82%E4%B8%BB%E5%BE%AA%E7%8E%AF/ rel=next>Pi Agent 源码解析(一):从调试环境到双层主循环 <i class="fa fa-angle-right"></i></a></div></div></footer></article></div><div class=comments id=disqus_thread><noscript>Please enable JavaScript to view the comments powered by Disqus.</noscript></div></div></main><footer class=footer><div class=footer-inner><div class=beian><a href=https://beian.miit.gov.cn/ rel=noopener target=_blank>浙ICP备16002580号-2 </a><img alt src=/uploads/beian.png><a href=https://beian.mps.gov.cn/#/query/webSearch?code=33010602011094 rel=noopener target=_blank>浙公网安备 33010602011094号</a></div><div class=copyright>© <span itemprop=copyrightYear>2026</span><span class=with-love><i class="fa fa-heart"></i> </span><span class=author itemprop=copyrightHolder>Nicksxs</span></div></div></footer><div class="toggle sidebar-toggle" role=button><span class=toggle-line></span><span class=toggle-line></span><span class=toggle-line></span></div><div class=sidebar-dimmer></div><div aria-label=返回顶部 class=back-to-top role=button><i class="fa fa-arrow-up fa-lg"></i><span>0%</span></div><a aria-label="在 GitHub 上关注我" title="在 GitHub 上关注我" class=github-corner href=https://github.com/nicksxs rel=noopener target=_blank><svg viewbox="0 0 250 250" aria-hidden=true height=80 width=80><path d="M0,0 L115,115 L130,115 L142,142 L250,250 L250,0 Z"></path><path d="M128.3,109.0 C113.8,99.7 119.0,89.6 119.0,89.6 C122.0,82.7 120.5,78.6 120.5,78.6 C119.2,72.0 123.4,76.3 123.4,76.3 C127.3,80.9 125.5,87.3 125.5,87.3 C122.9,97.6 130.6,101.9 134.4,103.2" style="transform-origin:130px 106px" class=octo-arm fill=currentColor></path><path d="M115.0,115.0 C114.9,115.1 118.7,116.5 119.8,115.4 L133.7,101.6 C136.9,99.2 139.9,98.4 142.2,98.6 C133.8,88.0 127.5,74.4 143.8,58.0 C148.5,53.4 154.0,51.2 159.7,51.0 C160.3,49.4 163.2,43.6 171.4,40.1 C171.4,40.1 176.1,42.5 178.8,56.2 C183.1,58.6 187.2,61.8 190.9,65.4 C194.5,69.0 197.7,73.2 200.1,77.6 C213.8,80.2 216.3,84.9 216.3,84.9 C212.7,93.1 206.9,96.0 205.4,96.6 C205.1,102.4 203.0,107.8 198.3,112.5 C181.9,128.9 168.3,122.5 157.7,114.1 C157.9,116.9 156.7,120.9 152.7,124.9 L141.0,136.5 C139.8,137.7 141.6,141.9 141.8,141.8 Z" class=octo-body fill=currentColor></path></svg></a><noscript><div class=noscript-warning>Theme NexT works best with JavaScript enabled</div></noscript><script data-cloudflare-page-counter-config type=application/json>{"enable":true,"endpoint":"https://counter.nicksxs.me","site":"nicksxs","local_cache":true,"timeout":5000,"icon":"far fa-eye","item_text":true}</script><script src=/js/cloudflare-page-counter.js></script><script class=next-config data-name=disqus type=application/json>{"enable":true,"shortname":"nicksxs","count":false,"i18n":{"disqus":"disqus"}}</script><script defer src=/js/third-party/comments/disqus.js></script>