You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 

1 lines
72 KiB

<!doctypehtml><html lang=zh-CN><meta charset=UTF-8><meta content=width=device-width name=viewport><meta media="(prefers-color-scheme: light)" content=#222 name=theme-color><meta media="(prefers-color-scheme: dark)" content=#222 name=theme-color><meta content="Hexo 7.3.0" name=generator><link href=https://cdnjs.cloudflare.com rel=preconnect><link href=/images/apple-touch-icon-next.png rel=apple-touch-icon sizes=180x180><link href=/images/favicon-32x32-next.png rel=icon sizes=32x32 type=image/png><link href=/images/favicon-16x16-next.png rel=icon sizes=16x16 type=image/png><link color=#222 href=/images/logo.svg rel=mask-icon><meta content=2X6S9P7CAjXjVvw8YyQR8pCu-B0oEu7O9quLgxXuWyA name=google-site-verification><meta content=dV8JGNzi0c name=baidu-site-verification><script>!function(){var t=!1;function e(){var e;t||(t=!0,(e=document.createElement("script")).setAttribute("data-ad-client","ca-pub-7480618470784058"),e.async=!0,e.src="https://pagead2.googlesyndication.com/pagead/js/adsbygoogle.js",document.head.appendChild(e))}function a(){"requestIdleCallback"in window?window.requestIdleCallback(e,{timeout:5e3}):window.setTimeout(e,5e3)}"complete"===document.readyState?a():window.addEventListener("load",a,{once:!0})}()</script><link href=/css/main.css rel=stylesheet><link crossorigin=anonymous href=https://cdnjs.cloudflare.com/ajax/libs/font-awesome/7.0.0/css/all.min.css integrity=sha256-VHqXKFhhMxcpubYf9xiWdCiojEbY9NexQ4jh8AxbvcM= rel=stylesheet><link crossorigin=anonymous href=https://cdnjs.cloudflare.com/ajax/libs/fancyapps-ui/6.1.14/fancybox/fancybox.css integrity=sha256-mdaMxrItEW6hMIVJJNPKuX0B7KCSJQVEv3mVY4N2we4= rel=stylesheet><script class=next-config data-name=main type=application/json>{"hostname":"nicksxs.me","root":"/","images":"/images","scheme":"Pisces","darkmode":true,"version":"8.29.0","exturl":false,"sidebar":{"position":"left","width_expanded":320,"width_dual_column":240,"display":"post","padding":18,"offset":12},"hljswrap":true,"codeblock":{"theme":{"light":"default","dark":"stackoverflow-dark"},"prism":{"light":"prism","dark":"prism-dark"},"copy_button":{"enable":true,"style":"default"},"fold":{"enable":false,"height":500},"language":false},"bookmark":{"enable":false,"color":"#222","save":"auto"},"mediumzoom":false,"pangu":false,"comments":{"style":"tabs","active":null,"storage":true,"lazyload":true,"nav":null},"stickytabs":false,"motion":{"enable":false,"async":false,"duration":200,"transition":{"menu_item":"fadeInDown","post_block":"fadeIn","post_header":"fadeInDown","post_body":"fadeInDown","coll_header":"fadeInLeft","sidebar":"fadeInUp"}},"prism":false,"i18n":{"placeholder":"搜索...","empty":"没有找到任何搜索结果:${query}","hits_time":"找到 ${hits} 个搜索结果(用时 ${time} 毫秒)","hits":"找到 ${hits} 个搜索结果"},"algolia":{"appID":"663Q99OQQD","apiKey":"e461a1371d12cec1491c1153b288a9ed","indexName":"nicksxs","hits":{"per_page":10}}}</script><script defer src=/js/config.js></script><meta content="CommitLog 是 rocketmq 的服务端,也就是 broker 存储消息的的文件,跟 kafka 一样,也是顺序写入,当然消息是变长的,生成的规则是每个文件的默认1G =1024 1024 1024,commitlog的文件名fileName,名字长度为20位,左边补…" name=description><meta content=article property=og:type><meta content="聊一下 RocketMQ 的消息存储二" property=og:title><meta content=https://nicksxs.me/2021/09/12/%E8%81%8A%E4%B8%80%E4%B8%8B-RocketMQ-%E7%9A%84%E6%B6%88%E6%81%AF%E5%AD%98%E5%82%A8%E4%BA%8C/index.html property=og:url><meta content="Nicksxs's Blog" property=og:site_name><meta content="CommitLog 是 rocketmq 的服务端,也就是 broker 存储消息的的文件,跟 kafka 一样,也是顺序写入,当然消息是变长的,生成的规则是每个文件的默认1G =1024 1024 1024,commitlog的文件名fileName,名字长度为20位,左边补…" property=og:description><meta content=zh_CN property=og:locale><meta content=https://img.nicksxs.me/uPic/kLahwW.png property=og:image><meta content=2021-09-12T12:49:18.000Z property=article:published_time><meta content=2021-09-12T12:49:18.000Z property=article:modified_time><meta content=Nicksxs property=article:author><meta content=MQ property=article:tag><meta content=消息队列 property=article:tag><meta content=RocketMQ property=article:tag><meta content=summary name=twitter:card><meta content=https://img.nicksxs.me/uPic/kLahwW.png name=twitter:image><link href=https://nicksxs.me/2021/09/12/%E8%81%8A%E4%B8%80%E4%B8%8B-RocketMQ-%E7%9A%84%E6%B6%88%E6%81%AF%E5%AD%98%E5%82%A8%E4%BA%8C/ rel=canonical><script class=next-config data-name=page type=application/json>{"sidebar":"","isHome":false,"isPost":true,"lang":"zh-CN","comments":true,"permalink":"https://nicksxs.me/2021/09/12/%E8%81%8A%E4%B8%80%E4%B8%8B-RocketMQ-%E7%9A%84%E6%B6%88%E6%81%AF%E5%AD%98%E5%82%A8%E4%BA%8C/","path":"2021/09/12/聊一下-RocketMQ-的消息存储二/","title":"聊一下 RocketMQ 的消息存储二"}</script><script class=next-config data-name=calendar type=application/json>""</script><title>聊一下 RocketMQ 的消息存储二 | Nicksxs's Blog</title><script async src=https://www.googletagmanager.com/gtag/js?id=UA-61358619-1></script><script class=next-config data-name=google_analytics type=application/json>{"tracking_id":"UA-61358619-1","only_pageview":false,"measure_protocol_api_secret":null}</script><script defer src=/js/third-party/analytics/google-analytics.js></script><script defer src=/js/third-party/analytics/baidu-analytics.js></script><script async src=https://hm.baidu.com/hm.js?20f33b3c0c0eff9b1522999c0015646d></script><script crossorigin=anonymous defer integrity=sha256-QnJV4HT5r8i89ovZGRD86iynZqHerEbT2xt69vDM36I= src=https://cdnjs.cloudflare.com/ajax/libs/fancyapps-ui/6.1.14/fancybox/fancybox.umd.js></script><script defer src=/js/utils.js></script><script defer src=/js/sidebar.js></script><script defer src=/js/next-boot.js></script><script data-algolia-integrity=sha256-o49zoHLDPBm9WLg6AiliAVi3axCRTqe3ltTxLvhzAVg= data-algolia-src=https://cdnjs.cloudflare.com/ajax/libs/algoliasearch/5.36.0/lite/builds/browser.umd.js data-search-src=/js/third-party/search/algolia-search.js src=/js/third-party/search/algolia-search-loader.js></script><script defer src=/js/third-party/fancybox.js></script><noscript><link href=/css/noscript.css rel=stylesheet></noscript><link title="Nicksxs's Blog" href=/atom.xml rel=alternate type=application/atom+xml><body itemscope itemtype=http://schema.org/WebPage><div class=headband></div><main class=main><div class=column><header class=header itemscope itemtype=http://schema.org/WPHeader><div class=site-brand-container><div class=site-nav-toggle><div aria-label=切换导航栏 class=toggle role=button><span class=toggle-line></span><span class=toggle-line></span><span class=toggle-line></span></div></div><div class=site-meta><a class=brand href=/ rel=start><i class=logo-line></i><p class=site-title>Nicksxs's Blog</p><i class=logo-line></i></a><p class=site-subtitle itemprop=description>Java 运行时、生产诊断与 AI 工程实践</div><div class=site-nav-right><div class="toggle popup-trigger" aria-label=搜索 role=button><i class="fa fa-search fa-fw fa-lg"></i></div></div></div><nav class=site-nav><ul class="main-menu menu"><li class="menu-item menu-item-home"><a href=/ rel=section><i class="fa fa-home fa-fw"></i>首页</a><li class="menu-item menu-item-专题"><a href=/topics/ rel=section><i class="fa fa-layer-group fa-fw"></i>专题</a><li class="menu-item menu-item-工具"><a href=/tools/ rel=section><i class="fa fa-toolbox fa-fw"></i>工具</a><li class="menu-item menu-item-about"><a href=/about/ rel=section><i class="fa fa-user fa-fw"></i>关于我</a><li class="menu-item menu-item-tags"><a href=/tags/ rel=section><i class="fa fa-tags fa-fw"></i>标签</a><li class="menu-item menu-item-categories"><a href=/categories/ rel=section><i class="fa fa-th fa-fw"></i>分类</a><li class="menu-item menu-item-archives"><a href=/archives/ rel=section><i class="fa fa-archive fa-fw"></i>归档</a><li class="menu-item menu-item-search"><a class=popup-trigger role=button><i class="fa fa-search fa-fw"></i>搜索</a></ul></nav><div class=search-pop-overlay><div class="popup search-popup"><div class=search-header><span class=search-icon><i class="fa fa-search"></i></span><div class=search-input-container><input autocapitalize=off autocomplete=off class=search-input maxlength=80 placeholder=搜索... spellcheck=false type=search></div><span class=popup-btn-close role=button><i class="fa fa-times-circle"></i></span></div><div class=search-result-container><div class=search-result-icon><i class="fab fa-algolia fa-5x"></i></div></div></div></div></header><aside class=sidebar><div class="sidebar-inner sidebar-nav-active sidebar-toc-active"><ul class=sidebar-nav><li class=sidebar-nav-toc>文章目录<li class=sidebar-nav-overview>站点概览</ul><div class=sidebar-panel-container><div class="post-toc-wrap sidebar-panel"><div class="post-toc animated"><ol class=nav><li class="nav-item nav-level-3"><a class=nav-link href=#CommitLog-%E7%BB%93%E6%9E%84><span class=nav-number>1.</span> <span class=nav-text>CommitLog 结构</span></a></ol></div></div><div class="site-overview-wrap sidebar-panel"><div class="site-author animated" itemprop=author itemscope itemtype=http://schema.org/Person><img alt=Nicksxs class=site-author-image itemprop=image src=/uploads/avatar.jpg><p class=site-author-name itemprop=name>Nicksxs<div class=site-description itemprop=description>Nicksxs 的技术博客,深入 Java 运行时与生产诊断、低成本大模型推理和 Agent 工程实践。</div></div><div class="site-state-wrap animated"><nav class=site-state><div class="site-state-item site-state-posts"><a href=/archives/><span class=site-state-item-count>394</span> <span class=site-state-item-name>日志</span></a></div><div class="site-state-item site-state-categories"><a href=/categories/><span class=site-state-item-count>181</span> <span class=site-state-item-name>分类</span></a></div><div class="site-state-item site-state-tags"><a href=/tags/><span class=site-state-item-count>347</span> <span class=site-state-item-name>标签</span></a></div></nav></div><div class="links-of-author animated"><span class=links-of-author-item><a rel="noopener me" title="GitHub → https://github.com/nicksxs" href=https://github.com/nicksxs target=_blank><i class="fab fa-github fa-fw"></i>GitHub</a> </span><span class=links-of-author-item><a rel="noopener me" title="E-Mail → mailto:nicksxs1202@gmail.com" href=mailto:nicksxs1202@gmail.com target=_blank><i class="fa fa-envelope fa-fw"></i>E-Mail</a></span></div><div class="cc-license animated" itemprop=license><a class=cc-opacity href=https://creativecommons.org/licenses/by-nc-sa/4.0/ rel=noopener target=_blank><img alt="Creative Commons" src=https://cdnjs.cloudflare.com/ajax/libs/creativecommons-vocabulary/2020.11.3/assets/license_badges/small/by_nc_sa.svg></a></div><div class="widget-wrap sidebar-tag-cloud"><div class=tag-cloud-tags><a class=tag-cloud-10 href=/tags/Java/ style=font-size:16px>Java</a><a class=tag-cloud-9 href=/tags/leetcode/ style=font-size:15.62px>leetcode</a><a class=tag-cloud-8 href=/tags/LLM/ style=font-size:15.23px>LLM</a><a class=tag-cloud-8 href=/tags/%E7%94%9F%E6%B4%BB/ style=font-size:14.85px>生活</a><a class=tag-cloud-7 href=/tags/%E9%A2%98%E8%A7%A3/ style=font-size:14.46px>题解</a><a class=tag-cloud-6 href=/tags/SpringBoot/ style=font-size:14.08px>SpringBoot</a><a class=tag-cloud-6 href=/tags/MySQL/ style=font-size:14.08px>MySQL</a><a class=tag-cloud-5 href=/tags/C/ style=font-size:13.69px>C++</a><a class=tag-cloud-5 href=/tags/Redis/ style=font-size:13.31px>Redis</a><a class=tag-cloud-4 href=/tags/Tomcat/ style=font-size:12.92px>Tomcat</a><a class=tag-cloud-4 href=/tags/Mybatis/ style=font-size:12.92px>Mybatis</a><a class=tag-cloud-3 href=/tags/%E5%B0%8F%E6%8A%80%E5%B7%A7/ style=font-size:12.54px>小技巧</a><a class=tag-cloud-2 href=/tags/%E6%95%B0%E6%8D%AE%E7%BB%93%E6%9E%84/ style=font-size:12.15px>数据结构</a><a class=tag-cloud-2 href=/tags/%E6%BA%90%E7%A0%81/ style=font-size:12.15px>源码</a><a class=tag-cloud-2 href=/tags/MQ/ style=font-size:11.77px>MQ</a><a class=tag-cloud-2 href=/tags/%E6%B6%88%E6%81%AF%E9%98%9F%E5%88%97/ style=font-size:11.77px>消息队列</a><a class=tag-cloud-2 href=/tags/RocketMQ/ style=font-size:11.77px>RocketMQ</a><a class=tag-cloud-2 href=/tags/%E8%BF%90%E5%8A%A8/ style=font-size:11.77px>运动</a><a class=tag-cloud-1 href=/tags/php/ style=font-size:11.38px>php</a><a class=tag-cloud-0 href=/tags/Spring/ style=font-size:11px>Spring</a><a class=tag-cloud-0 href=/tags/linux/ style=font-size:11px>linux</a><a class=tag-cloud-0 href=/tags/Docker/ style=font-size:11px>Docker</a><a class=tag-cloud-0 href=/tags/%E5%87%8F%E8%82%A5/ style=font-size:11px>减肥</a><a class=tag-cloud-0 href=/tags/%E8%B7%91%E6%AD%A5/ style=font-size:11px>跑步</a></div></div><div class="affiliate-creative affiliate-creative--sidebar" data-affiliate-id=tencent-cloud-general><a rel="sponsored noopener" aria-label=腾讯云产品优惠 href=https://cloud.tencent.com/act/cps/redirect?redirect=2446&cps_key=2e33a1eedb380997d03618b4a5ba6ca7&from=console target=_blank><img alt=腾讯云服务器、数据库和云产品优惠 decoding=async height=419 loading=lazy src=/uploads/affiliate/tencent-cloud-general.webp width=690> </a><span class=affiliate-disclosure>推广</span></div></div></div></div><div class="sidebar-inner sidebar-blogroll"><div class="links-of-blogroll animated"><div class=links-of-blogroll-title><i class="fa fa-globe fa-fw"></i> 链接</div><ul class=links-of-blogroll-list><li class=links-of-blogroll-item><a href=https://covermusic.cn/ rel=noopener target=_blank title=https://covermusic.cn>69伙伴</a></ul></div></div></aside></div><div class="main-inner post posts-expand"><div class=post-block><article class=post-content itemscope itemtype=http://schema.org/Article lang=zh-CN><link href=https://nicksxs.me/2021/09/12/%E8%81%8A%E4%B8%80%E4%B8%8B-RocketMQ-%E7%9A%84%E6%B6%88%E6%81%AF%E5%AD%98%E5%82%A8%E4%BA%8C/ itemprop=mainEntityOfPage><span hidden itemprop=author itemscope itemtype=http://schema.org/Person><meta content=/uploads/avatar.jpg itemprop=image><meta content=Nicksxs itemprop=name></span><span hidden itemprop=publisher itemscope itemtype=http://schema.org/Organization><meta content="Nicksxs's Blog" itemprop=name><meta content="Nicksxs 的技术博客,深入 Java 运行时与生产诊断、低成本大模型推理和 Agent 工程实践。" itemprop=description></span><span hidden itemprop=post itemscope itemtype=http://schema.org/CreativeWork><meta content="聊一下 RocketMQ 的消息存储二 | Nicksxs's Blog" itemprop=name><meta content="CommitLog 是 rocketmq 的服务端,也就是 broker 存储消息的的文件,跟 kafka 一样,也是顺序写入,当然消息是变长的,生成的规则是每个文件的默认1G =1024 1024 1024,commitlog的文件名fileName,名字长度为20位,左边补…" itemprop=description></span><header class=post-header><h1 itemprop="name headline" class=post-title>聊一下 RocketMQ 的消息存储二</h1><div class=post-meta-container><div class=post-meta><span class=post-meta-item><span class=post-meta-item-icon><i class="far fa-calendar"></i> </span><span class=post-meta-item-text>发表于</span> <time itemprop="dateCreated datePublished" title="创建时间:2021-09-12 20:49:18" datetime=2021-09-12T20:49:18+08:00>2021-09-12</time> </span><span class=post-meta-item><span class=post-meta-item-icon><i class="far fa-folder"></i> </span><span class=post-meta-item-text>分类于</span> <span itemprop=about itemscope itemtype=http://schema.org/Thing><a href=/categories/MQ/ itemprop=url rel=index><span itemprop=name>MQ</span></a> </span>, <span itemprop=about itemscope itemtype=http://schema.org/Thing><a href=/categories/MQ/RocketMQ/ itemprop=url rel=index><span itemprop=name>RocketMQ</span></a> </span>, <span itemprop=about itemscope itemtype=http://schema.org/Thing><a href=/categories/MQ/RocketMQ/%E6%B6%88%E6%81%AF%E9%98%9F%E5%88%97/ itemprop=url rel=index><span itemprop=name>消息队列</span></a> </span></span><span class="post-meta-item cloudflare-page-counter" data-title="聊一下 RocketMQ 的消息存储二" data-increment=true data-path=/2021/09/12/%E8%81%8A%E4%B8%80%E4%B8%8B-RocketMQ-%E7%9A%84%E6%B6%88%E6%81%AF%E5%AD%98%E5%82%A8%E4%BA%8C/ title=阅读次数><span class=post-meta-item-icon><i class="far fa-eye"></i> </span><span class=post-meta-item-text>阅读次数:</span> <span class=cloudflare-page-counter-count></span></span></div><div class=post-description>CommitLog 是 rocketmq 的服务端,也就是 broker 存储消息的的文件,跟 kafka 一样,也是顺序写入,当然消息是变长的,生成的规则是每个文件的默认1G =1024 1024 1024,commitlog的文件名fileName,名字长度为20位,左边补…</div></div></header><div class=post-body itemprop=articleBody><h3 id=CommitLog-结构><a title="CommitLog 结构" class=headerlink href=#CommitLog-结构></a>CommitLog 结构</h3><p>CommitLog 是 rocketmq 的服务端,也就是 broker 存储消息的的文件,跟 kafka 一样,也是顺序写入,当然消息是变长的,生成的规则是每个文件的默认1G =1024 * 1024 * 1024,commitlog的文件名fileName,名字长度为20位,左边补零,剩余为起始偏移量;比如00000000000000000000代表了第一个文件,起始偏移量为0,文件大小为1G=1 073 741 824Byte;当这个文件满了,第二个文件名字为00000000001073741824,起始偏移量为1073741824, 消息存储的时候会顺序写入文件,当文件满了则写入下一个文件,代码中的定义<figure class="highlight java"><table><tr><td class=gutter><pre><span class=line>1</span><br><span class=line>2</span><br></pre><td class=code><pre><span class=line><span class=comment>// CommitLog file size,default is 1G</span></span><br><span class=line><span class=keyword>private</span> <span class=type>int</span> <span class=variable>mapedFileSizeCommitLog</span> <span class=operator>=</span> <span class=number>1024</span> * <span class=number>1024</span> * <span class=number>1024</span>;</span><br></pre></table></figure><p><img alt=kLahwW loading=lazy src=https://img.nicksxs.me/uPic/kLahwW.png><p>本地跑个 demo 验证下,也是这样,这里奇妙有几个比较巧妙的点(个人观点),首先文件就刚好是 1G,并且按照大小偏移量去生成下一个文件,这样获取消息的时候按大小算一下就知道在哪个文件里了,<p>代码中写入 CommitLog 的逻辑可以从这开始看<figure class="highlight java"><table><tr><td class=gutter><pre><span class=line>1</span><br><span class=line>2</span><br><span class=line>3</span><br><span class=line>4</span><br><span class=line>5</span><br><span class=line>6</span><br><span class=line>7</span><br><span class=line>8</span><br><span class=line>9</span><br><span class=line>10</span><br><span class=line>11</span><br><span class=line>12</span><br><span class=line>13</span><br><span class=line>14</span><br><span class=line>15</span><br><span class=line>16</span><br><span class=line>17</span><br><span class=line>18</span><br><span class=line>19</span><br><span class=line>20</span><br><span class=line>21</span><br><span class=line>22</span><br><span class=line>23</span><br><span class=line>24</span><br><span class=line>25</span><br><span class=line>26</span><br><span class=line>27</span><br><span class=line>28</span><br><span class=line>29</span><br><span class=line>30</span><br><span class=line>31</span><br><span class=line>32</span><br><span class=line>33</span><br><span class=line>34</span><br><span class=line>35</span><br><span class=line>36</span><br><span class=line>37</span><br><span class=line>38</span><br><span class=line>39</span><br><span class=line>40</span><br><span class=line>41</span><br><span class=line>42</span><br><span class=line>43</span><br><span class=line>44</span><br><span class=line>45</span><br><span class=line>46</span><br><span class=line>47</span><br><span class=line>48</span><br><span class=line>49</span><br><span class=line>50</span><br><span class=line>51</span><br><span class=line>52</span><br><span class=line>53</span><br><span class=line>54</span><br><span class=line>55</span><br><span class=line>56</span><br><span class=line>57</span><br><span class=line>58</span><br><span class=line>59</span><br><span class=line>60</span><br><span class=line>61</span><br><span class=line>62</span><br><span class=line>63</span><br><span class=line>64</span><br><span class=line>65</span><br><span class=line>66</span><br><span class=line>67</span><br><span class=line>68</span><br><span class=line>69</span><br><span class=line>70</span><br><span class=line>71</span><br><span class=line>72</span><br><span class=line>73</span><br><span class=line>74</span><br><span class=line>75</span><br><span class=line>76</span><br><span class=line>77</span><br><span class=line>78</span><br><span class=line>79</span><br><span class=line>80</span><br><span class=line>81</span><br><span class=line>82</span><br><span class=line>83</span><br><span class=line>84</span><br><span class=line>85</span><br><span class=line>86</span><br><span class=line>87</span><br><span class=line>88</span><br><span class=line>89</span><br><span class=line>90</span><br><span class=line>91</span><br><span class=line>92</span><br><span class=line>93</span><br><span class=line>94</span><br><span class=line>95</span><br><span class=line>96</span><br><span class=line>97</span><br><span class=line>98</span><br><span class=line>99</span><br><span class=line>100</span><br><span class=line>101</span><br><span class=line>102</span><br><span class=line>103</span><br><span class=line>104</span><br><span class=line>105</span><br><span class=line>106</span><br><span class=line>107</span><br><span class=line>108</span><br><span class=line>109</span><br><span class=line>110</span><br><span class=line>111</span><br></pre><td class=code><pre><span class=line><span class=keyword>public</span> PutMessageResult <span class="title function_">putMessage</span><span class=params>(<span class=keyword>final</span> MessageExtBrokerInner msg)</span> {</span><br><span class=line> <span class=comment>// Set the storage time</span></span><br><span class=line> msg.setStoreTimestamp(System.currentTimeMillis());</span><br><span class=line> <span class=comment>// Set the message body BODY CRC (consider the most appropriate setting</span></span><br><span class=line> <span class=comment>// on the client)</span></span><br><span class=line> msg.setBodyCRC(UtilAll.crc32(msg.getBody()));</span><br><span class=line> <span class=comment>// Back to Results</span></span><br><span class=line> <span class=type>AppendMessageResult</span> <span class=variable>result</span> <span class=operator>=</span> <span class=literal>null</span>;</span><br><span class=line></span><br><span class=line> <span class=type>StoreStatsService</span> <span class=variable>storeStatsService</span> <span class=operator>=</span> <span class=built_in>this</span>.defaultMessageStore.getStoreStatsService();</span><br><span class=line></span><br><span class=line> <span class=type>String</span> <span class=variable>topic</span> <span class=operator>=</span> msg.getTopic();</span><br><span class=line> <span class=type>int</span> <span class=variable>queueId</span> <span class=operator>=</span> msg.getQueueId();</span><br><span class=line></span><br><span class=line> <span class=keyword>final</span> <span class=type>int</span> <span class=variable>tranType</span> <span class=operator>=</span> MessageSysFlag.getTransactionValue(msg.getSysFlag());</span><br><span class=line> <span class=keyword>if</span> (tranType == MessageSysFlag.TRANSACTION_NOT_TYPE</span><br><span class=line> || tranType == MessageSysFlag.TRANSACTION_COMMIT_TYPE) {</span><br><span class=line> <span class=comment>// Delay Delivery</span></span><br><span class=line> <span class=keyword>if</span> (msg.getDelayTimeLevel() > <span class=number>0</span>) {</span><br><span class=line> <span class=keyword>if</span> (msg.getDelayTimeLevel() > <span class=built_in>this</span>.defaultMessageStore.getScheduleMessageService().getMaxDelayLevel()) {</span><br><span class=line> msg.setDelayTimeLevel(<span class=built_in>this</span>.defaultMessageStore.getScheduleMessageService().getMaxDelayLevel());</span><br><span class=line> }</span><br><span class=line></span><br><span class=line> topic = ScheduleMessageService.SCHEDULE_TOPIC;</span><br><span class=line> queueId = ScheduleMessageService.delayLevel2QueueId(msg.getDelayTimeLevel());</span><br><span class=line></span><br><span class=line> <span class=comment>// Backup real topic, queueId</span></span><br><span class=line> MessageAccessor.putProperty(msg, MessageConst.PROPERTY_REAL_TOPIC, msg.getTopic());</span><br><span class=line> MessageAccessor.putProperty(msg, MessageConst.PROPERTY_REAL_QUEUE_ID, String.valueOf(msg.getQueueId()));</span><br><span class=line> msg.setPropertiesString(MessageDecoder.messageProperties2String(msg.getProperties()));</span><br><span class=line></span><br><span class=line> msg.setTopic(topic);</span><br><span class=line> msg.setQueueId(queueId);</span><br><span class=line> }</span><br><span class=line> }</span><br><span class=line></span><br><span class=line> <span class=type>long</span> <span class=variable>eclipseTimeInLock</span> <span class=operator>=</span> <span class=number>0</span>;</span><br><span class=line> <span class=type>MappedFile</span> <span class=variable>unlockMappedFile</span> <span class=operator>=</span> <span class=literal>null</span>;</span><br><span class=line> <span class=type>MappedFile</span> <span class=variable>mappedFile</span> <span class=operator>=</span> <span class=built_in>this</span>.mappedFileQueue.getLastMappedFile();</span><br><span class=line></span><br><span class=line> putMessageLock.lock(); <span class=comment>//spin or ReentrantLock ,depending on store config</span></span><br><span class=line> <span class=keyword>try</span> {</span><br><span class=line> <span class=type>long</span> <span class=variable>beginLockTimestamp</span> <span class=operator>=</span> <span class=built_in>this</span>.defaultMessageStore.getSystemClock().now();</span><br><span class=line> <span class=built_in>this</span>.beginTimeInLock = beginLockTimestamp;</span><br><span class=line></span><br><span class=line> <span class=comment>// Here settings are stored timestamp, in order to ensure an orderly</span></span><br><span class=line> <span class=comment>// global</span></span><br><span class=line> msg.setStoreTimestamp(beginLockTimestamp);</span><br><span class=line></span><br><span class=line> <span class=keyword>if</span> (<span class=literal>null</span> == mappedFile || mappedFile.isFull()) {</span><br><span class=line> mappedFile = <span class=built_in>this</span>.mappedFileQueue.getLastMappedFile(<span class=number>0</span>); <span class=comment>// Mark: NewFile may be cause noise</span></span><br><span class=line> }</span><br><span class=line> <span class=keyword>if</span> (<span class=literal>null</span> == mappedFile) {</span><br><span class=line> log.error(<span class=string>"create mapped file1 error, topic: "</span> + msg.getTopic() + <span class=string>" clientAddr: "</span> + msg.getBornHostString());</span><br><span class=line> beginTimeInLock = <span class=number>0</span>;</span><br><span class=line> <span class=keyword>return</span> <span class=keyword>new</span> <span class="title class_">PutMessageResult</span>(PutMessageStatus.CREATE_MAPEDFILE_FAILED, <span class=literal>null</span>);</span><br><span class=line> }</span><br><span class=line></span><br><span class=line> result = mappedFile.appendMessage(msg, <span class=built_in>this</span>.appendMessageCallback);</span><br><span class=line> <span class=keyword>switch</span> (result.getStatus()) {</span><br><span class=line> <span class=keyword>case</span> PUT_OK:</span><br><span class=line> <span class=keyword>break</span>;</span><br><span class=line> <span class=keyword>case</span> END_OF_FILE:</span><br><span class=line> unlockMappedFile = mappedFile;</span><br><span class=line> <span class=comment>// Create a new file, re-write the message</span></span><br><span class=line> mappedFile = <span class=built_in>this</span>.mappedFileQueue.getLastMappedFile(<span class=number>0</span>);</span><br><span class=line> <span class=keyword>if</span> (<span class=literal>null</span> == mappedFile) {</span><br><span class=line> <span class=comment>// <span class=doctag>XXX:</span> warn and notify me</span></span><br><span class=line> log.error(<span class=string>"create mapped file2 error, topic: "</span> + msg.getTopic() + <span class=string>" clientAddr: "</span> + msg.getBornHostString());</span><br><span class=line> beginTimeInLock = <span class=number>0</span>;</span><br><span class=line> <span class=keyword>return</span> <span class=keyword>new</span> <span class="title class_">PutMessageResult</span>(PutMessageStatus.CREATE_MAPEDFILE_FAILED, result);</span><br><span class=line> }</span><br><span class=line> result = mappedFile.appendMessage(msg, <span class=built_in>this</span>.appendMessageCallback);</span><br><span class=line> <span class=keyword>break</span>;</span><br><span class=line> <span class=keyword>case</span> MESSAGE_SIZE_EXCEEDED:</span><br><span class=line> <span class=keyword>case</span> PROPERTIES_SIZE_EXCEEDED:</span><br><span class=line> beginTimeInLock = <span class=number>0</span>;</span><br><span class=line> <span class=keyword>return</span> <span class=keyword>new</span> <span class="title class_">PutMessageResult</span>(PutMessageStatus.MESSAGE_ILLEGAL, result);</span><br><span class=line> <span class=keyword>case</span> UNKNOWN_ERROR:</span><br><span class=line> beginTimeInLock = <span class=number>0</span>;</span><br><span class=line> <span class=keyword>return</span> <span class=keyword>new</span> <span class="title class_">PutMessageResult</span>(PutMessageStatus.UNKNOWN_ERROR, result);</span><br><span class=line> <span class=keyword>default</span>:</span><br><span class=line> beginTimeInLock = <span class=number>0</span>;</span><br><span class=line> <span class=keyword>return</span> <span class=keyword>new</span> <span class="title class_">PutMessageResult</span>(PutMessageStatus.UNKNOWN_ERROR, result);</span><br><span class=line> }</span><br><span class=line></span><br><span class=line> eclipseTimeInLock = <span class=built_in>this</span>.defaultMessageStore.getSystemClock().now() - beginLockTimestamp;</span><br><span class=line> beginTimeInLock = <span class=number>0</span>;</span><br><span class=line> } <span class=keyword>finally</span> {</span><br><span class=line> putMessageLock.unlock();</span><br><span class=line> }</span><br><span class=line></span><br><span class=line> <span class=keyword>if</span> (eclipseTimeInLock > <span class=number>500</span>) {</span><br><span class=line> log.warn(<span class=string>"[NOTIFYME]putMessage in lock cost time(ms)={}, bodyLength={} AppendMessageResult={}"</span>, eclipseTimeInLock, msg.getBody().length, result);</span><br><span class=line> }</span><br><span class=line></span><br><span class=line> <span class=keyword>if</span> (<span class=literal>null</span> != unlockMappedFile && <span class=built_in>this</span>.defaultMessageStore.getMessageStoreConfig().isWarmMapedFileEnable()) {</span><br><span class=line> <span class=built_in>this</span>.defaultMessageStore.unlockMappedFile(unlockMappedFile);</span><br><span class=line> }</span><br><span class=line></span><br><span class=line> <span class=type>PutMessageResult</span> <span class=variable>putMessageResult</span> <span class=operator>=</span> <span class=keyword>new</span> <span class="title class_">PutMessageResult</span>(PutMessageStatus.PUT_OK, result);</span><br><span class=line></span><br><span class=line> <span class=comment>// Statistics</span></span><br><span class=line> storeStatsService.getSinglePutMessageTopicTimesTotal(msg.getTopic()).incrementAndGet();</span><br><span class=line> storeStatsService.getSinglePutMessageTopicSizeTotal(topic).addAndGet(result.getWroteBytes());</span><br><span class=line></span><br><span class=line> handleDiskFlush(result, putMessageResult, msg);</span><br><span class=line> handleHA(result, putMessageResult, msg);</span><br><span class=line></span><br><span class=line> <span class=keyword>return</span> putMessageResult;</span><br><span class=line> }</span><br></pre></table></figure><p>前面也看到在CommitLog 目录下是有大小为 1G 的文件组成,在实现逻辑中,其实是通过 <code>org.apache.rocketmq.store.MappedFileQueue</code> ,内部是存的一个<code>MappedFile</code>的队列,对于写入的场景每次都是通过<code>org.apache.rocketmq.store.MappedFileQueue#getLastMappedFile()</code> 获取最后一个文件,如果还没有创建,或者最后这个文件已经满了,那就调用 <code>org.apache.rocketmq.store.MappedFileQueue#getLastMappedFile(long)</code><figure class="highlight java"><table><tr><td class=gutter><pre><span class=line>1</span><br><span class=line>2</span><br><span class=line>3</span><br><span class=line>4</span><br><span class=line>5</span><br><span class=line>6</span><br><span class=line>7</span><br><span class=line>8</span><br><span class=line>9</span><br><span class=line>10</span><br><span class=line>11</span><br><span class=line>12</span><br><span class=line>13</span><br><span class=line>14</span><br><span class=line>15</span><br><span class=line>16</span><br><span class=line>17</span><br><span class=line>18</span><br><span class=line>19</span><br><span class=line>20</span><br><span class=line>21</span><br><span class=line>22</span><br><span class=line>23</span><br><span class=line>24</span><br><span class=line>25</span><br><span class=line>26</span><br><span class=line>27</span><br><span class=line>28</span><br><span class=line>29</span><br><span class=line>30</span><br><span class=line>31</span><br><span class=line>32</span><br><span class=line>33</span><br><span class=line>34</span><br><span class=line>35</span><br><span class=line>36</span><br><span class=line>37</span><br><span class=line>38</span><br><span class=line>39</span><br><span class=line>40</span><br><span class=line>41</span><br><span class=line>42</span><br><span class=line>43</span><br><span class=line>44</span><br><span class=line>45</span><br><span class=line>46</span><br><span class=line>47</span><br><span class=line>48</span><br></pre><td class=code><pre><span class=line><span class=keyword>public</span> MappedFile <span class="title function_">getLastMappedFile</span><span class=params>(<span class=keyword>final</span> <span class=type>long</span> startOffset, <span class=type>boolean</span> needCreate)</span> {</span><br><span class=line> <span class=type>long</span> <span class=variable>createOffset</span> <span class=operator>=</span> -<span class=number>1</span>;</span><br><span class=line> <span class=comment>// 调用前面的方法,只是从 mappedFileQueue 获取最后一个</span></span><br><span class=line> <span class=type>MappedFile</span> <span class=variable>mappedFileLast</span> <span class=operator>=</span> getLastMappedFile();</span><br><span class=line></span><br><span class=line> <span class=comment>// 如果为空,计算下创建的偏移量</span></span><br><span class=line> <span class=keyword>if</span> (mappedFileLast == <span class=literal>null</span>) {</span><br><span class=line> createOffset = startOffset - (startOffset % <span class=built_in>this</span>.mappedFileSize);</span><br><span class=line> }</span><br><span class=line> </span><br><span class=line> <span class=comment>// 如果不为空,但是当前的文件写满了</span></span><br><span class=line> <span class=keyword>if</span> (mappedFileLast != <span class=literal>null</span> && mappedFileLast.isFull()) {</span><br><span class=line> <span class=comment>// 前一个的偏移量加上单个文件的偏移量,也就是 1G</span></span><br><span class=line> createOffset = mappedFileLast.getFileFromOffset() + <span class=built_in>this</span>.mappedFileSize;</span><br><span class=line> }</span><br><span class=line></span><br><span class=line> <span class=keyword>if</span> (createOffset != -<span class=number>1</span> && needCreate) {</span><br><span class=line> <span class=comment>// 根据 createOffset 转换成文件名进行创建</span></span><br><span class=line> <span class=type>String</span> <span class=variable>nextFilePath</span> <span class=operator>=</span> <span class=built_in>this</span>.storePath + File.separator + UtilAll.offset2FileName(createOffset);</span><br><span class=line> <span class=type>String</span> <span class=variable>nextNextFilePath</span> <span class=operator>=</span> <span class=built_in>this</span>.storePath + File.separator</span><br><span class=line> + UtilAll.offset2FileName(createOffset + <span class=built_in>this</span>.mappedFileSize);</span><br><span class=line> <span class=type>MappedFile</span> <span class=variable>mappedFile</span> <span class=operator>=</span> <span class=literal>null</span>;</span><br><span class=line></span><br><span class=line> <span class=comment>// 这里如果allocateMappedFileService 存在,就提交请求</span></span><br><span class=line> <span class=keyword>if</span> (<span class=built_in>this</span>.allocateMappedFileService != <span class=literal>null</span>) {</span><br><span class=line> mappedFile = <span class=built_in>this</span>.allocateMappedFileService.putRequestAndReturnMappedFile(nextFilePath,</span><br><span class=line> nextNextFilePath, <span class=built_in>this</span>.mappedFileSize);</span><br><span class=line> } <span class=keyword>else</span> {</span><br><span class=line> <span class=keyword>try</span> {</span><br><span class=line> <span class=comment>// 否则就直接创建</span></span><br><span class=line> mappedFile = <span class=keyword>new</span> <span class="title class_">MappedFile</span>(nextFilePath, <span class=built_in>this</span>.mappedFileSize);</span><br><span class=line> } <span class=keyword>catch</span> (IOException e) {</span><br><span class=line> log.error(<span class=string>"create mappedFile exception"</span>, e);</span><br><span class=line> }</span><br><span class=line> }</span><br><span class=line></span><br><span class=line> <span class=keyword>if</span> (mappedFile != <span class=literal>null</span>) {</span><br><span class=line> <span class=keyword>if</span> (<span class=built_in>this</span>.mappedFiles.isEmpty()) {</span><br><span class=line> mappedFile.setFirstCreateInQueue(<span class=literal>true</span>);</span><br><span class=line> }</span><br><span class=line> <span class=built_in>this</span>.mappedFiles.add(mappedFile);</span><br><span class=line> }</span><br><span class=line></span><br><span class=line> <span class=keyword>return</span> mappedFile;</span><br><span class=line> }</span><br><span class=line></span><br><span class=line> <span class=keyword>return</span> mappedFileLast;</span><br><span class=line> }</span><br></pre></table></figure><p>首先看下直接创建的,<figure class="highlight java"><table><tr><td class=gutter><pre><span class=line>1</span><br><span class=line>2</span><br><span class=line>3</span><br><span class=line>4</span><br><span class=line>5</span><br><span class=line>6</span><br><span class=line>7</span><br><span class=line>8</span><br><span class=line>9</span><br><span class=line>10</span><br><span class=line>11</span><br><span class=line>12</span><br><span class=line>13</span><br><span class=line>14</span><br><span class=line>15</span><br><span class=line>16</span><br><span class=line>17</span><br><span class=line>18</span><br><span class=line>19</span><br><span class=line>20</span><br><span class=line>21</span><br><span class=line>22</span><br><span class=line>23</span><br><span class=line>24</span><br><span class=line>25</span><br><span class=line>26</span><br><span class=line>27</span><br><span class=line>28</span><br><span class=line>29</span><br><span class=line>30</span><br><span class=line>31</span><br><span class=line>32</span><br></pre><td class=code><pre><span class=line><span class=keyword>public</span> <span class="title function_">MappedFile</span><span class=params>(<span class=keyword>final</span> String fileName, <span class=keyword>final</span> <span class=type>int</span> fileSize)</span> <span class=keyword>throws</span> IOException {</span><br><span class=line> init(fileName, fileSize);</span><br><span class=line> }</span><br><span class=line><span class=keyword>private</span> <span class=keyword>void</span> <span class="title function_">init</span><span class=params>(<span class=keyword>final</span> String fileName, <span class=keyword>final</span> <span class=type>int</span> fileSize)</span> <span class=keyword>throws</span> IOException {</span><br><span class=line> <span class=built_in>this</span>.fileName = fileName;</span><br><span class=line> <span class=built_in>this</span>.fileSize = fileSize;</span><br><span class=line> <span class=built_in>this</span>.file = <span class=keyword>new</span> <span class="title class_">File</span>(fileName);</span><br><span class=line> <span class=built_in>this</span>.fileFromOffset = Long.parseLong(<span class=built_in>this</span>.file.getName());</span><br><span class=line> <span class=type>boolean</span> <span class=variable>ok</span> <span class=operator>=</span> <span class=literal>false</span>;</span><br><span class=line></span><br><span class=line> ensureDirOK(<span class=built_in>this</span>.file.getParent());</span><br><span class=line></span><br><span class=line> <span class=keyword>try</span> {</span><br><span class=line> <span class=comment>// 通过 RandomAccessFile 创建 fileChannel</span></span><br><span class=line> <span class=built_in>this</span>.fileChannel = <span class=keyword>new</span> <span class="title class_">RandomAccessFile</span>(<span class=built_in>this</span>.file, <span class=string>"rw"</span>).getChannel();</span><br><span class=line> <span class=comment>// 做 mmap 映射</span></span><br><span class=line> <span class=built_in>this</span>.mappedByteBuffer = <span class=built_in>this</span>.fileChannel.map(MapMode.READ_WRITE, <span class=number>0</span>, fileSize);</span><br><span class=line> TOTAL_MAPPED_VIRTUAL_MEMORY.addAndGet(fileSize);</span><br><span class=line> TOTAL_MAPPED_FILES.incrementAndGet();</span><br><span class=line> ok = <span class=literal>true</span>;</span><br><span class=line> } <span class=keyword>catch</span> (FileNotFoundException e) {</span><br><span class=line> log.error(<span class=string>"create file channel "</span> + <span class=built_in>this</span>.fileName + <span class=string>" Failed. "</span>, e);</span><br><span class=line> <span class=keyword>throw</span> e;</span><br><span class=line> } <span class=keyword>catch</span> (IOException e) {</span><br><span class=line> log.error(<span class=string>"map file "</span> + <span class=built_in>this</span>.fileName + <span class=string>" Failed. "</span>, e);</span><br><span class=line> <span class=keyword>throw</span> e;</span><br><span class=line> } <span class=keyword>finally</span> {</span><br><span class=line> <span class=keyword>if</span> (!ok && <span class=built_in>this</span>.fileChannel != <span class=literal>null</span>) {</span><br><span class=line> <span class=built_in>this</span>.fileChannel.close();</span><br><span class=line> }</span><br><span class=line> }</span><br><span class=line> }</span><br></pre></table></figure><p>如果是提交给<code>AllocateMappedFileService</code>的话就用到了一些异步操作<figure class="highlight java"><table><tr><td class=gutter><pre><span class=line>1</span><br><span class=line>2</span><br><span class=line>3</span><br><span class=line>4</span><br><span class=line>5</span><br><span class=line>6</span><br><span class=line>7</span><br><span class=line>8</span><br><span class=line>9</span><br><span class=line>10</span><br><span class=line>11</span><br><span class=line>12</span><br><span class=line>13</span><br><span class=line>14</span><br><span class=line>15</span><br><span class=line>16</span><br><span class=line>17</span><br><span class=line>18</span><br><span class=line>19</span><br><span class=line>20</span><br><span class=line>21</span><br><span class=line>22</span><br><span class=line>23</span><br><span class=line>24</span><br><span class=line>25</span><br><span class=line>26</span><br><span class=line>27</span><br><span class=line>28</span><br><span class=line>29</span><br><span class=line>30</span><br><span class=line>31</span><br><span class=line>32</span><br><span class=line>33</span><br><span class=line>34</span><br><span class=line>35</span><br><span class=line>36</span><br><span class=line>37</span><br><span class=line>38</span><br><span class=line>39</span><br><span class=line>40</span><br><span class=line>41</span><br><span class=line>42</span><br><span class=line>43</span><br><span class=line>44</span><br><span class=line>45</span><br><span class=line>46</span><br><span class=line>47</span><br><span class=line>48</span><br><span class=line>49</span><br><span class=line>50</span><br><span class=line>51</span><br><span class=line>52</span><br><span class=line>53</span><br><span class=line>54</span><br><span class=line>55</span><br><span class=line>56</span><br><span class=line>57</span><br><span class=line>58</span><br><span class=line>59</span><br><span class=line>60</span><br><span class=line>61</span><br><span class=line>62</span><br><span class=line>63</span><br><span class=line>64</span><br><span class=line>65</span><br><span class=line>66</span><br><span class=line>67</span><br><span class=line>68</span><br><span class=line>69</span><br><span class=line>70</span><br></pre><td class=code><pre><span class=line><span class=keyword>public</span> MappedFile <span class="title function_">putRequestAndReturnMappedFile</span><span class=params>(String nextFilePath, String nextNextFilePath, <span class=type>int</span> fileSize)</span> {</span><br><span class=line> <span class=type>int</span> <span class=variable>canSubmitRequests</span> <span class=operator>=</span> <span class=number>2</span>;</span><br><span class=line> <span class=keyword>if</span> (<span class=built_in>this</span>.messageStore.getMessageStoreConfig().isTransientStorePoolEnable()) {</span><br><span class=line> <span class=keyword>if</span> (<span class=built_in>this</span>.messageStore.getMessageStoreConfig().isFastFailIfNoBufferInStorePool()</span><br><span class=line> && BrokerRole.SLAVE != <span class=built_in>this</span>.messageStore.getMessageStoreConfig().getBrokerRole()) { <span class=comment>//if broker is slave, don't fast fail even no buffer in pool</span></span><br><span class=line> canSubmitRequests = <span class=built_in>this</span>.messageStore.getTransientStorePool().remainBufferNumbs() - <span class=built_in>this</span>.requestQueue.size();</span><br><span class=line> }</span><br><span class=line> }</span><br><span class=line> <span class=comment>// 将请求放在 requestTable 中</span></span><br><span class=line> <span class=type>AllocateRequest</span> <span class=variable>nextReq</span> <span class=operator>=</span> <span class=keyword>new</span> <span class="title class_">AllocateRequest</span>(nextFilePath, fileSize);</span><br><span class=line> <span class=type>boolean</span> <span class=variable>nextPutOK</span> <span class=operator>=</span> <span class=built_in>this</span>.requestTable.putIfAbsent(nextFilePath, nextReq) == <span class=literal>null</span>;</span><br><span class=line> <span class=comment>// requestTable 使用了 concurrentHashMap,用文件名作为 key,防止并发</span></span><br><span class=line> <span class=keyword>if</span> (nextPutOK) {</span><br><span class=line> <span class=comment>// 这里判断了是否可以提交到 TransientStorePool,涉及读写分离,后面再细聊</span></span><br><span class=line> <span class=keyword>if</span> (canSubmitRequests <= <span class=number>0</span>) {</span><br><span class=line> log.warn(<span class=string>"[NOTIFYME]TransientStorePool is not enough, so create mapped file error, "</span> +</span><br><span class=line> <span class=string>"RequestQueueSize : {}, StorePoolSize: {}"</span>, <span class=built_in>this</span>.requestQueue.size(), <span class=built_in>this</span>.messageStore.getTransientStorePool().remainBufferNumbs());</span><br><span class=line> <span class=built_in>this</span>.requestTable.remove(nextFilePath);</span><br><span class=line> <span class=keyword>return</span> <span class=literal>null</span>;</span><br><span class=line> }</span><br><span class=line> <span class=comment>// 塞到阻塞队列中</span></span><br><span class=line> <span class=type>boolean</span> <span class=variable>offerOK</span> <span class=operator>=</span> <span class=built_in>this</span>.requestQueue.offer(nextReq);</span><br><span class=line> <span class=keyword>if</span> (!offerOK) {</span><br><span class=line> log.warn(<span class=string>"never expected here, add a request to preallocate queue failed"</span>);</span><br><span class=line> }</span><br><span class=line> canSubmitRequests--;</span><br><span class=line> }</span><br><span class=line></span><br><span class=line> <span class=comment>// 这里的两个提交我猜测是为了多生成一个 CommitLog,</span></span><br><span class=line> <span class=type>AllocateRequest</span> <span class=variable>nextNextReq</span> <span class=operator>=</span> <span class=keyword>new</span> <span class="title class_">AllocateRequest</span>(nextNextFilePath, fileSize);</span><br><span class=line> <span class=type>boolean</span> <span class=variable>nextNextPutOK</span> <span class=operator>=</span> <span class=built_in>this</span>.requestTable.putIfAbsent(nextNextFilePath, nextNextReq) == <span class=literal>null</span>;</span><br><span class=line> <span class=keyword>if</span> (nextNextPutOK) {</span><br><span class=line> <span class=keyword>if</span> (canSubmitRequests <= <span class=number>0</span>) {</span><br><span class=line> log.warn(<span class=string>"[NOTIFYME]TransientStorePool is not enough, so skip preallocate mapped file, "</span> +</span><br><span class=line> <span class=string>"RequestQueueSize : {}, StorePoolSize: {}"</span>, <span class=built_in>this</span>.requestQueue.size(), <span class=built_in>this</span>.messageStore.getTransientStorePool().remainBufferNumbs());</span><br><span class=line> <span class=built_in>this</span>.requestTable.remove(nextNextFilePath);</span><br><span class=line> } <span class=keyword>else</span> {</span><br><span class=line> <span class=type>boolean</span> <span class=variable>offerOK</span> <span class=operator>=</span> <span class=built_in>this</span>.requestQueue.offer(nextNextReq);</span><br><span class=line> <span class=keyword>if</span> (!offerOK) {</span><br><span class=line> log.warn(<span class=string>"never expected here, add a request to preallocate queue failed"</span>);</span><br><span class=line> }</span><br><span class=line> }</span><br><span class=line> }</span><br><span class=line></span><br><span class=line> <span class=keyword>if</span> (hasException) {</span><br><span class=line> log.warn(<span class=built_in>this</span>.getServiceName() + <span class=string>" service has exception. so return null"</span>);</span><br><span class=line> <span class=keyword>return</span> <span class=literal>null</span>;</span><br><span class=line> }</span><br><span class=line></span><br><span class=line> <span class=type>AllocateRequest</span> <span class=variable>result</span> <span class=operator>=</span> <span class=built_in>this</span>.requestTable.get(nextFilePath);</span><br><span class=line> <span class=keyword>try</span> {</span><br><span class=line> <span class=comment>// 这里就异步等着</span></span><br><span class=line> <span class=keyword>if</span> (result != <span class=literal>null</span>) {</span><br><span class=line> <span class=type>boolean</span> <span class=variable>waitOK</span> <span class=operator>=</span> result.getCountDownLatch().await(waitTimeOut, TimeUnit.MILLISECONDS);</span><br><span class=line> <span class=keyword>if</span> (!waitOK) {</span><br><span class=line> log.warn(<span class=string>"create mmap timeout "</span> + result.getFilePath() + <span class=string>" "</span> + result.getFileSize());</span><br><span class=line> <span class=keyword>return</span> <span class=literal>null</span>;</span><br><span class=line> } <span class=keyword>else</span> {</span><br><span class=line> <span class=built_in>this</span>.requestTable.remove(nextFilePath);</span><br><span class=line> <span class=keyword>return</span> result.getMappedFile();</span><br><span class=line> }</span><br><span class=line> } <span class=keyword>else</span> {</span><br><span class=line> log.error(<span class=string>"find preallocate mmap failed, this never happen"</span>);</span><br><span class=line> }</span><br><span class=line> } <span class=keyword>catch</span> (InterruptedException e) {</span><br><span class=line> log.warn(<span class=built_in>this</span>.getServiceName() + <span class=string>" service has exception. "</span>, e);</span><br><span class=line> }</span><br><span class=line></span><br><span class=line> <span class=keyword>return</span> <span class=literal>null</span>;</span><br><span class=line> }</span><br></pre></table></figure><p>而真正去执行文件操作的就是 <code>AllocateMappedFileService</code>的 run 方法<figure class="highlight java"><table><tr><td class=gutter><pre><span class=line>1</span><br><span class=line>2</span><br><span class=line>3</span><br><span class=line>4</span><br><span class=line>5</span><br><span class=line>6</span><br><span class=line>7</span><br><span class=line>8</span><br><span class=line>9</span><br><span class=line>10</span><br><span class=line>11</span><br><span class=line>12</span><br><span class=line>13</span><br><span class=line>14</span><br><span class=line>15</span><br><span class=line>16</span><br><span class=line>17</span><br><span class=line>18</span><br><span class=line>19</span><br><span class=line>20</span><br><span class=line>21</span><br><span class=line>22</span><br><span class=line>23</span><br><span class=line>24</span><br><span class=line>25</span><br><span class=line>26</span><br><span class=line>27</span><br><span class=line>28</span><br><span class=line>29</span><br><span class=line>30</span><br><span class=line>31</span><br><span class=line>32</span><br><span class=line>33</span><br><span class=line>34</span><br><span class=line>35</span><br><span class=line>36</span><br><span class=line>37</span><br><span class=line>38</span><br><span class=line>39</span><br><span class=line>40</span><br><span class=line>41</span><br><span class=line>42</span><br><span class=line>43</span><br><span class=line>44</span><br><span class=line>45</span><br><span class=line>46</span><br><span class=line>47</span><br><span class=line>48</span><br><span class=line>49</span><br><span class=line>50</span><br><span class=line>51</span><br><span class=line>52</span><br><span class=line>53</span><br><span class=line>54</span><br><span class=line>55</span><br><span class=line>56</span><br><span class=line>57</span><br><span class=line>58</span><br><span class=line>59</span><br><span class=line>60</span><br><span class=line>61</span><br><span class=line>62</span><br><span class=line>63</span><br><span class=line>64</span><br><span class=line>65</span><br><span class=line>66</span><br><span class=line>67</span><br><span class=line>68</span><br><span class=line>69</span><br><span class=line>70</span><br><span class=line>71</span><br><span class=line>72</span><br><span class=line>73</span><br><span class=line>74</span><br><span class=line>75</span><br><span class=line>76</span><br><span class=line>77</span><br><span class=line>78</span><br><span class=line>79</span><br><span class=line>80</span><br><span class=line>81</span><br><span class=line>82</span><br><span class=line>83</span><br><span class=line>84</span><br><span class=line>85</span><br><span class=line>86</span><br></pre><td class=code><pre><span class=line><span class=keyword>public</span> <span class=keyword>void</span> <span class="title function_">run</span><span class=params>()</span> {</span><br><span class=line> log.info(<span class=built_in>this</span>.getServiceName() + <span class=string>" service started"</span>);</span><br><span class=line></span><br><span class=line> <span class=keyword>while</span> (!<span class=built_in>this</span>.isStopped() && <span class=built_in>this</span>.mmapOperation()) {</span><br><span class=line></span><br><span class=line> }</span><br><span class=line> log.info(<span class=built_in>this</span>.getServiceName() + <span class=string>" service end"</span>);</span><br><span class=line> }</span><br><span class=line><span class=keyword>private</span> <span class=type>boolean</span> <span class="title function_">mmapOperation</span><span class=params>()</span> {</span><br><span class=line> <span class=type>boolean</span> <span class=variable>isSuccess</span> <span class=operator>=</span> <span class=literal>false</span>;</span><br><span class=line> <span class=type>AllocateRequest</span> <span class=variable>req</span> <span class=operator>=</span> <span class=literal>null</span>;</span><br><span class=line> <span class=keyword>try</span> {</span><br><span class=line> <span class=comment>// 从阻塞队列里获取请求</span></span><br><span class=line> req = <span class=built_in>this</span>.requestQueue.take();</span><br><span class=line> <span class=type>AllocateRequest</span> <span class=variable>expectedRequest</span> <span class=operator>=</span> <span class=built_in>this</span>.requestTable.get(req.getFilePath());</span><br><span class=line> <span class=keyword>if</span> (<span class=literal>null</span> == expectedRequest) {</span><br><span class=line> log.warn(<span class=string>"this mmap request expired, maybe cause timeout "</span> + req.getFilePath() + <span class=string>" "</span></span><br><span class=line> + req.getFileSize());</span><br><span class=line> <span class=keyword>return</span> <span class=literal>true</span>;</span><br><span class=line> }</span><br><span class=line> <span class=keyword>if</span> (expectedRequest != req) {</span><br><span class=line> log.warn(<span class=string>"never expected here, maybe cause timeout "</span> + req.getFilePath() + <span class=string>" "</span></span><br><span class=line> + req.getFileSize() + <span class=string>", req:"</span> + req + <span class=string>", expectedRequest:"</span> + expectedRequest);</span><br><span class=line> <span class=keyword>return</span> <span class=literal>true</span>;</span><br><span class=line> }</span><br><span class=line></span><br><span class=line> <span class=keyword>if</span> (req.getMappedFile() == <span class=literal>null</span>) {</span><br><span class=line> <span class=type>long</span> <span class=variable>beginTime</span> <span class=operator>=</span> System.currentTimeMillis();</span><br><span class=line></span><br><span class=line> MappedFile mappedFile;</span><br><span class=line> <span class=keyword>if</span> (messageStore.getMessageStoreConfig().isTransientStorePoolEnable()) {</span><br><span class=line> <span class=keyword>try</span> {</span><br><span class=line> <span class=comment>// 通过 transientStorePool 创建</span></span><br><span class=line> mappedFile = ServiceLoader.load(MappedFile.class).iterator().next();</span><br><span class=line> mappedFile.init(req.getFilePath(), req.getFileSize(), messageStore.getTransientStorePool());</span><br><span class=line> } <span class=keyword>catch</span> (RuntimeException e) {</span><br><span class=line> log.warn(<span class=string>"Use default implementation."</span>);</span><br><span class=line> <span class=comment>// 默认创建</span></span><br><span class=line> mappedFile = <span class=keyword>new</span> <span class="title class_">MappedFile</span>(req.getFilePath(), req.getFileSize(), messageStore.getTransientStorePool());</span><br><span class=line> }</span><br><span class=line> } <span class=keyword>else</span> {</span><br><span class=line> <span class=comment>// 默认创建</span></span><br><span class=line> mappedFile = <span class=keyword>new</span> <span class="title class_">MappedFile</span>(req.getFilePath(), req.getFileSize());</span><br><span class=line> }</span><br><span class=line></span><br><span class=line> <span class=type>long</span> <span class=variable>eclipseTime</span> <span class=operator>=</span> UtilAll.computeEclipseTimeMilliseconds(beginTime);</span><br><span class=line> <span class=keyword>if</span> (eclipseTime > <span class=number>10</span>) {</span><br><span class=line> <span class=type>int</span> <span class=variable>queueSize</span> <span class=operator>=</span> <span class=built_in>this</span>.requestQueue.size();</span><br><span class=line> log.warn(<span class=string>"create mappedFile spent time(ms) "</span> + eclipseTime + <span class=string>" queue size "</span> + queueSize</span><br><span class=line> + <span class=string>" "</span> + req.getFilePath() + <span class=string>" "</span> + req.getFileSize());</span><br><span class=line> }</span><br><span class=line></span><br><span class=line> <span class=comment>// pre write mappedFile</span></span><br><span class=line> <span class=keyword>if</span> (mappedFile.getFileSize() >= <span class=built_in>this</span>.messageStore.getMessageStoreConfig()</span><br><span class=line> .getMapedFileSizeCommitLog()</span><br><span class=line> &&</span><br><span class=line> <span class=built_in>this</span>.messageStore.getMessageStoreConfig().isWarmMapedFileEnable()) {</span><br><span class=line> mappedFile.warmMappedFile(<span class=built_in>this</span>.messageStore.getMessageStoreConfig().getFlushDiskType(),</span><br><span class=line> <span class=built_in>this</span>.messageStore.getMessageStoreConfig().getFlushLeastPagesWhenWarmMapedFile());</span><br><span class=line> }</span><br><span class=line></span><br><span class=line> req.setMappedFile(mappedFile);</span><br><span class=line> <span class=built_in>this</span>.hasException = <span class=literal>false</span>;</span><br><span class=line> isSuccess = <span class=literal>true</span>;</span><br><span class=line> }</span><br><span class=line> } <span class=keyword>catch</span> (InterruptedException e) {</span><br><span class=line> log.warn(<span class=built_in>this</span>.getServiceName() + <span class=string>" interrupted, possibly by shutdown."</span>);</span><br><span class=line> <span class=built_in>this</span>.hasException = <span class=literal>true</span>;</span><br><span class=line> <span class=keyword>return</span> <span class=literal>false</span>;</span><br><span class=line> } <span class=keyword>catch</span> (IOException e) {</span><br><span class=line> log.warn(<span class=built_in>this</span>.getServiceName() + <span class=string>" service has exception. "</span>, e);</span><br><span class=line> <span class=built_in>this</span>.hasException = <span class=literal>true</span>;</span><br><span class=line> <span class=keyword>if</span> (<span class=literal>null</span> != req) {</span><br><span class=line> requestQueue.offer(req);</span><br><span class=line> <span class=keyword>try</span> {</span><br><span class=line> Thread.sleep(<span class=number>1</span>);</span><br><span class=line> } <span class=keyword>catch</span> (InterruptedException ignored) {</span><br><span class=line> }</span><br><span class=line> }</span><br><span class=line> } <span class=keyword>finally</span> {</span><br><span class=line> <span class=keyword>if</span> (req != <span class=literal>null</span> && isSuccess)</span><br><span class=line> <span class=comment>// 通知前面等待的</span></span><br><span class=line> req.getCountDownLatch().countDown();</span><br><span class=line> }</span><br><span class=line> <span class=keyword>return</span> <span class=literal>true</span>;</span><br><span class=line> }</span><br></pre></table></figure></div><footer class=post-footer><div class="affiliate-creative affiliate-creative--post-end" data-affiliate-id=tencent-cloud-general><a rel="sponsored noopener" aria-label=腾讯云产品优惠 href=https://cloud.tencent.com/act/cps/redirect?redirect=2446&cps_key=2e33a1eedb380997d03618b4a5ba6ca7&from=console target=_blank><img alt=腾讯云服务器、数据库和云产品优惠 decoding=async height=419 loading=lazy src=/uploads/affiliate/tencent-cloud-general.webp width=690> </a><span class=affiliate-disclosure>推广</span></div><div class=reward-container><div>请我一杯咖啡吧!</div><button>赞赏</button><div class=post-reward><div><img alt="Nicksxs 微信" src=https://i.loli.net/2020/01/12/NUlWanT31E8LQGx.png><span>微信</span></div><div><img alt="Nicksxs 支付宝" src=https://ooo.0o0.ooo/2016/08/17/57b421e773057.jpg><span>支付宝</span></div></div></div><div class=post-copyright><ul><li class=post-copyright-author><strong>本文作者: </strong>Nicksxs<li class=post-copyright-link><strong>本文链接:</strong> <a title="聊一下 RocketMQ 的消息存储二" href=https://nicksxs.me/2021/09/12/%E8%81%8A%E4%B8%80%E4%B8%8B-RocketMQ-%E7%9A%84%E6%B6%88%E6%81%AF%E5%AD%98%E5%82%A8%E4%BA%8C/>https://nicksxs.me/2021/09/12/聊一下-RocketMQ-的消息存储二/</a><li class=post-copyright-license><strong>版权声明: </strong>本博客所有文章除特别声明外,均采用 <a href=https://creativecommons.org/licenses/by-nc-sa/4.0/ rel=noopener target=_blank><i class="fab fa-fw fa-creative-commons"></i>BY-NC-SA</a> 许可协议。转载请注明出处!</ul></div><div class=post-tags><a href=/tags/MQ/ rel=tag># MQ</a><a href=/tags/%E6%B6%88%E6%81%AF%E9%98%9F%E5%88%97/ rel=tag># 消息队列</a><a href=/tags/RocketMQ/ rel=tag># RocketMQ</a></div><div class=post-nav><div class=post-nav-item><a title="聊一下 RocketMQ 的消息存储之 MMAP" href=/2021/09/04/%E8%81%8A%E4%B8%80%E4%B8%8B-RocketMQ-%E7%9A%84%E6%B6%88%E6%81%AF%E5%AD%98%E5%82%A8/ rel=prev><i class="fa fa-angle-left"></i> 聊一下 RocketMQ 的消息存储之 MMAP</a></div><div class=post-nav-item><a title="聊一下 SpringBoot 中使用的 cglib 作为动态代理中的一个注意点" href=/2021/09/19/%E8%81%8A%E4%B8%80%E4%B8%8B-SpringBoot-%E4%B8%AD%E4%BD%BF%E7%94%A8%E7%9A%84-cglib-%E4%BD%9C%E4%B8%BA%E5%8A%A8%E6%80%81%E4%BB%A3%E7%90%86%E4%B8%AD%E7%9A%84%E4%B8%80%E4%B8%AA%E6%B3%A8%E6%84%8F%E7%82%B9/ rel=next>聊一下 SpringBoot 中使用的 cglib 作为动态代理中的一个注意点 <i class="fa fa-angle-right"></i></a></div></div></footer></article></div><div class=comments id=disqus_thread><noscript>Please enable JavaScript to view the comments powered by Disqus.</noscript></div></div></main><footer class=footer><div class=footer-inner><div class=beian><a href=https://beian.miit.gov.cn/ rel=noopener target=_blank>浙ICP备16002580号-2 </a><img alt src=/uploads/beian.png><a href=https://beian.mps.gov.cn/#/query/webSearch?code=33010602011094 rel=noopener target=_blank>浙公网安备 33010602011094号</a></div><div class=copyright>© <span itemprop=copyrightYear>2026</span><span class=with-love><i class="fa fa-heart"></i> </span><span class=author itemprop=copyrightHolder>Nicksxs</span></div></div></footer><div class="toggle sidebar-toggle" role=button><span class=toggle-line></span><span class=toggle-line></span><span class=toggle-line></span></div><div class=sidebar-dimmer></div><div aria-label=返回顶部 class=back-to-top role=button><i class="fa fa-arrow-up fa-lg"></i><span>0%</span></div><a aria-label="在 GitHub 上关注我" title="在 GitHub 上关注我" class=github-corner href=https://github.com/nicksxs rel=noopener target=_blank><svg viewbox="0 0 250 250" aria-hidden=true height=80 width=80><path d="M0,0 L115,115 L130,115 L142,142 L250,250 L250,0 Z"></path><path d="M128.3,109.0 C113.8,99.7 119.0,89.6 119.0,89.6 C122.0,82.7 120.5,78.6 120.5,78.6 C119.2,72.0 123.4,76.3 123.4,76.3 C127.3,80.9 125.5,87.3 125.5,87.3 C122.9,97.6 130.6,101.9 134.4,103.2" style="transform-origin:130px 106px" class=octo-arm fill=currentColor></path><path d="M115.0,115.0 C114.9,115.1 118.7,116.5 119.8,115.4 L133.7,101.6 C136.9,99.2 139.9,98.4 142.2,98.6 C133.8,88.0 127.5,74.4 143.8,58.0 C148.5,53.4 154.0,51.2 159.7,51.0 C160.3,49.4 163.2,43.6 171.4,40.1 C171.4,40.1 176.1,42.5 178.8,56.2 C183.1,58.6 187.2,61.8 190.9,65.4 C194.5,69.0 197.7,73.2 200.1,77.6 C213.8,80.2 216.3,84.9 216.3,84.9 C212.7,93.1 206.9,96.0 205.4,96.6 C205.1,102.4 203.0,107.8 198.3,112.5 C181.9,128.9 168.3,122.5 157.7,114.1 C157.9,116.9 156.7,120.9 152.7,124.9 L141.0,136.5 C139.8,137.7 141.6,141.9 141.8,141.8 Z" class=octo-body fill=currentColor></path></svg></a><noscript><div class=noscript-warning>Theme NexT works best with JavaScript enabled</div></noscript><script data-cloudflare-page-counter-config type=application/json>{"enable":true,"endpoint":"https://counter.nicksxs.me","site":"nicksxs","local_cache":true,"timeout":5000,"icon":"far fa-eye","item_text":true}</script><script src=/js/cloudflare-page-counter.js></script><script class=next-config data-name=disqus type=application/json>{"enable":true,"shortname":"nicksxs","count":false,"i18n":{"disqus":"disqus"}}</script><script defer src=/js/third-party/comments/disqus.js></script>