llm-conf

Running

App Files Files Community

muellerzr HF staff commited on May 21

Commit

b06653e

•

1 Parent(s): fbff59d

Mv

Browse files

Files changed (2) hide show

index.html +594 -346
llm_conf.html +0 -1337

index.html CHANGED Viewed

@@ -1,21 +1,21 @@
 <!DOCTYPE html>
 <html lang="en"><head>
-<script src="Accelerate_files/libs/clipboard/clipboard.min.js"></script>
-<script src="Accelerate_files/libs/quarto-html/tabby.min.js"></script>
-<script src="Accelerate_files/libs/quarto-html/popper.min.js"></script>
-<script src="Accelerate_files/libs/quarto-html/tippy.umd.min.js"></script>
-<link href="Accelerate_files/libs/quarto-html/tippy.css" rel="stylesheet">
-<link href="Accelerate_files/libs/quarto-html/quarto-html.min.css" rel="stylesheet" data-mode="light">
-<link href="Accelerate_files/libs/quarto-html/quarto-syntax-highlighting-dark.css" rel="stylesheet" id="quarto-text-highlighting-styles"><meta charset="utf-8">
-  <meta name="generator" content="quarto-1.2.237">
-  <meta name="author" content="Zachary Mueller">
-  <title>Accelerate, Three Powerful Sublibraries for PyTorch</title>
   <meta name="apple-mobile-web-app-capable" content="yes">
   <meta name="apple-mobile-web-app-status-bar-style" content="black-translucent">
   <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no, minimal-ui">
-  <link rel="stylesheet" href="Accelerate_files/libs/revealjs/dist/reset.css">
-  <link rel="stylesheet" href="Accelerate_files/libs/revealjs/dist/reveal.css">
   <style>
     code{white-space: pre-wrap;}
     span.smallcaps{font-variant: small-caps;}
@@ -25,11 +25,12 @@
     ul.task-list{list-style: none;}
     ul.task-list li input[type="checkbox"] {
       width: 0.8em;
-      margin: 0 0.8em 0.2em -1.6em;
       vertical-align: middle;
     }
     pre > code.sourceCode { white-space: pre; position: relative; }
-    pre > code.sourceCode > span { display: inline-block; line-height: 1.25; }
     pre > code.sourceCode > span:empty { height: 1.2em; }
     .sourceCode { overflow: visible; }
     code.sourceCode > span { color: inherit; text-decoration: inherit; }
@@ -93,11 +94,11 @@
     code span.vs { color: #abe338; } /* VerbatimString */
     code span.wa { color: #dcc6e0; } /* Warning */
   </style>
-  <link rel="stylesheet" href="Accelerate_files/libs/revealjs/dist/theme/quarto.css" id="theme">
-  <link href="Accelerate_files/libs/revealjs/plugin/quarto-line-highlight/line-highlight.css" rel="stylesheet">
-  <link href="Accelerate_files/libs/revealjs/plugin/reveal-menu/menu.css" rel="stylesheet">
-  <link href="Accelerate_files/libs/revealjs/plugin/reveal-menu/quarto-menu.css" rel="stylesheet">
-  <link href="Accelerate_files/libs/revealjs/plugin/quarto-support/footer.css" rel="stylesheet">
   <style type="text/css">
   .callout {
@@ -136,44 +137,44 @@
     font-weight: 400;
   }
-  .callout.callout-captioned.callout-style-simple .callout-body {
     margin-top: 0.2em;
   }
-  .callout:not(.callout-captioned) .callout-body {
       display: flex;
   }
-  .callout:not(.no-icon).callout-captioned.callout-style-simple .callout-content {
     padding-left: 1.6em;
   }
-  .callout.callout-captioned .callout-header {
     padding-top: 0.2em;
     margin-bottom: -0.2em;
   }
-  .callout.callout-captioned .callout-caption  p {
     margin-top: 0.5em;
     margin-bottom: 0.5em;
   }
-  .callout.callout-captioned.callout-style-simple .callout-content  p {
     margin-top: 0;
   }
-  .callout.callout-captioned.callout-style-default .callout-content  p {
     margin-top: 0.7em;
   }
-  .callout.callout-style-simple div.callout-caption {
     border-bottom: none;
     font-size: .9rem;
     font-weight: 600;
     opacity: 75%;
   }
-  .callout.callout-style-default  div.callout-caption {
     border-bottom: none;
     font-weight: 600;
     opacity: 85%;
@@ -205,7 +206,7 @@
     background-size: 0.9rem 0.9rem;
   }
-  .callout-caption {
     display: flex
   }
@@ -218,16 +219,17 @@
     display: none !important;
   }
-  .callout.callout-captioned .callout-body > .callout-content > :last-child {
-    margin-bottom: 0.5rem;
   }
-  .callout.callout-captioned .callout-icon::before {
     margin-top: .5rem;
     padding-right: .5rem;
   }
-  .callout:not(.callout-captioned) .callout-icon::before {
     margin-top: 1rem;
     padding-right: .5rem;
   }
@@ -242,7 +244,7 @@
     background-image: url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAAAXNSR0IArs4c6QAAAERlWElmTU0AKgAAAAgAAYdpAAQAAAABAAAAGgAAAAAAA6ABAAMAAAABAAEAAKACAAQAAAABAAAAIKADAAQAAAABAAAAIAAAAACshmLzAAAEU0lEQVRYCcVXTWhcVRQ+586kSUMMxkyaElstCto2SIhitS5Ek8xUKV2poatCcVHtUlFQk8mbaaziwpWgglJwVaquitBOfhQXFlqlzSJpFSpIYyXNjBNiTCck7x2/8/LeNDOZxDuEkgOXe++553zfefee+/OYLOXFk3+1LLrRdiO81yNqZ6K9cG0P3MeFaMIQjXssE8Z1JzLO9ls20MBZX7oG8w9GxB0goaPrW5aNMp1yOZIa7Wv6o2ykpLtmAPs/vrG14Z+6d4jpbSKuhdcSyq9wGMPXjonwmESXrriLzFGOdDBLB8Y6MNYBu0dRokSygMA/mrun8MGFN3behm6VVAwg4WR3i6FvYK1T7MHo9BK7ydH+1uurECoouk5MPRyVSBrBHMYwVobG2aOXM07sWrn5qgB60rc6mcwIDJtQrnrEr44kmy+UO9r0u9O5/YbkS9juQckLed3DyW2XV/qWBBB3ptvI8EUY3I9p/67OW+g967TNr3Sotn3IuVlfMLVnsBwH4fsnebJvyGm5GeIUA3jljERmrv49SizPYuq+z7c2H/jlGC+Ghhupn/hcapqmcudB9jwJ/3jvnvu6vu5lVzF1fXyZuZZ7U8nRmVzytvT+H3kilYvH09mLWrQdwFSsFEsxFVs5fK7A0g8gMZjbif4ACpKbjv7gNGaD8bUrlk8x+KRflttr22JEMRUbTUwwDQScyzPgedQHZT0xnx7ujw2jfVfExwYHwOsDTjLdJ2ebmeQIlJ7neo41s/DrsL3kl+W2lWvAga0tR3zueGr6GL78M3ifH0rGXrBC2aAR8uYcIA5gwV8zIE8onoh8u0Fca/ciF7j1uOzEnqcIm59sEXoGc0+z6+H45V1CvAvHcD7THztu669cnp+L0okAeIc6zjbM/24LgGM1gZk7jnRu1aQWoU9sfUOuhrmtaPIO3YY1KLLWZaEO5TKUbMY5zx8W9UJ6elpLwKXbsaZ4EFl7B4bMtDv0iRipKoDQT2sNQI9b1utXFdYisi+wzZ/ri/1m7QfDgEuvgUUEIJPq3DhX/5DWNqIXDOweC2wvIR90Oq3lDpdMIgD2r0dXvGdsEW5H6x6HLRJYU7C69VefO1x8Gde1ZFSJLfWS1jbCnhtOPxmpfv2LXOA2Xk2tvnwKKPFuZ/oRmwBwqRQDcKNeVQkYcOjtWVBuM/JuYw5b6isojIkYxyYAFn5K7ZBF10fea52y8QltAg6jnMqNHFBmGkQ1j+U43HMi2xMar1Nv0zGsf1s8nUsmUtPOOrbFIR8bHFDMB5zL13Gmr/kGlCkUzedTzzmzsaJXhYawnA3UmARpiYj5ooJZiUoxFRtK3X6pgNPv+IZVPcnwbOl6f+aBaO1CNvPW9n9LmCp01nuSaTRF2YxHqZ8DYQT6WsXT+RD6eUztwYLZ8rM+rcPxamv1VQzFUkzFXvkiVrySGQgJNvXHJAxiU3/NwiC03rSf05VBaPtu/Z7/B8Yn/w7eguloAAAAAElFTkSuQmCC');
   }
-  div.callout-note.callout-style-default .callout-caption {
     background-color: #dae6fb
   }
@@ -254,7 +256,7 @@
     background-image: url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAAAXNSR0IArs4c6QAAAERlWElmTU0AKgAAAAgAAYdpAAQAAAABAAAAGgAAAAAAA6ABAAMAAAABAAEAAKACAAQAAAABAAAAIKADAAQAAAABAAAAIAAAAACshmLzAAAEKklEQVRYCcVXTWhcVRS+575MJym48A+hSRFr00ySRQhURRfd2HYjk2SSTokuBCkU2o0LoSKKraKIBTcuFCoidGFD08nkBzdREbpQ1EDNIv8qSGMFUboImMSZd4/f9zJv8ibJMC8xJQfO3HPPPef7zrvvvnvviIkpC9nsw0UttFunbUhpFzFtarSd6WJkStVMw5xyVqYTvkwfzuf/5FgtkVoB0729j1rjXwThS7Vio+Mo6DNnvLfahoZ+i/o32lULuJ3NNiz7q6+pyAUkJaFF6JwaM2lUJlV0MlnQn5aTRbEu0SEqHUa0A4AdiGuB1kFXRfVyg5d87+Dg4DL6m2TLAub60ilj7A1Ec4odSAc8X95sHh7+ZRPCFo6Fnp7HfU/fBng/hi10CjCnWnJjsxvDNxWw0NfV6Rv5GgP3I3jGWXumdTD/3cbEOP2ZbOZp69yniG3FQ9z1jD7bnBu9Fc2tKGC2q+uAJOQHBDRiZX1x36o7fWBs7J9ownbtO+n0/qWkvW7UPIfc37WgT6ZGR++EOJyeQDSb9UB+DZ1G6DdLDzyS+b/kBCYGsYgJbSQHuThGKRcw5xdeQf8YdNHsc6ePXrlSYMBuSIAFTGAtQo+VuALo4BX83N190NWZWbynBjhOHsmNfFWLeL6v+ynsA58zDvvAC8j5PkbOcXCMg2PZFk3q8MjI7WAG/Dp9AwP7jdGBOOQkAvlFUB+irtm16I1Zw9YBcpGTGXYmk3kQIC/Cds55l+iMI3jqhjAuaoe+am2Jw5GT3Nbz3CkE12NavmzN5+erJW7046n/CH1RO/RVa8lBLozXk9uqykkGAyRXLWlLv5jyp4RFsG5vGVzpDLnIjTWgnRy2Rr+tDKvRc7Y8AyZq10jj8DqXdnIRNtFZb+t/ZRtXcDiVnzpqx8mPcDWxgARUqx0W1QB9MeUZiNrV4qP+Ehc+BpNgATsTX8ozYKL2NtFYAHc84fG7ndxUPr+AR/iQSns7uSUufAymwDOb2+NjK27lEFocm/EE2WpyIy/Hi66MWuMKJn8RvxIcj87IM5Vh9663ziW36kR0HNenXuxmfaD8JC7tfKbrhFr7LiZCrMjrzTeGx+PmkosrkNzW94ObzwocJ7A1HokLolY+AvkTiD/q1H0cN48c5EL8Crkttsa/AXQVDmutfyku0E7jShx49XqV3MFK8IryDhYVbj7Sj2P2eBxwcXoe8T8idsKKPRcnZw1b+slFTubwUwhktrfnAt7J++jwQtLZcm3sr9LQrjRzz6cfMv9aLvgmnAGvpoaGLxM4mAEaLV7iAzQ3oU0IvD5x9ix3yF2RAAuYAOO2f7PEFWCXZ4C9Pb2UsgDeVnFSpbFK7/IWu7TPTvBqzbGdCHOJQSxiEjt6IyZmxQyEJHv6xyQsYk//moVFsN2zP6fRImjfq7/n/wFDguUQFNEwugAAAABJRU5ErkJggg==');
   }
-  div.callout-important.callout-style-default .callout-caption {
     background-color: #f7dddc
   }
@@ -266,7 +268,7 @@
     background-image: url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAAAXNSR0IArs4c6QAAAERlWElmTU0AKgAAAAgAAYdpAAQAAAABAAAAGgAAAAAAA6ABAAMAAAABAAEAAKACAAQAAAABAAAAIKADAAQAAAABAAAAIAAAAACshmLzAAAETklEQVRYCeVWW2gcVRg+58yaTUnizqbipZeX4uWhBEniBaoUX1Ioze52t7sRq6APio9V9MEaoWlVsFasRq0gltaAPuxms8lu0gcviE/FFOstVbSIxgcv6SU7EZqmdc7v9+9mJtNks51NTUH84ed889/PP+cmxP+d5FIbMJmNbpREu4WUkiTtCicKny0l1pIKmBzovF2S+hIJHX8iEu3hZJ5lNZGqyRrGSIQpq15AzF28jgpeY6yk6GVdrfFqdrD6Iw+QlB8g0YS2g7dyQmXM/IDhBhT0UCiRf59lfqmmDvzRt6kByV/m4JjtzuaujMUM2c5Z2d6JdKrRb3K2q6mA+oYVz8JnDdKPmmNthzkAk/lN63sYPgevrguc72aZX/L9C6x09GYyxBgCX4NlvyGUHOKELlm5rXeR1kchuChJt4SSwyddZRXgvwMGvYo4QSlk3/zkHD8UHxwVJA6zjZZqP8v8kK8OWLnIZtLyCAJagYC4rTGW/9Pqj92N/c+LUaAj27movwbi19tk/whRCIE7Q9vyI6yvRpftAKVTdUjOW40X3h5OXsKCdmFcx0xlLJoSuQngnrJe7Kcjm4OMq9FlC7CMmScQANuNvjfP3PjGXDBaUQmbp296S5L4DrpbrHN1T87ZVEZVCzg1FF0Ft+dKrlLukI+/c9ENo+TvlTDbYFvuKPtQ9+l052rXrgKoWkDAFnvh0wTOmYn8R5f4k/jN/fZiCM1tQx9jQQ4ANhqG4hiL0qIFTGViG9DKB7GYzgubnpofgYRwO+DFjh0Zin2m4b/97EDkXkc+f6xYAPX0KK2I/7fUQuwzuwo/L3AkcjugPNixC8cHf0FyPjWlItmLxWw4Ou9YsQCr5fijMGoD/zpdRy95HRysyXA74MWOnscpO4j2y3HAVisw85hX5+AFBRSHt4ShfLFkIMXTqyKFc46xdzQM6XbAi702a7sy04J0+feReMFKp5q9esYLCqAZYw/k14E/xcLLsFElaornTuJB0svMuJINy8xkIYuL+xPAlWRceH6+HX7THJ0djLUom46zREu7tTkxwmf/FdOZ/sh6Q8qvEAiHpm4PJ4a/doJe0gH1t+aHRgCzOvBvJedEK5OFE5jpm4AGP2a8Dxe3gGJ/pAutug9Gp6he92CsSsWBaEcxGx0FHytmIpuqGkOpldqNYQK8cSoXvd+xLxXADw0kf6UkJNFtdo5MOgaLjiQOQHcn+A6h5NuL2s0qsC2LOM75PcF3yr5STuBSAcGG+meA14K/CI21HcS4LBT6tv0QAh8Dr5l93AhZzG5ZJ4VxAqdZUEl9z7WJ4aN+svMvwHHL21UKTd1mqvChH7/Za5xzXBBKrUcB0TQ+Ulgkfbi/H/YT5EptrGzsEK7tR1B7ln9BBwckYfMiuSqklSznIuoIIOM42MQO+QnduCoFCI0bpkzjCjddHPN/F+2Yu+sd9bKNpVwHhbS3LluK/0zgfwD0xYI5dXuzlQAAAABJRU5ErkJggg==');
   }
-  div.callout-warning.callout-style-default .callout-caption {
     background-color: #fcefdc
   }
@@ -278,7 +280,7 @@
     background-image: url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAAAXNSR0IArs4c6QAAAERlWElmTU0AKgAAAAgAAYdpAAQAAAABAAAAGgAAAAAAA6ABAAMAAAABAAEAAKACAAQAAAABAAAAIKADAAQAAAABAAAAIAAAAACshmLzAAADr0lEQVRYCe1XTWgTQRj9ZjZV8a9SPIkKgj8I1bMHsUWrqYLVg4Ue6v9BwZOxSYsIerFao7UiUryIqJcqgtpimhbBXoSCVxUFe9CTiogUrUp2Pt+3aUI2u5vdNh4dmMzOzHvvezuz8xNFM0mjnbXaNu1MvFWRXkXEyE6aYOYJpdW4IXuA4r0fo8qqSMDBU0v1HJUgVieAXxzCsdE/YJTdFcVIZQNMyhruOMJKXYFoLfIfIvVIMWdsrd+Rpd86ZmyzzjJmLStqRn0v8lzkb4rVIXvnpScOJuAn2ACC65FkPzEdEy4TPWRLJ2h7z4cArXzzaOdKlbOvKKX25Wl00jSnrwVxAg3o4dRxhO13RBSdNvH0xSARv3adTXbBdTf64IWO2vH0LT+cv4GR1DJt+DUItaQogeBX/chhbTBxEiZ6gftlDNXTrvT7co4ub5A6gp9HIcHvzTa46OS5fBeP87Qm0fQkr4FsYgVQ7Qg+ZayaDg9jhg1GkWj8RG6lkeSacrrHgDaxdoBiZPg+NXV/KifMuB6//JmYH4CntVEHy/keA6x4h4CU5oFy8GzrBS18cLJMXcljAKB6INjWsRcuZBWVaS3GDrqB7rdapVIeA+isQ57Eev9eCqzqOa81CY05VLd6SamW2wA2H3SiTbnbSxmzfp7WtKZkqy4mdyAlGx7ennghYf8voqp9cLSgKdqNfa6RdRsAAkPwRuJZNbpByn+RrJi1RXTwdi8RQF6ymDwGMAtZ6TVE+4uoKh+MYkcLsT0Hk8eAienbiGdjJHZTpmNjlbFJNKDVAp2fJlYju6IreQxQ08UJDNYdoLSl6AadO+fFuCQqVMB1NJwPm69T04Wv5WhfcWyfXQB+wXRs1pt+nCknRa0LVzSA/2B+a9+zQJadb7IyyV24YAxKp2Jqs3emZTuNnKxsah+uabKbMk7CbTgJx/zIgQYErIeTKRQ9yD9wxVof5YolPHqaWo7TD6tJlh7jQnK5z2n3+fGdggIOx2kaa2YI9QWarc5Ce1ipNWMKeSG4DysFF52KBmTNMmn5HqCFkwy34rDg05gDwgH3bBi+sgFhN/e8QvRn8kbamCOhgrZ9GJhFDgfcMHzFb6BAtjKpFhzTjwv1KCVuxHvCbsSiEz4CANnj84cwHdFXAbAOJ4LTSAawGWFn5tDhLMYz6nWeU2wJfIhmIJBefcd/A5FWQWGgrWzyORZ3Q6HuV+Jf0Bj+BTX69fm1zWgK7By1YTXchFDORywnfQ7GpzOo6S+qECrsx2ifVQAAAABJRU5ErkJggg==');
   }
-  div.callout-tip.callout-style-default .callout-caption {
     background-color: #ccf1e3
   }
@@ -290,7 +292,7 @@
     background-image: url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAAAXNSR0IArs4c6QAAAERlWElmTU0AKgAAAAgAAYdpAAQAAAABAAAAGgAAAAAAA6ABAAMAAAABAAEAAKACAAQAAAABAAAAIKADAAQAAAABAAAAIAAAAACshmLzAAACV0lEQVRYCdVWzWoUQRCuqp2ICBLJXgITZL1EfQDBW/bkzUMUD7klD+ATSHBEfAIfQO+iXsWDxJsHL96EHAwhgzlkg8nBg25XWb0zIb0zs9muYYWkoKeru+vn664fBqElyZNuyh167NXJ8Ut8McjbmEraKHkd7uAnAFku+VWdb3reSmRV8PKSLfZ0Gjn3a6Xlcq9YGb6tADjn+lUfTXtVmaZ1KwBIvFI11rRXlWlatwIAAv2asaa9mlB9wwygiDX26qaw1yYPzFXg2N1GgG0FMF8Oj+VIx7E/03lHx8UhvYyNZLN7BwSPgekXXLribw7w5/c8EF+DBK5idvDVYtEEwMeYefjjLAdEyQ3M9nfOkgnPTEkYU+sxMq0BxNR6jExrAI31H1rzvLEfRIdgcv1XEdj6QTQAS2wtstEALLG1yEZ3QhH6oDX7ExBSFEkFINXH98NTrme5IOaaA7kIfiu2L8A3qhH9zRbukdCqdsA98TdElyeMe5BI8Rs2xHRIsoTSSVFfCFCWGPn9XHb4cdobRIWABNf0add9jakDjQJpJ1bTXOJXnnRXHRf+dNL1ZV1MBRCXhMbaHqGI1JkKIL7+i8uffuP6wVQAzO7+qVEbF6NbS0LJureYcWXUUhH66nLR5rYmva+2tjRFtojkM2aD76HEGAD3tPtKM309FJg5j/K682ywcWJ3PASCcycH/22u+Bh7Aa0ehM2Fu4z0SAE81HF9RkB21c5bEn4Dzw+/qNOyXr3DCTQDMBOdhi4nAgiFDGCinIa2owCEChUwD8qzd03PG+qdW/4fDzjUMcE1ZpIAAAAASUVORK5CYII=');
   }
-  div.callout-caution.callout-style-default .callout-caption {
     background-color: #ffe5d0
   }
@@ -382,23 +384,18 @@
       margin-right: 0;
     }
   </style>
-  <script src="Accelerate_files/libs/quarto-diagram/mermaid.min.js"></script>
-  <script src="Accelerate_files/libs/quarto-diagram/mermaid-init.js"></script>
-  <link href="Accelerate_files/libs/quarto-diagram/mermaid.css" rel="stylesheet">
 </head>
 <body class="quarto-dark">
   <div class="reveal">
     <div class="slides">
 <section id="title-slide" class="quarto-title-block center">
-  <h1 class="title">Accelerate, Three Powerful Sublibraries for PyTorch</h1>
 <div class="quarto-title-authors">
-<div class="quarto-title-author">
-<div class="quarto-title-author-name">
-Zachary Mueller
-</div>
-</div>
 </div>
 </section>
@@ -406,324 +403,554 @@ Zachary Mueller
 <h2>Who am I?</h2>
 <ul>
 <li>Zachary Mueller</li>
-<li>Deep Learning Software Engineer at 🤗</li>
 <li>API design geek</li>
 </ul>
 </section>
 <section id="what-is-accelerate" class="slide level2">
 <h2>What is 🤗 Accelerate?</h2>
 <div class="cell" data-reveal="true" data-fig-height="6">
 <div class="cell-output-display">
 <div>
-<p>
-</p><pre class="mermaid mermaid-js" data-tooltip-selector="#mermaid-tooltip-1">graph LR
-    A{"🤗 Accelerate#32;"}
-    A --&gt; B["Launching&lt;br&gt;Interface#32;"]
     A --&gt; C["Training Library#32;"]
     A --&gt; D["Big Model&lt;br&gt;Inference#32;"]
 </pre>
-<div id="mermaid-tooltip-1" class="mermaidTooltip">
 </div>
-<p></p>
 </div>
 </div>
 </div>
 </section>
-<section>
-<section id="a-launching-interface" class="title-slide slide level1 center">
-<h1>A Launching Interface</h1>
-<p>Can’t I just use <code>python do_the_thing.py</code>?</p>
 </section>
-<section id="a-launching-interface-1" class="slide level2">
-<h2>A Launching Interface</h2>
-<p>Launching scripts in different environments is complicated:</p>
 <ul>
-<li><div class="sourceCode" id="cb1"><pre class="sourceCode numberSource bash number-lines code-with-copy"><code class="sourceCode bash"><span id="cb1-1"><a href="#cb1-1"></a><span class="ex">python</span> script.py</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div></li>
-<li><div class="sourceCode" id="cb2"><pre class="sourceCode numberSource bash number-lines code-with-copy"><code class="sourceCode bash"><span id="cb2-1"><a href="#cb2-1"></a><span class="ex">torchrun</span> <span class="at">--nnodes</span><span class="op">=</span>1 <span class="at">--nproc_per_node</span><span class="op">=</span>2 script.py</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div></li>
-<li><div class="sourceCode" id="cb3"><pre class="sourceCode numberSource bash number-lines code-with-copy"><code class="sourceCode bash"><span id="cb3-1"><a href="#cb3-1"></a><span class="ex">deepspeed</span> <span class="at">--num_gpus</span><span class="op">=</span>2 script.py</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div></li>
 </ul>
-<p>And more!</p>
 </section>
-<section id="a-launching-interface-2" class="slide level2">
-<h2>A Launching Interface</h2>
-<p>But it doesn’t have to be:</p>
-<div class="sourceCode" id="cb4"><pre class="sourceCode numberSource bash number-lines code-with-copy"><code class="sourceCode bash"><span id="cb4-1"><a href="#cb4-1"></a><span class="ex">accelerate</span> launch script.py</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
-<p>A single command to launch with <code>DeepSpeed</code>, Fully Sharded Data Parallelism, across single and multi CPUs and GPUs, and to train on TPUs<sup>1</sup> too!</p>
-<aside><ol class="aside-footnotes"><li id="fn1"><p>Without needing to modify your code and create a <code>_mp_fn</code></p></li></ol></aside></section>
-<section id="a-launching-interface-3" class="slide level2">
-<h2>A Launching Interface</h2>
-<p>Generate a device-specific configuration through <code>accelerate config</code></p>
-<img data-src="CLI.gif" class="r-stretch"></section>
-<section id="a-launching-interface-4" class="slide level2">
-<h2>A Launching Interface</h2>
-<p>Or don’t. <code>accelerate config</code> doesn’t <em>have</em> to be done!</p>
-<div class="sourceCode" id="cb5"><pre class="sourceCode numberSource bash number-lines code-with-copy"><code class="sourceCode bash"><span id="cb5-1"><a href="#cb5-1"></a><span class="ex">torchrun</span> <span class="at">--nnodes</span><span class="op">=</span>1 <span class="at">--nproc_per_node</span><span class="op">=</span>2 script.py</span>
-<span id="cb5-2"><a href="#cb5-2"></a><span class="ex">accelerate</span> launch <span class="at">--multi_gpu</span> <span class="at">--nproc_per_node</span><span class="op">=</span>2 script.py</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
-<p>A quick default configuration can be made too:</p>
-<div class="sourceCode" id="cb6"><pre class="sourceCode numberSource bash number-lines code-with-copy"><code class="sourceCode bash"><span id="cb6-1"><a href="#cb6-1"></a><span class="ex">accelerate</span> config default</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
 </section>
-<section id="a-launching-interface-5" class="slide level2">
-<h2>A Launching Interface</h2>
-<p>With the <code>notebook_launcher</code> it’s also possible to launch code directly from your Jupyter environment too!</p>
-<div class="sourceCode" id="cb7"><pre class="sourceCode numberSource python number-lines code-with-copy"><code class="sourceCode python"><span id="cb7-1"><a href="#cb7-1"></a><span class="im">from</span> accelerate <span class="im">import</span> notebook_launcher</span>
-<span id="cb7-2"><a href="#cb7-2"></a>notebook_launcher(</span>
-<span id="cb7-3"><a href="#cb7-3"></a>    training_loop_function, </span>
-<span id="cb7-4"><a href="#cb7-4"></a>    args, </span>
-<span id="cb7-5"><a href="#cb7-5"></a>    num_processes<span class="op">=</span><span class="dv">2</span></span>
-<span id="cb7-6"><a href="#cb7-6"></a>)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
-<div class="sourceCode" id="cb8"><pre class="sourceCode numberSource python number-lines code-with-copy"><code class="sourceCode python"><span id="cb8-1"><a href="#cb8-1"></a>Launching training on <span class="dv">2</span> GPUs.</span>
-<span id="cb8-2"><a href="#cb8-2"></a>epoch <span class="dv">0</span>: <span class="fl">88.12</span></span>
-<span id="cb8-3"><a href="#cb8-3"></a>epoch <span class="dv">1</span>: <span class="fl">91.73</span></span>
-<span id="cb8-4"><a href="#cb8-4"></a>epoch <span class="dv">2</span>: <span class="fl">92.58</span></span>
-<span id="cb8-5"><a href="#cb8-5"></a>epoch <span class="dv">3</span>: <span class="fl">93.90</span></span>
-<span id="cb8-6"><a href="#cb8-6"></a>epoch <span class="dv">4</span>: <span class="fl">94.71</span></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
 </section></section>
 <section>
 <section id="a-training-library" class="title-slide slide level1 center">
 <h1>A Training Library</h1>
-<p>Okay, will <code>accelerate launch</code> make <code>do_the_thing.py</code> use all my GPUs magically?</p>
-</section>
-<section id="a-training-library-1" class="slide level2">
-<h2>A Training Library</h2>
-<ul>
-<li>Just showed that its possible using <code>accelerate launch</code> to <em>launch</em> a python script in various distributed environments</li>
-<li>This does <em>not</em> mean that the script will just “use” that code and still run on the new compute efficiently.</li>
-<li>Training on different computes often means <em>many</em> lines of code changed for each specific compute.</li>
-<li>🤗 <code>accelerate</code> solves this by ensuring the same code can be ran on a CPU or GPU, multiples, and on TPUs!</li>
-</ul>
-</section>
-<section id="a-training-library-2" class="slide level2">
-<h2>A Training Library</h2>
-<div class="sourceCode" id="cb9"><pre class="sourceCode numberSource python number-lines code-with-copy"><code class="sourceCode python"><span id="cb9-1"><a href="#cb9-1"></a><span class="cf">for</span> batch <span class="kw">in</span> dataloader:</span>
-<span id="cb9-2"><a href="#cb9-2"></a>    optimizer.zero_grad()</span>
-<span id="cb9-3"><a href="#cb9-3"></a>    inputs, targets <span class="op">=</span> batch</span>
-<span id="cb9-4"><a href="#cb9-4"></a>    inputs <span class="op">=</span> inputs.to(device)</span>
-<span id="cb9-5"><a href="#cb9-5"></a>    targets <span class="op">=</span> targets.to(device)</span>
-<span id="cb9-6"><a href="#cb9-6"></a>    outputs <span class="op">=</span> model(inputs)</span>
-<span id="cb9-7"><a href="#cb9-7"></a>    loss <span class="op">=</span> loss_function(outputs, targets)</span>
-<span id="cb9-8"><a href="#cb9-8"></a>    loss.backward()</span>
-<span id="cb9-9"><a href="#cb9-9"></a>    optimizer.step()</span>
-<span id="cb9-10"><a href="#cb9-10"></a>    scheduler.step()</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
 </section>
-<section id="a-training-library-3" class="slide level2 smaller">
-<h2>A Training Library</h2>
-<div class="columns">
-<div class="column" style="width:43%;">
 <p><br><br><br></p>
-<div class="sourceCode" id="cb10" data-code-line-numbers="5-6,9"><pre class="sourceCode numberSource python number-lines code-with-copy"><code class="sourceCode python"><span id="cb10-1"><a href="#cb10-1"></a><span class="co"># For alignment purposes</span></span>
-<span id="cb10-2"><a href="#cb10-2"></a><span class="cf">for</span> batch <span class="kw">in</span> dataloader:</span>
-<span id="cb10-3"><a href="#cb10-3"></a>    optimizer.zero_grad()</span>
-<span id="cb10-4"><a href="#cb10-4"></a>    inputs, targets <span class="op">=</span> batch</span>
-<span id="cb10-5"><a href="#cb10-5"></a>    inputs <span class="op">=</span> inputs.to(device)</span>
-<span id="cb10-6"><a href="#cb10-6"></a>    targets <span class="op">=</span> targets.to(device)</span>
-<span id="cb10-7"><a href="#cb10-7"></a>    outputs <span class="op">=</span> model(inputs)</span>
-<span id="cb10-8"><a href="#cb10-8"></a>    loss <span class="op">=</span> loss_function(outputs, targets)</span>
-<span id="cb10-9"><a href="#cb10-9"></a>    loss.backward()</span>
-<span id="cb10-10"><a href="#cb10-10"></a>    optimizer.step()</span>
-<span id="cb10-11"><a href="#cb10-11"></a>    scheduler.step()</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
-</div><div class="column" style="width:57%;">
-<div class="sourceCode" id="cb11" data-code-line-numbers="1-7,12-13,16"><pre class="sourceCode numberSource python number-lines code-with-copy"><code class="sourceCode python"><span id="cb11-1"><a href="#cb11-1"></a><span class="im">from</span> accelerate <span class="im">import</span> Accelerator</span>
-<span id="cb11-2"><a href="#cb11-2"></a>accelerator <span class="op">=</span> Accelerator()</span>
-<span id="cb11-3"><a href="#cb11-3"></a>dataloader, model, optimizer scheduler <span class="op">=</span> (</span>
-<span id="cb11-4"><a href="#cb11-4"></a>    accelerator.prepare(</span>
-<span id="cb11-5"><a href="#cb11-5"></a>        dataloader, model, optimizer, scheduler</span>
-<span id="cb11-6"><a href="#cb11-6"></a>    )</span>
-<span id="cb11-7"><a href="#cb11-7"></a>)</span>
-<span id="cb11-8"><a href="#cb11-8"></a></span>
-<span id="cb11-9"><a href="#cb11-9"></a><span class="cf">for</span> batch <span class="kw">in</span> dataloader:</span>
-<span id="cb11-10"><a href="#cb11-10"></a>    optimizer.zero_grad()</span>
-<span id="cb11-11"><a href="#cb11-11"></a>    inputs, targets <span class="op">=</span> batch</span>
-<span id="cb11-12"><a href="#cb11-12"></a>    <span class="co"># inputs = inputs.to(device)</span></span>
-<span id="cb11-13"><a href="#cb11-13"></a>    <span class="co"># targets = targets.to(device)</span></span>
-<span id="cb11-14"><a href="#cb11-14"></a>    outputs <span class="op">=</span> model(inputs)</span>
-<span id="cb11-15"><a href="#cb11-15"></a>    loss <span class="op">=</span> loss_function(outputs, targets)</span>
-<span id="cb11-16"><a href="#cb11-16"></a>    accelerator.backward(loss) <span class="co"># loss.backward()</span></span>
-<span id="cb11-17"><a href="#cb11-17"></a>    optimizer.step()</span>
-<span id="cb11-18"><a href="#cb11-18"></a>    scheduler.step()</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
 </div>
 </div>
 </section>
-<section id="a-training-library-4" class="slide level2">
-<h2>A Training Library</h2>
-<p>What all happened in <code>Accelerator.prepare</code>?</p>
-<div>
-<ol type="1">
-<li class="fragment"><code>Accelerator</code> looked at the configuration</li>
-<li class="fragment">The <code>dataloader</code> was converted into one that can dispatch each batch onto a seperate GPU</li>
-<li class="fragment">The <code>model</code> was wrapped with the appropriate DDP wrapper from either <code>torch.distributed</code> or <code>torch_xla</code></li>
-<li class="fragment">The <code>optimizer</code> and <code>scheduler</code> were both converted into an <code>AcceleratedOptimizer</code> and <code>AcceleratedScheduler</code> which knows how to handle any distributed scenario</li>
-</ol>
-</div>
 </section>
 <section id="a-training-library-mixed-precision" class="slide level2">
-<h2>A Training Library, Mixed Precision</h2>
-<p>🤗 <code>accelerate</code> also supports <em>automatic mixed precision</em>.</p>
-<p>Through a single flag to the <code>Accelerator</code> object when calling <code>accelerator.backward()</code> the mixed precision of your choosing (such as <code>bf16</code> or <code>fp16</code>) will be applied:</p>
-<div class="sourceCode" id="cb12" data-code-line-numbers="2,9"><pre class="sourceCode numberSource python number-lines code-with-copy"><code class="sourceCode python"><span id="cb12-1"><a href="#cb12-1"></a><span class="im">from</span> accelerate <span class="im">import</span> Accelerator</span>
-<span id="cb12-2"><a href="#cb12-2"></a>accelerator <span class="op">=</span> Accelerator(mixed_precision<span class="op">=</span><span class="st">"fp16"</span>)</span>
-<span id="cb12-3"><a href="#cb12-3"></a>...</span>
-<span id="cb12-4"><a href="#cb12-4"></a><span class="cf">for</span> batch <span class="kw">in</span> dataloader:</span>
-<span id="cb12-5"><a href="#cb12-5"></a>    optimizer.zero_grad()</span>
-<span id="cb12-6"><a href="#cb12-6"></a>    inputs, targets <span class="op">=</span> batch</span>
-<span id="cb12-7"><a href="#cb12-7"></a>    outputs <span class="op">=</span> model(inputs)</span>
-<span id="cb12-8"><a href="#cb12-8"></a>    loss <span class="op">=</span> loss_function(outputs, targets)</span>
-<span id="cb12-9"><a href="#cb12-9"></a>    accelerator.backward(loss)</span>
-<span id="cb12-10"><a href="#cb12-10"></a>    optimizer.step()</span>
-<span id="cb12-11"><a href="#cb12-11"></a>    scheduler.step()</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
-</section>
-<section id="a-training-library-gradient-accumulation" class="slide level2">
-<h2>A Training Library, Gradient Accumulation</h2>
-<p>Gradient accumulation in distributed setups often need extra care to ensure gradients are aligned when they need to be and the backward pass is computationally efficient.</p>
-<p>🤗 <code>accelerate</code> can just easily handle this for you:</p>
-<div class="sourceCode" id="cb13" data-code-line-numbers="2,5"><pre class="sourceCode numberSource python number-lines code-with-copy"><code class="sourceCode python"><span id="cb13-1"><a href="#cb13-1"></a><span class="im">from</span> accelerate <span class="im">import</span> Accelerator</span>
-<span id="cb13-2"><a href="#cb13-2"></a>accelerator <span class="op">=</span> Accelerator(gradient_accumulation_steps<span class="op">=</span><span class="dv">4</span>)</span>
-<span id="cb13-3"><a href="#cb13-3"></a>...</span>
-<span id="cb13-4"><a href="#cb13-4"></a><span class="cf">for</span> batch <span class="kw">in</span> dataloader:</span>
-<span id="cb13-5"><a href="#cb13-5"></a>    <span class="cf">with</span> accelerator.accumulate(model):</span>
-<span id="cb13-6"><a href="#cb13-6"></a>        optimizer.zero_grad()</span>
-<span id="cb13-7"><a href="#cb13-7"></a>        inputs, targets <span class="op">=</span> batch</span>
-<span id="cb13-8"><a href="#cb13-8"></a>        outputs <span class="op">=</span> model(inputs)</span>
-<span id="cb13-9"><a href="#cb13-9"></a>        loss <span class="op">=</span> loss_function(outputs, targets)</span>
-<span id="cb13-10"><a href="#cb13-10"></a>        accelerator.backward(loss)</span>
-<span id="cb13-11"><a href="#cb13-11"></a>        optimizer.step()</span>
-<span id="cb13-12"><a href="#cb13-12"></a>        scheduler.step()</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
-</section>
-<section id="a-training-library-gradient-accumulation-1" class="slide level2">
-<h2>A Training Library, Gradient Accumulation</h2>
-<div class="sourceCode" id="cb14" data-code-line-numbers="5-7,10,11,12,15"><pre class="sourceCode numberSource python number-lines code-with-copy"><code class="sourceCode python"><span id="cb14-1"><a href="#cb14-1"></a>ddp_model, dataloader <span class="op">=</span> accelerator.prepare(model, dataloader)</span>
-<span id="cb14-2"><a href="#cb14-2"></a></span>
-<span id="cb14-3"><a href="#cb14-3"></a><span class="cf">for</span> index, batch <span class="kw">in</span> <span class="bu">enumerate</span>(dataloader):</span>
-<span id="cb14-4"><a href="#cb14-4"></a>    inputs, targets <span class="op">=</span> batch</span>
-<span id="cb14-5"><a href="#cb14-5"></a>    <span class="cf">if</span> index <span class="op">!=</span> (<span class="bu">len</span>(dataloader)<span class="op">-</span><span class="dv">1</span>) <span class="kw">or</span> (index <span class="op">%</span> <span class="dv">4</span>) <span class="op">!=</span> <span class="dv">0</span>:</span>
-<span id="cb14-6"><a href="#cb14-6"></a>        <span class="co"># Gradients don't sync</span></span>
-<span id="cb14-7"><a href="#cb14-7"></a>        <span class="cf">with</span> accelerator.no_sync(model):</span>
-<span id="cb14-8"><a href="#cb14-8"></a>            outputs <span class="op">=</span> ddp_model(inputs)</span>
-<span id="cb14-9"><a href="#cb14-9"></a>            loss <span class="op">=</span> loss_func(outputs, targets)</span>
-<span id="cb14-10"><a href="#cb14-10"></a>            accelerator.backward(loss)</span>
-<span id="cb14-11"><a href="#cb14-11"></a>    <span class="cf">else</span>:</span>
-<span id="cb14-12"><a href="#cb14-12"></a>        <span class="co"># Gradients finally sync</span></span>
-<span id="cb14-13"><a href="#cb14-13"></a>        outputs <span class="op">=</span> ddp_model(inputs)</span>
-<span id="cb14-14"><a href="#cb14-14"></a>        loss <span class="op">=</span> loss_func(outputs)</span>
-<span id="cb14-15"><a href="#cb14-15"></a>        accelerator.backward(loss)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
-</section></section>
-<section>
-<section id="big-model-inference" class="title-slide slide level1 center">
-<h1>Big Model Inference</h1>
-<p>Stable Diffusion taking the world by storm</p>
-</section>
-<section id="bigger-models-higher-compute" class="slide level2">
-<h2>Bigger Models == Higher Compute</h2>
-<p>As more large models were being released, Hugging Face quickly realized there must be a way to continue our decentralization of Machine Learning and have the day-to-day programmer be able to leverage these big models.</p>
-<p>Born out of this effort by Sylvain Gugger:</p>
-<p>🤗 Accelerate: Big Model Inference.</p>
 </section>
-<section id="the-basic-premise" class="slide level2">
-<h2>The Basic Premise</h2>
-<div>
 <ul>
-<li class="fragment"><p>In PyTorch, there exists the <code>meta</code> device.</p></li>
-<li class="fragment"><p>Super small footprint to load in huge models quickly by not loading in their weights immediatly.</p></li>
-<li class="fragment"><p>As an input gets passed through each layer, we can load and unload <em>parts</em> of the PyTorch model quickly so that only a small portion of the big model is loaded in at a single time.</p></li>
-<li class="fragment"><p>The end result? Stable Diffusion v1 can be ran on &lt; 800mb of vRAM</p></li>
 </ul>
 </div>
 </section>
-<section id="the-code" class="slide level2">
-<h2>The Code</h2>
-<p>Generally you start with something like so:</p>
-<div class="sourceCode" id="cb15"><pre class="sourceCode numberSource python number-lines code-with-copy"><code class="sourceCode python"><span id="cb15-1"><a href="#cb15-1"></a><span class="im">import</span> torch</span>
-<span id="cb15-2"><a href="#cb15-2"></a></span>
-<span id="cb15-3"><a href="#cb15-3"></a>my_model <span class="op">=</span> ModelClass(...)</span>
-<span id="cb15-4"><a href="#cb15-4"></a>state_dict <span class="op">=</span> torch.load(checkpoint_file)</span>
-<span id="cb15-5"><a href="#cb15-5"></a>my_model.load_state_dict(state_dict)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
-<p>But this has issues:</p>
-<ol type="1">
-<li>The full version of the model is loaded at <code>3</code></li>
-<li>Another version of the model is loaded into memory at <code>4</code></li>
-</ol>
-<p>If a 6 <em>billion</em> parameter model is being loaded, each model class has a dictionary of 24GB so 48GB of vRAM is needed</p>
-</section>
-<section id="empty-model-weights" class="slide level2">
-<h2>Empty Model Weights</h2>
-<p>We can fix step 1 by loading in an empty model skeleton at first:</p>
-<div class="sourceCode" id="cb16" data-code-line-numbers="1,3-4"><pre class="sourceCode numberSource python number-lines code-with-copy"><code class="sourceCode python"><span id="cb16-1"><a href="#cb16-1"></a><span class="im">from</span> accelerate <span class="im">import</span> init_empty_weights</span>
-<span id="cb16-2"><a href="#cb16-2"></a></span>
-<span id="cb16-3"><a href="#cb16-3"></a><span class="cf">with</span> init_empty_weights():</span>
-<span id="cb16-4"><a href="#cb16-4"></a>    my_model <span class="op">=</span> ModelClass(...)</span>
-<span id="cb16-5"><a href="#cb16-5"></a>state_dict <span class="op">=</span> torch.load(checkpoint_file)</span>
-<span id="cb16-6"><a href="#cb16-6"></a>my_model.load_state_dict(state_dict)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
-<div class="callout callout-important callout-captioned callout-style-default">
-<div class="callout-body">
-<div class="callout-caption">
-<div class="callout-icon-container">
-<i class="callout-icon"></i>
-</div>
-<p><strong>This code will not run</strong></p>
-</div>
-<div class="callout-content">
-<p>It is likely that just calling <code>my_model(x)</code> will fail as not all tensor operations are supported on the <code>meta</code> device.</p>
-</div>
-</div>
 </div>
 </section>
-<section id="sharded-checkpoints---the-concept" class="slide level2">
-<h2>Sharded Checkpoints - The Concept</h2>
-<p>The next step is to have “Sharded Checkpoints” saved for your model.</p>
-<p>Basically smaller chunks of your model weights stored that can be brought in at any particular time.</p>
-<p>This reduces the amount of memory step 2 takes in since we can just load in a “chunk” of the model at a time, then swap it out for a new chunk through PyTorch hooks</p>
-</section>
-<section id="sharded-checkpoints---the-code" class="slide level2">
-<h2>Sharded Checkpoints - The Code</h2>
-<div class="sourceCode" id="cb17" data-code-line-numbers="1,6-8"><pre class="sourceCode numberSource python number-lines code-with-copy"><code class="sourceCode python"><span id="cb17-1"><a href="#cb17-1"></a><span class="im">from</span> accelerate <span class="im">import</span> init_empty_weights, load_checkpoint_and_dispatch</span>
-<span id="cb17-2"><a href="#cb17-2"></a></span>
-<span id="cb17-3"><a href="#cb17-3"></a><span class="cf">with</span> init_empty_weights():</span>
-<span id="cb17-4"><a href="#cb17-4"></a>    my_model <span class="op">=</span> ModelClass(...)</span>
-<span id="cb17-5"><a href="#cb17-5"></a></span>
-<span id="cb17-6"><a href="#cb17-6"></a>my_model <span class="op">=</span> load_checkpoint_and_dispatch(</span>
-<span id="cb17-7"><a href="#cb17-7"></a>    my_model, <span class="st">"sharded-weights"</span>, device_map<span class="op">=</span><span class="st">"auto"</span></span>
-<span id="cb17-8"><a href="#cb17-8"></a>)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
-<p><code>device_map="auto"</code> will tell 🤗 Accelerate that it should determine where to put each layer of the model:</p>
-<ol type="1">
-<li>Maximum space on the GPU(s)</li>
-<li>Maximum space on the CPU(s)</li>
-<li>Utilize disk space through memory-mapped tensors</li>
-</ol>
-</section>
-<section id="big-model-inference-put-together" class="slide level2">
-<h2>Big Model Inference Put Together</h2>
-<div class="sourceCode" id="cb18"><pre class="sourceCode numberSource python number-lines code-with-copy"><code class="sourceCode python"><span id="cb18-1"><a href="#cb18-1"></a><span class="im">from</span> accelerate <span class="im">import</span> init_empty_weights, load_checkpoint_and_dispatch</span>
-<span id="cb18-2"><a href="#cb18-2"></a></span>
-<span id="cb18-3"><a href="#cb18-3"></a><span class="cf">with</span> init_empty_weights():</span>
-<span id="cb18-4"><a href="#cb18-4"></a>    my_model <span class="op">=</span> ModelClass(...)</span>
-<span id="cb18-5"><a href="#cb18-5"></a></span>
-<span id="cb18-6"><a href="#cb18-6"></a>my_model <span class="op">=</span> load_checkpoint_and_dispatch(</span>
-<span id="cb18-7"><a href="#cb18-7"></a>    my_model, <span class="st">"sharded-weights"</span>, device_map<span class="op">=</span><span class="st">"auto"</span></span>
-<span id="cb18-8"><a href="#cb18-8"></a>)</span>
-<span id="cb18-9"><a href="#cb18-9"></a>my_model.<span class="bu">eval</span>()</span>
-<span id="cb18-10"><a href="#cb18-10"></a></span>
-<span id="cb18-11"><a href="#cb18-11"></a><span class="cf">for</span> batch <span class="kw">in</span> dataloader:</span>
-<span id="cb18-12"><a href="#cb18-12"></a>    output <span class="op">=</span> my_model(batch)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
-</section>
-<section id="is-there-an-easier-way" class="slide level2">
-<h2>Is there an easier way?</h2>
-<p>The <code>transformers</code> library combined with the Hub makes all this code wrapping much easier for you with the <code>pipeline</code></p>
-<div class="sourceCode" id="cb19"><pre class="sourceCode numberSource python number-lines code-with-copy"><code class="sourceCode python"><span id="cb19-1"><a href="#cb19-1"></a><span class="im">import</span> torch</span>
-<span id="cb19-2"><a href="#cb19-2"></a><span class="im">from</span> transformers <span class="im">import</span> pipeline</span>
-<span id="cb19-3"><a href="#cb19-3"></a>pipe <span class="op">=</span> pipeline(</span>
-<span id="cb19-4"><a href="#cb19-4"></a>    task<span class="op">=</span><span class="st">"text-generation"</span>,</span>
-<span id="cb19-5"><a href="#cb19-5"></a>    model<span class="op">=</span><span class="st">"EleutherAI/gpt-j-6B"</span>,</span>
-<span id="cb19-6"><a href="#cb19-6"></a>    device_map<span class="op">=</span><span class="st">"auto"</span>,</span>
-<span id="cb19-7"><a href="#cb19-7"></a>    torch_dtype<span class="op">=</span>torch.float16</span>
-<span id="cb19-8"><a href="#cb19-8"></a>)</span>
-<span id="cb19-9"><a href="#cb19-9"></a></span>
-<span id="cb19-10"><a href="#cb19-10"></a>text <span class="op">=</span> pipe(<span class="st">"This is some generated text, I think"</span>)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
-</section></section>
-<section>
-<section id="what-about-stable-diffusion" class="title-slide slide level1 center">
-<h1>What about Stable Diffusion?</h1>
-<p>A demo with <code>diffusers</code> &amp; Weights and Biases</p>
 </section>
 <section id="some-handy-resources" class="slide level2">
 <h2>Some Handy Resources</h2>
@@ -735,29 +962,29 @@ Zachary Mueller
 <li><a href="https://huggingface.co/docs/accelerate/usage_guides/big_modeling">Big Model Inference tutorial</a></li>
 <li><a href="https://huggingface.co/docs/accelerate/usage_guides/deepspeed">DeepSpeed and 🤗 Accelerate</a></li>
 <li><a href="https://huggingface.co/docs/accelerate/usage_guides/fsdp">Fully Sharded Data Parallelism and 🤗 Accelerate</a></li>
 </ul>
 <div class="footer footer-default">
 </div>
 </section></section>
     </div>
   </div>
   <script>window.backupDefine = window.define; window.define = undefined;</script>
-  <script src="Accelerate_files/libs/revealjs/dist/reveal.js"></script>
   <!-- reveal.js plugins -->
-  <script src="Accelerate_files/libs/revealjs/plugin/quarto-line-highlight/line-highlight.js"></script>
-  <script src="Accelerate_files/libs/revealjs/plugin/pdf-export/pdfexport.js"></script>
-  <script src="Accelerate_files/libs/revealjs/plugin/reveal-menu/menu.js"></script>
-  <script src="Accelerate_files/libs/revealjs/plugin/reveal-menu/quarto-menu.js"></script>
-  <script src="Accelerate_files/libs/revealjs/plugin/quarto-support/support.js"></script>
-  <script src="Accelerate_files/libs/revealjs/plugin/notes/notes.js"></script>
-  <script src="Accelerate_files/libs/revealjs/plugin/search/search.js"></script>
-  <script src="Accelerate_files/libs/revealjs/plugin/zoom/zoom.js"></script>
-  <script src="Accelerate_files/libs/revealjs/plugin/math/math.js"></script>
   <script>window.define = window.backupDefine; window.backupDefine = undefined;</script>
   <script>
@@ -767,12 +994,11 @@ Zachary Mueller
       Reveal.initialize({
 'controlsAuto': true,
 'previewLinksAuto': false,
-'smaller': false,
 'pdfSeparateFragments': false,
 'autoAnimateEasing': "ease",
 'autoAnimateDuration': 1,
 'autoAnimateUnmatched': true,
-'menu': {"side":"left","useTextContentForMissingTitles":true,"markers":false,"loadIcons":false,"custom":[{"title":"Tools","icon":"<i class=\"fas fa-gear\"></i>","content":"<ul class=\"slide-menu-items\">\n<li class=\"slide-tool-item active\" data-item=\"0\"><a href=\"#\" onclick=\"RevealMenuToolHandlers.fullscreen(event)\"><kbd>f</kbd> Fullscreen</a></li>\n<li class=\"slide-tool-item\" data-item=\"1\"><a href=\"#\" onclick=\"RevealMenuToolHandlers.speakerMode(event)\"><kbd>s</kbd> Speaker View</a></li>\n<li class=\"slide-tool-item\" data-item=\"2\"><a href=\"#\" onclick=\"RevealMenuToolHandlers.overview(event)\"><kbd>o</kbd> Slide Overview</a></li>\n<li class=\"slide-tool-item\" data-item=\"3\"><a href=\"#\" onclick=\"RevealMenuToolHandlers.overview(event)\"><kbd>e</kbd> PDF Export Mode</a></li>\n<li class=\"slide-tool-item\" data-item=\"4\"><a href=\"#\" onclick=\"RevealMenuToolHandlers.keyboardHelp(event)\"><kbd>?</kbd> Keyboard Help</a></li>\n</ul>"}],"openButton":true},
 'smaller': false,
         // Display controls in the bottom right corner
@@ -976,9 +1202,23 @@ Zachary Mueller
       tabsets.forEach(function(tabset) {
         const tabby = new Tabby('#' + tabset.id);
       });
       const clipboard = new window.ClipboardJS('.code-copy-button', {
-        target: function(trigger) {
-          return trigger.previousElementSibling;
         }
       });
       clipboard.on('success', function(e) {
@@ -1014,10 +1254,9 @@ Zachary Mueller
         // clear code selection
         e.clearSelection();
       });
-      function tippyHover(el, contentFn) {
         const config = {
           allowHTML: true,
-          content: contentFn,
           maxWidth: 500,
           delay: 100,
           arrow: false,
@@ -1026,9 +1265,18 @@ Zachary Mueller
           },
           interactive: true,
           interactiveBorder: 10,
-          theme: 'quarto-reveal',
-          placement: 'bottom-start'
         };
           config['offset'] = [0,0];
           config['maxWidth'] = 700;
         window.tippy(el, config);

 <!DOCTYPE html>
 <html lang="en"><head>
+<script src="llm_conf_files/libs/clipboard/clipboard.min.js"></script>
+<script src="llm_conf_files/libs/quarto-html/tabby.min.js"></script>
+<script src="llm_conf_files/libs/quarto-html/popper.min.js"></script>
+<script src="llm_conf_files/libs/quarto-html/tippy.umd.min.js"></script>
+<link href="llm_conf_files/libs/quarto-html/tippy.css" rel="stylesheet">
+<link href="llm_conf_files/libs/quarto-html/light-border.css" rel="stylesheet">
+<link href="llm_conf_files/libs/quarto-html/quarto-html.min.css" rel="stylesheet" data-mode="light">
+<link href="llm_conf_files/libs/quarto-html/quarto-syntax-highlighting-dark.css" rel="stylesheet" id="quarto-text-highlighting-styles"><meta charset="utf-8">
+  <meta name="generator" content="quarto-99.9.9">
+  <title>Scaling Model Training with More Compute, How Do They Do It?</title>
   <meta name="apple-mobile-web-app-capable" content="yes">
   <meta name="apple-mobile-web-app-status-bar-style" content="black-translucent">
   <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no, minimal-ui">
+  <link rel="stylesheet" href="llm_conf_files/libs/revealjs/dist/reset.css">
+  <link rel="stylesheet" href="llm_conf_files/libs/revealjs/dist/reveal.css">
   <style>
     code{white-space: pre-wrap;}
     span.smallcaps{font-variant: small-caps;}
     ul.task-list{list-style: none;}
     ul.task-list li input[type="checkbox"] {
       width: 0.8em;
+      margin: 0 0.8em 0.2em -1em; /* quarto-specific, see https://github.com/quarto-dev/quarto-cli/issues/4556 */
       vertical-align: middle;
     }
+    /* CSS for syntax highlighting */
     pre > code.sourceCode { white-space: pre; position: relative; }
+    pre > code.sourceCode > span { line-height: 1.25; }
     pre > code.sourceCode > span:empty { height: 1.2em; }
     .sourceCode { overflow: visible; }
     code.sourceCode > span { color: inherit; text-decoration: inherit; }
     code span.vs { color: #abe338; } /* VerbatimString */
     code span.wa { color: #dcc6e0; } /* Warning */
   </style>
+  <link rel="stylesheet" href="llm_conf_files/libs/revealjs/dist/theme/quarto.css">
+  <link href="llm_conf_files/libs/revealjs/plugin/quarto-line-highlight/line-highlight.css" rel="stylesheet">
+  <link href="llm_conf_files/libs/revealjs/plugin/reveal-menu/menu.css" rel="stylesheet">
+  <link href="llm_conf_files/libs/revealjs/plugin/reveal-menu/quarto-menu.css" rel="stylesheet">
+  <link href="llm_conf_files/libs/revealjs/plugin/quarto-support/footer.css" rel="stylesheet">
   <style type="text/css">
   .callout {
     font-weight: 400;
   }
+  .callout.callout-titled.callout-style-simple .callout-body {
     margin-top: 0.2em;
   }
+  .callout:not(.callout-titled) .callout-body {
       display: flex;
   }
+  .callout:not(.no-icon).callout-titled.callout-style-simple .callout-content {
     padding-left: 1.6em;
   }
+  .callout.callout-titled .callout-header {
     padding-top: 0.2em;
     margin-bottom: -0.2em;
   }
+  .callout.callout-titled .callout-title  p {
     margin-top: 0.5em;
     margin-bottom: 0.5em;
   }
+  .callout.callout-titled.callout-style-simple .callout-content  p {
     margin-top: 0;
   }
+  .callout.callout-titled.callout-style-default .callout-content  p {
     margin-top: 0.7em;
   }
+  .callout.callout-style-simple div.callout-title {
     border-bottom: none;
     font-size: .9rem;
     font-weight: 600;
     opacity: 75%;
   }
+  .callout.callout-style-default  div.callout-title {
     border-bottom: none;
     font-weight: 600;
     opacity: 85%;
     background-size: 0.9rem 0.9rem;
   }
+  .callout-title {
     display: flex
   }
     display: none !important;
   }
+  .callout.callout-titled .callout-body > .callout-content > :last-child {
+    padding-bottom: 0.5rem;
+    margin-bottom: 0;
   }
+  .callout.callout-titled .callout-icon::before {
     margin-top: .5rem;
     padding-right: .5rem;
   }
+  .callout:not(.callout-titled) .callout-icon::before {
     margin-top: 1rem;
     padding-right: .5rem;
   }
     background-image: url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAAAXNSR0IArs4c6QAAAERlWElmTU0AKgAAAAgAAYdpAAQAAAABAAAAGgAAAAAAA6ABAAMAAAABAAEAAKACAAQAAAABAAAAIKADAAQAAAABAAAAIAAAAACshmLzAAAEU0lEQVRYCcVXTWhcVRQ+586kSUMMxkyaElstCto2SIhitS5Ek8xUKV2poatCcVHtUlFQk8mbaaziwpWgglJwVaquitBOfhQXFlqlzSJpFSpIYyXNjBNiTCck7x2/8/LeNDOZxDuEkgOXe++553zfefee+/OYLOXFk3+1LLrRdiO81yNqZ6K9cG0P3MeFaMIQjXssE8Z1JzLO9ls20MBZX7oG8w9GxB0goaPrW5aNMp1yOZIa7Wv6o2ykpLtmAPs/vrG14Z+6d4jpbSKuhdcSyq9wGMPXjonwmESXrriLzFGOdDBLB8Y6MNYBu0dRokSygMA/mrun8MGFN3behm6VVAwg4WR3i6FvYK1T7MHo9BK7ydH+1uurECoouk5MPRyVSBrBHMYwVobG2aOXM07sWrn5qgB60rc6mcwIDJtQrnrEr44kmy+UO9r0u9O5/YbkS9juQckLed3DyW2XV/qWBBB3ptvI8EUY3I9p/67OW+g967TNr3Sotn3IuVlfMLVnsBwH4fsnebJvyGm5GeIUA3jljERmrv49SizPYuq+z7c2H/jlGC+Ghhupn/hcapqmcudB9jwJ/3jvnvu6vu5lVzF1fXyZuZZ7U8nRmVzytvT+H3kilYvH09mLWrQdwFSsFEsxFVs5fK7A0g8gMZjbif4ACpKbjv7gNGaD8bUrlk8x+KRflttr22JEMRUbTUwwDQScyzPgedQHZT0xnx7ujw2jfVfExwYHwOsDTjLdJ2ebmeQIlJ7neo41s/DrsL3kl+W2lWvAga0tR3zueGr6GL78M3ifH0rGXrBC2aAR8uYcIA5gwV8zIE8onoh8u0Fca/ciF7j1uOzEnqcIm59sEXoGc0+z6+H45V1CvAvHcD7THztu669cnp+L0okAeIc6zjbM/24LgGM1gZk7jnRu1aQWoU9sfUOuhrmtaPIO3YY1KLLWZaEO5TKUbMY5zx8W9UJ6elpLwKXbsaZ4EFl7B4bMtDv0iRipKoDQT2sNQI9b1utXFdYisi+wzZ/ri/1m7QfDgEuvgUUEIJPq3DhX/5DWNqIXDOweC2wvIR90Oq3lDpdMIgD2r0dXvGdsEW5H6x6HLRJYU7C69VefO1x8Gde1ZFSJLfWS1jbCnhtOPxmpfv2LXOA2Xk2tvnwKKPFuZ/oRmwBwqRQDcKNeVQkYcOjtWVBuM/JuYw5b6isojIkYxyYAFn5K7ZBF10fea52y8QltAg6jnMqNHFBmGkQ1j+U43HMi2xMar1Nv0zGsf1s8nUsmUtPOOrbFIR8bHFDMB5zL13Gmr/kGlCkUzedTzzmzsaJXhYawnA3UmARpiYj5ooJZiUoxFRtK3X6pgNPv+IZVPcnwbOl6f+aBaO1CNvPW9n9LmCp01nuSaTRF2YxHqZ8DYQT6WsXT+RD6eUztwYLZ8rM+rcPxamv1VQzFUkzFXvkiVrySGQgJNvXHJAxiU3/NwiC03rSf05VBaPtu/Z7/B8Yn/w7eguloAAAAAElFTkSuQmCC');
   }
+  div.callout-note.callout-style-default .callout-title {
     background-color: #dae6fb
   }
     background-image: url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAAAXNSR0IArs4c6QAAAERlWElmTU0AKgAAAAgAAYdpAAQAAAABAAAAGgAAAAAAA6ABAAMAAAABAAEAAKACAAQAAAABAAAAIKADAAQAAAABAAAAIAAAAACshmLzAAAEKklEQVRYCcVXTWhcVRS+575MJym48A+hSRFr00ySRQhURRfd2HYjk2SSTokuBCkU2o0LoSKKraKIBTcuFCoidGFD08nkBzdREbpQ1EDNIv8qSGMFUboImMSZd4/f9zJv8ibJMC8xJQfO3HPPPef7zrvvvnvviIkpC9nsw0UttFunbUhpFzFtarSd6WJkStVMw5xyVqYTvkwfzuf/5FgtkVoB0729j1rjXwThS7Vio+Mo6DNnvLfahoZ+i/o32lULuJ3NNiz7q6+pyAUkJaFF6JwaM2lUJlV0MlnQn5aTRbEu0SEqHUa0A4AdiGuB1kFXRfVyg5d87+Dg4DL6m2TLAub60ilj7A1Ec4odSAc8X95sHh7+ZRPCFo6Fnp7HfU/fBng/hi10CjCnWnJjsxvDNxWw0NfV6Rv5GgP3I3jGWXumdTD/3cbEOP2ZbOZp69yniG3FQ9z1jD7bnBu9Fc2tKGC2q+uAJOQHBDRiZX1x36o7fWBs7J9ownbtO+n0/qWkvW7UPIfc37WgT6ZGR++EOJyeQDSb9UB+DZ1G6DdLDzyS+b/kBCYGsYgJbSQHuThGKRcw5xdeQf8YdNHsc6ePXrlSYMBuSIAFTGAtQo+VuALo4BX83N190NWZWbynBjhOHsmNfFWLeL6v+ynsA58zDvvAC8j5PkbOcXCMg2PZFk3q8MjI7WAG/Dp9AwP7jdGBOOQkAvlFUB+irtm16I1Zw9YBcpGTGXYmk3kQIC/Cds55l+iMI3jqhjAuaoe+am2Jw5GT3Nbz3CkE12NavmzN5+erJW7046n/CH1RO/RVa8lBLozXk9uqykkGAyRXLWlLv5jyp4RFsG5vGVzpDLnIjTWgnRy2Rr+tDKvRc7Y8AyZq10jj8DqXdnIRNtFZb+t/ZRtXcDiVnzpqx8mPcDWxgARUqx0W1QB9MeUZiNrV4qP+Ehc+BpNgATsTX8ozYKL2NtFYAHc84fG7ndxUPr+AR/iQSns7uSUufAymwDOb2+NjK27lEFocm/EE2WpyIy/Hi66MWuMKJn8RvxIcj87IM5Vh9663ziW36kR0HNenXuxmfaD8JC7tfKbrhFr7LiZCrMjrzTeGx+PmkosrkNzW94ObzwocJ7A1HokLolY+AvkTiD/q1H0cN48c5EL8Crkttsa/AXQVDmutfyku0E7jShx49XqV3MFK8IryDhYVbj7Sj2P2eBxwcXoe8T8idsKKPRcnZw1b+slFTubwUwhktrfnAt7J++jwQtLZcm3sr9LQrjRzz6cfMv9aLvgmnAGvpoaGLxM4mAEaLV7iAzQ3oU0IvD5x9ix3yF2RAAuYAOO2f7PEFWCXZ4C9Pb2UsgDeVnFSpbFK7/IWu7TPTvBqzbGdCHOJQSxiEjt6IyZmxQyEJHv6xyQsYk//moVFsN2zP6fRImjfq7/n/wFDguUQFNEwugAAAABJRU5ErkJggg==');
   }
+  div.callout-important.callout-style-default .callout-title {
     background-color: #f7dddc
   }
     background-image: url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAAAXNSR0IArs4c6QAAAERlWElmTU0AKgAAAAgAAYdpAAQAAAABAAAAGgAAAAAAA6ABAAMAAAABAAEAAKACAAQAAAABAAAAIKADAAQAAAABAAAAIAAAAACshmLzAAAETklEQVRYCeVWW2gcVRg+58yaTUnizqbipZeX4uWhBEniBaoUX1Ioze52t7sRq6APio9V9MEaoWlVsFasRq0gltaAPuxms8lu0gcviE/FFOstVbSIxgcv6SU7EZqmdc7v9+9mJtNks51NTUH84ed889/PP+cmxP+d5FIbMJmNbpREu4WUkiTtCicKny0l1pIKmBzovF2S+hIJHX8iEu3hZJ5lNZGqyRrGSIQpq15AzF28jgpeY6yk6GVdrfFqdrD6Iw+QlB8g0YS2g7dyQmXM/IDhBhT0UCiRf59lfqmmDvzRt6kByV/m4JjtzuaujMUM2c5Z2d6JdKrRb3K2q6mA+oYVz8JnDdKPmmNthzkAk/lN63sYPgevrguc72aZX/L9C6x09GYyxBgCX4NlvyGUHOKELlm5rXeR1kchuChJt4SSwyddZRXgvwMGvYo4QSlk3/zkHD8UHxwVJA6zjZZqP8v8kK8OWLnIZtLyCAJagYC4rTGW/9Pqj92N/c+LUaAj27movwbi19tk/whRCIE7Q9vyI6yvRpftAKVTdUjOW40X3h5OXsKCdmFcx0xlLJoSuQngnrJe7Kcjm4OMq9FlC7CMmScQANuNvjfP3PjGXDBaUQmbp296S5L4DrpbrHN1T87ZVEZVCzg1FF0Ft+dKrlLukI+/c9ENo+TvlTDbYFvuKPtQ9+l052rXrgKoWkDAFnvh0wTOmYn8R5f4k/jN/fZiCM1tQx9jQQ4ANhqG4hiL0qIFTGViG9DKB7GYzgubnpofgYRwO+DFjh0Zin2m4b/97EDkXkc+f6xYAPX0KK2I/7fUQuwzuwo/L3AkcjugPNixC8cHf0FyPjWlItmLxWw4Ou9YsQCr5fijMGoD/zpdRy95HRysyXA74MWOnscpO4j2y3HAVisw85hX5+AFBRSHt4ShfLFkIMXTqyKFc46xdzQM6XbAi702a7sy04J0+feReMFKp5q9esYLCqAZYw/k14E/xcLLsFElaornTuJB0svMuJINy8xkIYuL+xPAlWRceH6+HX7THJ0djLUom46zREu7tTkxwmf/FdOZ/sh6Q8qvEAiHpm4PJ4a/doJe0gH1t+aHRgCzOvBvJedEK5OFE5jpm4AGP2a8Dxe3gGJ/pAutug9Gp6he92CsSsWBaEcxGx0FHytmIpuqGkOpldqNYQK8cSoXvd+xLxXADw0kf6UkJNFtdo5MOgaLjiQOQHcn+A6h5NuL2s0qsC2LOM75PcF3yr5STuBSAcGG+meA14K/CI21HcS4LBT6tv0QAh8Dr5l93AhZzG5ZJ4VxAqdZUEl9z7WJ4aN+svMvwHHL21UKTd1mqvChH7/Za5xzXBBKrUcB0TQ+Ulgkfbi/H/YT5EptrGzsEK7tR1B7ln9BBwckYfMiuSqklSznIuoIIOM42MQO+QnduCoFCI0bpkzjCjddHPN/F+2Yu+sd9bKNpVwHhbS3LluK/0zgfwD0xYI5dXuzlQAAAABJRU5ErkJggg==');
   }
+  div.callout-warning.callout-style-default .callout-title {
     background-color: #fcefdc
   }
     background-image: url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAAAXNSR0IArs4c6QAAAERlWElmTU0AKgAAAAgAAYdpAAQAAAABAAAAGgAAAAAAA6ABAAMAAAABAAEAAKACAAQAAAABAAAAIKADAAQAAAABAAAAIAAAAACshmLzAAADr0lEQVRYCe1XTWgTQRj9ZjZV8a9SPIkKgj8I1bMHsUWrqYLVg4Ue6v9BwZOxSYsIerFao7UiUryIqJcqgtpimhbBXoSCVxUFe9CTiogUrUp2Pt+3aUI2u5vdNh4dmMzOzHvvezuz8xNFM0mjnbXaNu1MvFWRXkXEyE6aYOYJpdW4IXuA4r0fo8qqSMDBU0v1HJUgVieAXxzCsdE/YJTdFcVIZQNMyhruOMJKXYFoLfIfIvVIMWdsrd+Rpd86ZmyzzjJmLStqRn0v8lzkb4rVIXvnpScOJuAn2ACC65FkPzEdEy4TPWRLJ2h7z4cArXzzaOdKlbOvKKX25Wl00jSnrwVxAg3o4dRxhO13RBSdNvH0xSARv3adTXbBdTf64IWO2vH0LT+cv4GR1DJt+DUItaQogeBX/chhbTBxEiZ6gftlDNXTrvT7co4ub5A6gp9HIcHvzTa46OS5fBeP87Qm0fQkr4FsYgVQ7Qg+ZayaDg9jhg1GkWj8RG6lkeSacrrHgDaxdoBiZPg+NXV/KifMuB6//JmYH4CntVEHy/keA6x4h4CU5oFy8GzrBS18cLJMXcljAKB6INjWsRcuZBWVaS3GDrqB7rdapVIeA+isQ57Eev9eCqzqOa81CY05VLd6SamW2wA2H3SiTbnbSxmzfp7WtKZkqy4mdyAlGx7ennghYf8voqp9cLSgKdqNfa6RdRsAAkPwRuJZNbpByn+RrJi1RXTwdi8RQF6ymDwGMAtZ6TVE+4uoKh+MYkcLsT0Hk8eAienbiGdjJHZTpmNjlbFJNKDVAp2fJlYju6IreQxQ08UJDNYdoLSl6AadO+fFuCQqVMB1NJwPm69T04Wv5WhfcWyfXQB+wXRs1pt+nCknRa0LVzSA/2B+a9+zQJadb7IyyV24YAxKp2Jqs3emZTuNnKxsah+uabKbMk7CbTgJx/zIgQYErIeTKRQ9yD9wxVof5YolPHqaWo7TD6tJlh7jQnK5z2n3+fGdggIOx2kaa2YI9QWarc5Ce1ipNWMKeSG4DysFF52KBmTNMmn5HqCFkwy34rDg05gDwgH3bBi+sgFhN/e8QvRn8kbamCOhgrZ9GJhFDgfcMHzFb6BAtjKpFhzTjwv1KCVuxHvCbsSiEz4CANnj84cwHdFXAbAOJ4LTSAawGWFn5tDhLMYz6nWeU2wJfIhmIJBefcd/A5FWQWGgrWzyORZ3Q6HuV+Jf0Bj+BTX69fm1zWgK7By1YTXchFDORywnfQ7GpzOo6S+qECrsx2ifVQAAAABJRU5ErkJggg==');
   }
+  div.callout-tip.callout-style-default .callout-title {
     background-color: #ccf1e3
   }
     background-image: url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAAAXNSR0IArs4c6QAAAERlWElmTU0AKgAAAAgAAYdpAAQAAAABAAAAGgAAAAAAA6ABAAMAAAABAAEAAKACAAQAAAABAAAAIKADAAQAAAABAAAAIAAAAACshmLzAAACV0lEQVRYCdVWzWoUQRCuqp2ICBLJXgITZL1EfQDBW/bkzUMUD7klD+ATSHBEfAIfQO+iXsWDxJsHL96EHAwhgzlkg8nBg25XWb0zIb0zs9muYYWkoKeru+vn664fBqElyZNuyh167NXJ8Ut8McjbmEraKHkd7uAnAFku+VWdb3reSmRV8PKSLfZ0Gjn3a6Xlcq9YGb6tADjn+lUfTXtVmaZ1KwBIvFI11rRXlWlatwIAAv2asaa9mlB9wwygiDX26qaw1yYPzFXg2N1GgG0FMF8Oj+VIx7E/03lHx8UhvYyNZLN7BwSPgekXXLribw7w5/c8EF+DBK5idvDVYtEEwMeYefjjLAdEyQ3M9nfOkgnPTEkYU+sxMq0BxNR6jExrAI31H1rzvLEfRIdgcv1XEdj6QTQAS2wtstEALLG1yEZ3QhH6oDX7ExBSFEkFINXH98NTrme5IOaaA7kIfiu2L8A3qhH9zRbukdCqdsA98TdElyeMe5BI8Rs2xHRIsoTSSVFfCFCWGPn9XHb4cdobRIWABNf0add9jakDjQJpJ1bTXOJXnnRXHRf+dNL1ZV1MBRCXhMbaHqGI1JkKIL7+i8uffuP6wVQAzO7+qVEbF6NbS0LJureYcWXUUhH66nLR5rYmva+2tjRFtojkM2aD76HEGAD3tPtKM309FJg5j/K682ywcWJ3PASCcycH/22u+Bh7Aa0ehM2Fu4z0SAE81HF9RkB21c5bEn4Dzw+/qNOyXr3DCTQDMBOdhi4nAgiFDGCinIa2owCEChUwD8qzd03PG+qdW/4fDzjUMcE1ZpIAAAAASUVORK5CYII=');
   }
+  div.callout-caution.callout-style-default .callout-title {
     background-color: #ffe5d0
   }
       margin-right: 0;
     }
   </style>
+  <script src="llm_conf_files/libs/quarto-diagram/mermaid.min.js"></script>
+  <script src="llm_conf_files/libs/quarto-diagram/mermaid-init.js"></script>
+  <link href="llm_conf_files/libs/quarto-diagram/mermaid.css" rel="stylesheet">
 </head>
 <body class="quarto-dark">
   <div class="reveal">
     <div class="slides">
 <section id="title-slide" class="quarto-title-block center">
+  <h1 class="title">Scaling Model Training with More Compute, How Do They Do It?</h1>
 <div class="quarto-title-authors">
 </div>
 </section>
 <h2>Who am I?</h2>
 <ul>
 <li>Zachary Mueller</li>
+<li>Technical Lead for the 🤗 Accelerate project</li>
 <li>API design geek</li>
 </ul>
 </section>
+<section id="understanding-gpu-usage" class="slide level2">
+<h2>Understanding GPU Usage</h2>
+<ul>
+<li>We can somewhat estimate the memory usage in vanilla full-fine-tuning of models</li>
+<li>Requires certain assumptions (that I’ll be covering):
+<ul>
+<li>Adam optimizer</li>
+<li>Batch size of 1</li>
+</ul></li>
+</ul>
+</section>
+<section id="understanding-gpu-usage-1" class="slide level2">
+<h2>Understanding GPU Usage</h2>
+<p>General estimate (<code>bert-base-cased</code>, 108M params):</p>
+<ul>
+<li>Each parameter is 4 bytes</li>
+<li>Backward ~= 2x the model size</li>
+<li>The optimizer step ~= 4x the model size (1x model, 1x gradients, 2x optimizer):</li>
+</ul>
+<div style="font-size: 50%;">
+<table>
+<thead>
+<tr class="header">
+<th>dtype</th>
+<th style="text-align: left;">Model</th>
+<th style="text-align: center;">Gradients</th>
+<th style="text-align: center;">Backward pass</th>
+<th style="text-align: center;">Optimizer step</th>
+<th style="text-align: center;">Highest</th>
+</tr>
+</thead>
+<tbody>
+<tr class="odd">
+<td>float32</td>
+<td style="text-align: left;">413.18 MB</td>
+<td style="text-align: center;">413.18 MB</td>
+<td style="text-align: center;">826.36 MB</td>
+<td style="text-align: center;">1.61 GB</td>
+<td style="text-align: center;">1.61 GB</td>
+</tr>
+<tr class="even">
+<td>float16</td>
+<td style="text-align: left;">413.18 MB*</td>
+<td style="text-align: center;">619.77 MB</td>
+<td style="text-align: center;">826.36 MB</td>
+<td style="text-align: center;">826.36 MB</td>
+<td style="text-align: center;">826.36 MB</td>
+</tr>
+</tbody>
+</table>
+<p>*All estimations were based off the <a href="https://huggingface.co/spaces/hf-accelerate/model-memory-usage">Model Estimator Tool</a></p>
+</div>
+</section>
+<section id="understanding-gpu-usage-2" class="slide level2">
+<h2>Understanding GPU Usage</h2>
+<p>This works fine for small models, we have cards with anywhere from 12-24GB of GPU memory (on the GPU-poor side).</p>
+<p>But what happens as we scale?</p>
+<p>Here’s <code>llama-3-8B</code> (8.03B parameters)</p>
+<div style="font-size: 50%;">
+<table>
+<thead>
+<tr class="header">
+<th>dtype</th>
+<th style="text-align: left;">Model</th>
+<th style="text-align: center;">Gradients</th>
+<th style="text-align: center;">Backward pass</th>
+<th style="text-align: center;">Optimizer step</th>
+<th style="text-align: center;">Highest</th>
+</tr>
+</thead>
+<tbody>
+<tr class="odd">
+<td>float32</td>
+<td style="text-align: left;">28.21 GB</td>
+<td style="text-align: center;">28.21 GB</td>
+<td style="text-align: center;">56.43 GB</td>
+<td style="text-align: center;">112.84 GB</td>
+<td style="text-align: center;">112.84 GB</td>
+</tr>
+<tr class="even">
+<td>float16</td>
+<td style="text-align: left;">28.21 GB*</td>
+<td style="text-align: center;">42.32 GB</td>
+<td style="text-align: center;">56.43 GB</td>
+<td style="text-align: center;">56.43 GB</td>
+<td style="text-align: center;">56.43 GB</td>
+</tr>
+</tbody>
+</table>
+</div>
+<p>Well, <em>I</em> don’t have 56GB of GPU memory in a single card, let alone 112GB.</p>
+<p>What can we do?</p>
+</section>
+<section>
+<section id="distributed-training" class="title-slide slide level1 center">
+<h1>Distributed Training</h1>
+</section>
+<section id="kinds-of-training" class="slide level2">
+<h2>Kinds of Training</h2>
+<ul>
+<li>Single GPU:
+<ul>
+<li>No distributed techniques at play</li>
+</ul></li>
+<li>DDP:
+<ul>
+<li>A full copy of the model exists on each device, but data is chunked between each GPU</li>
+</ul></li>
+<li>FSDP &amp; DeepSpeed:
+<ul>
+<li>Split chunks of the model and optimizer states across GPUs, allowing for training bigger models on smaller (multiple) GPUs</li>
+</ul></li>
+</ul>
+</section></section>
+<section>
+<section id="fully-sharded-data-parallelism" class="title-slide slide level1 center">
+<h1>Fully Sharded Data Parallelism</h1>
+</section>
+<section id="fully-sharded-data-parallelism-1" class="slide level2">
+<h2>Fully Sharded Data Parallelism</h2>
+<img data-src="fsdp.png" id="fig-539a35d47e664c97a50115a146a7f1bd-1" class="r-stretch quarto-figure-center"><aside class="notes">
+<ul>
+<li>Take the model and split it across <code>n</code> GPUs</li>
+<li>Each GPU computes the shard’s gradients</li>
+<li>At the end, all gradients are synchronized and the final full model gradient is calculated</li>
+<li>The backward pass can then be performed</li>
+</ul>
+<style type="text/css">
+        span.MJX_Assistive_MathML {
+          position:absolute!important;
+          clip: rect(1px, 1px, 1px, 1px);
+          padding: 1px 0 0 0!important;
+          border: 0!important;
+          height: 1px!important;
+          width: 1px!important;
+          overflow: hidden!important;
+          display:block!important;
+      }</style></aside>
+</section>
+<section id="fsdp-getting-parameter-specific" class="slide level2">
+<h2>FSDP: Getting parameter specific</h2>
+<ul>
+<li>Different parameters can dicatate how much memory is needed for total GPU training across multiple GPUs</li>
+<li>These include how model weights are sharded, gradients, and more.</li>
+<li>I’ll cover some important ones I needed when doing a Full-Fine-Tune of Llama-3-8B <em>without PEFT</em> on 2x4090’s</li>
+</ul>
+</section>
+<section id="sharding_strategy" class="slide level2">
+<h2><code>sharding_strategy</code></h2>
+<ul>
+<li>Dictates the level of divving resources to perform
+<ul>
+<li><code>FULL_SHARD</code>: Includes optimizer states, gradients, and parameters</li>
+<li><code>SHARD_GRAD_OP</code>: Includes optimizer states and gradients</li>
+<li><code>NO_SHARD</code>: Normal DDP</li>
+<li><code>HYBRID_SHARD</code>: Includes optimizer states, gradients, and parameters but each node has the full model</li>
+</ul>
+<aside class="notes">
+<pre><code>FULL_SHARD:
+  Parameters, Gradients, Optimizer States: All are sharded.
+  Parameters Handling: Unshard before forward pass, reshard after forward pass, unshard before backward pass, reshard after backward pass.
+  Gradients Handling: Synchronize and shard after backward pass.
+  Optimizer States: Updated locally per rank.</code></pre>
+<p>SHARD_GRAD_OP: Gradients and Optimizer States: Sharded during computation. Parameters: Unshard before forward pass, remain unsharded during forward pass, reshard after backward pass. Inside no_sync(): Parameters are not resharded after backward computation. Optimizer States: Updated locally per rank.</p>
+<p>NO_SHARD: Parameters, Gradients, Optimizer States: Not sharded, replicated across ranks. Gradients Handling: Synchronized via all-reduce after backward pass. Optimizer States: Updated locally per rank.</p>
+<p>HYBRID_SHARD: Parameters, Gradients, Optimizer States: Combines FULL_SHARD within a node and replicates parameters across nodes. Communication: Expensive operations like all-gathers and reduce-scatters are limited to within a node, enhancing performance for medium-sized models.</p>
+<style type="text/css">
+        span.MJX_Assistive_MathML {
+          position:absolute!important;
+          clip: rect(1px, 1px, 1px, 1px);
+          padding: 1px 0 0 0!important;
+          border: 0!important;
+          height: 1px!important;
+          width: 1px!important;
+          overflow: hidden!important;
+          display:block!important;
+      }</style></aside></li>
+</ul>
+</section>
+<section id="auto_wrap_policy" class="slide level2">
+<h2><code>auto_wrap_policy</code>:</h2>
+<ul>
+<li>How the model should be split</li>
+<li>Can be either <code>TRANSFORMER_BASED_WRAP</code> or <code>SIZE_BASED_WRAP</code></li>
+<li><code>TRANSFORMER</code>/<code>fsdp_transformers_layer_cls_to_wrap</code>:
+<ul>
+<li>Need to declare the layer</li>
+<li>Generally <code>transformers</code> has good defaults</li>
+</ul></li>
+<li><code>SIZE</code>/<code>fsdp_min_num_param</code>:
+<ul>
+<li>Number of total parameters in a shard</li>
+</ul></li>
+</ul>
+</section>
+<section id="offload_params" class="slide level2">
+<h2><code>offload_params</code>:</h2>
+<ul>
+<li>Offloads the parameters and gradients to the CPU if they can’t fit into memory</li>
+<li>Allows you to train much larger models locally, but will be much slower</li>
+</ul>
+<blockquote>
+<p>Case: FFT of Llama-3-8B with <code>fsdp_offload_params</code> on 2x4090 GPUs was 72hrs, vs ~an hour or two when using 1xH100</p>
+</blockquote>
+</section>
+<section id="cpu_ram_efficient_loading-and-sync_module_states" class="slide level2">
+<h2><code>cpu_ram_efficient_loading</code> and <code>sync_module_states</code></h2>
+<ul>
+<li>Uses the idea behind big model inference/the <code>meta</code> device to load in the model to the GPU in a low-ram scenario</li>
+<li>Rather than needing <code>model_size</code> * <code>n_gpus</code> RAM, we can load the model on a single node and then send the weights directly to each shard when the time is right via <code>sync_module_states</code></li>
+</ul>
+</section></section>
+<section>
+<section id="tying-this-to-accelerate" class="title-slide slide level1 center">
+<h1>Tying this to 🤗 Accelerate</h1>
+</section>
+<section id="tying-this-to-accelerate-1" class="slide level2">
+<h2>Tying this to 🤗 Accelerate</h2>
+<ul>
+<li>So far we’ve covered the theory, but how do we put it into practice</li>
+<li>By using a library that’s at the heart of the entire open-source ecosystem</li>
+</ul>
+<div style="font-size: 60%;padding-left:10%;padding-top:0%;">
+<ul>
+<li>Nearly all of 🤗</li>
+<li><code>axolotl</code></li>
+<li><code>fastai</code></li>
+<li><code>FastChat</code></li>
+<li><code>lucidrains</code></li>
+<li><code>kornia</code></li>
+</ul>
+</div>
+<p>Are you using it and you don’t even know?</p>
+</section>
 <section id="what-is-accelerate" class="slide level2">
 <h2>What is 🤗 Accelerate?</h2>
 <div class="cell" data-reveal="true" data-fig-height="6">
 <div class="cell-output-display">
 <div>
+<div>
+<pre class="mermaid mermaid-js">graph LR
+    A(("🤗 Accelerate#32;"))
+    A --&gt; B["CLI Interface#32;"]
     A --&gt; C["Training Library#32;"]
     A --&gt; D["Big Model&lt;br&gt;Inference#32;"]
 </pre>
 </div>
 </div>
 </div>
 </div>
 </section>
+<section id="a-cli-interface" class="slide level2">
+<h2>A CLI Interface</h2>
+<ul>
+<li><code>accelerate config</code>
+<ul>
+<li>Configure the environment</li>
+</ul></li>
+<li><code>accelerate estimate-memory</code>
+<ul>
+<li>How to guess vRAM requirements</li>
+</ul></li>
+<li><code>accelerate launch</code>
+<ul>
+<li>How to run your script</li>
+</ul></li>
+</ul>
 </section>
+<section id="launching-distributed-training-is-hard" class="slide level2">
+<h2>Launching distributed training is hard</h2>
 <ul>
+<li><div class="sourceCode" id="cb2"><pre class="sourceCode numberSource bash number-lines code-with-copy"><code class="sourceCode bash"><span id="cb2-1"><a href="#cb2-1"></a><span class="ex">python</span> script.py</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div></li>
+<li><div class="sourceCode" id="cb3"><pre class="sourceCode numberSource bash number-lines code-with-copy"><code class="sourceCode bash"><span id="cb3-1"><a href="#cb3-1"></a><span class="ex">torchrun</span> <span class="at">--nnodes</span><span class="op">=</span>1 <span class="at">--nproc_per_node</span><span class="op">=</span>2 script.py</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div></li>
+<li><div class="sourceCode" id="cb4"><pre class="sourceCode numberSource bash number-lines code-with-copy"><code class="sourceCode bash"><span id="cb4-1"><a href="#cb4-1"></a><span class="ex">deepspeed</span> <span class="at">--num_gpus</span><span class="op">=</span>2 script.py</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div></li>
 </ul>
+<p>How can we make this better?</p>
 </section>
+<section id="accelerate-launch" class="slide level2">
+<h2><code>accelerate launch</code></h2>
+<div class="sourceCode" id="cb5"><pre class="sourceCode numberSource bash number-lines code-with-copy"><code class="sourceCode bash"><span id="cb5-1"><a href="#cb5-1"></a><span class="ex">accelerate</span> launch script.py</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
 </section>
+<section id="accelerate-config" class="slide level2">
+<h2><code>accelerate config</code></h2>
+<ul>
+<li>Rely on <code>config.yaml</code> files</li>
+<li>Choose to either running <code>accelerate config</code> or write your own:</li>
+</ul>
+<div class="columns" style="font-size: 50%;padding-left:10%;">
+<div class="column" style="width:40%;">
+<div class="code-with-filename">
+<div class="code-with-filename-file">
+<pre><strong>ddp_config.yaml</strong></pre>
+</div>
+<div class="sourceCode" id="cb6" data-filename="ddp_config.yaml"><pre class="sourceCode numberSource yaml number-lines code-with-copy"><code class="sourceCode yaml"><span id="cb6-1"><a href="#cb6-1"></a><span class="fu">compute_environment</span><span class="kw">:</span><span class="at"> LOCAL_MACHINE</span></span>
+<span id="cb6-2"><a href="#cb6-2"></a><span class="fu">distributed_type</span><span class="kw">:</span><span class="at"> MULTI_GPU</span></span>
+<span id="cb6-3"><a href="#cb6-3"></a><span class="fu">main_training_function</span><span class="kw">:</span><span class="at"> main</span></span>
+<span id="cb6-4"><a href="#cb6-4"></a><span class="fu">mixed_precision</span><span class="kw">:</span><span class="at"> bf16</span></span>
+<span id="cb6-5"><a href="#cb6-5"></a><span class="fu">num_machines</span><span class="kw">:</span><span class="at"> </span><span class="dv">1</span></span>
+<span id="cb6-6"><a href="#cb6-6"></a><span class="fu">num_processes</span><span class="kw">:</span><span class="at"> </span><span class="dv">8</span></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
+</div>
+</div><div class="column" style="width:40%;">
+<div class="code-with-filename">
+<div class="code-with-filename-file">
+<pre><strong>fsdp_config.yaml</strong></pre>
+</div>
+<div class="sourceCode" id="cb7" data-filename="fsdp_config.yaml"><pre class="sourceCode numberSource yaml number-lines code-with-copy"><code class="sourceCode yaml"><span id="cb7-1"><a href="#cb7-1"></a><span class="fu">compute_environment</span><span class="kw">:</span><span class="at"> LOCAL_MACHINE</span></span>
+<span id="cb7-2"><a href="#cb7-2"></a><span class="fu">distributed_type</span><span class="kw">:</span><span class="at"> FSDP</span></span>
+<span id="cb7-3"><a href="#cb7-3"></a><span class="fu">fsdp_config</span><span class="kw">:</span></span>
+<span id="cb7-4"><a href="#cb7-4"></a><span class="at">  </span><span class="fu">fsdp_auto_wrap_policy</span><span class="kw">:</span><span class="at"> TRANSFORMER_BASED_WRAP</span></span>
+<span id="cb7-5"><a href="#cb7-5"></a><span class="at">  </span><span class="fu">fsdp_backward_prefetch</span><span class="kw">:</span><span class="at"> BACKWARD_PRE</span></span>
+<span id="cb7-6"><a href="#cb7-6"></a><span class="at">  </span><span class="fu">fsdp_cpu_ram_efficient_loading</span><span class="kw">:</span><span class="at"> </span><span class="ch">true</span></span>
+<span id="cb7-7"><a href="#cb7-7"></a><span class="at">  </span><span class="fu">fsdp_forward_prefetch</span><span class="kw">:</span><span class="at"> </span><span class="ch">false</span></span>
+<span id="cb7-8"><a href="#cb7-8"></a><span class="at">  </span><span class="fu">fsdp_offload_params</span><span class="kw">:</span><span class="at"> </span><span class="ch">false</span></span>
+<span id="cb7-9"><a href="#cb7-9"></a><span class="at">  </span><span class="fu">fsdp_sharding_strategy</span><span class="kw">:</span><span class="at"> FULL_SHARD</span></span>
+<span id="cb7-10"><a href="#cb7-10"></a><span class="at">  </span><span class="fu">fsdp_state_dict_type</span><span class="kw">:</span><span class="at"> SHARDED_STATE_DICT</span></span>
+<span id="cb7-11"><a href="#cb7-11"></a><span class="at">  </span><span class="fu">fsdp_sync_module_states</span><span class="kw">:</span><span class="at"> </span><span class="ch">true</span></span>
+<span id="cb7-12"><a href="#cb7-12"></a><span class="at">  </span><span class="fu">fsdp_use_orig_params</span><span class="kw">:</span><span class="at"> </span><span class="ch">false</span></span>
+<span id="cb7-13"><a href="#cb7-13"></a><span class="fu">main_training_function</span><span class="kw">:</span><span class="at"> main</span></span>
+<span id="cb7-14"><a href="#cb7-14"></a><span class="fu">mixed_precision</span><span class="kw">:</span><span class="at"> bf16</span></span>
+<span id="cb7-15"><a href="#cb7-15"></a><span class="fu">num_machines</span><span class="kw">:</span><span class="at"> </span><span class="dv">1</span></span>
+<span id="cb7-16"><a href="#cb7-16"></a><span class="fu">num_processes</span><span class="kw">:</span><span class="at"> </span><span class="dv">8</span></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
+</div>
+</div>
+</div>
 </section></section>
 <section>
 <section id="a-training-library" class="title-slide slide level1 center">
 <h1>A Training Library</h1>
 </section>
+<section id="a-training-library-the-code" class="slide level2">
+<h2>A Training Library: The Code</h2>
+<div class="columns" style="font-size: 50%;">
+<div class="column">
 <p><br><br><br></p>
+<div class="sourceCode" id="cb8" data-code-line-numbers="5-6,9"><pre class="sourceCode numberSource python number-lines code-with-copy"><code class="sourceCode python"><span id="cb8-1"><a href="#cb8-1"></a><span class="co"># For alignment purposes</span></span>
+<span id="cb8-2"><a href="#cb8-2"></a><span class="cf">for</span> batch <span class="kw">in</span> dataloader:</span>
+<span id="cb8-3"><a href="#cb8-3"></a>    optimizer.zero_grad()</span>
+<span id="cb8-4"><a href="#cb8-4"></a>    inputs, targets <span class="op">=</span> batch</span>
+<span id="cb8-5"><a href="#cb8-5"></a>    inputs <span class="op">=</span> inputs.to(device)</span>
+<span id="cb8-6"><a href="#cb8-6"></a>    targets <span class="op">=</span> targets.to(device)</span>
+<span id="cb8-7"><a href="#cb8-7"></a>    outputs <span class="op">=</span> model(inputs)</span>
+<span id="cb8-8"><a href="#cb8-8"></a>    loss <span class="op">=</span> loss_function(outputs, targets)</span>
+<span id="cb8-9"><a href="#cb8-9"></a>    loss.backward()</span>
+<span id="cb8-10"><a href="#cb8-10"></a>    optimizer.step()</span>
+<span id="cb8-11"><a href="#cb8-11"></a>    scheduler.step()</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
+</div><div class="column">
+<div class="sourceCode" id="cb9" data-code-line-numbers="1-7,12-13,16"><pre class="sourceCode numberSource python number-lines code-with-copy"><code class="sourceCode python"><span id="cb9-1"><a href="#cb9-1"></a><span class="im">from</span> accelerate <span class="im">import</span> Accelerator</span>
+<span id="cb9-2"><a href="#cb9-2"></a>accelerator <span class="op">=</span> Accelerator()</span>
+<span id="cb9-3"><a href="#cb9-3"></a>dataloader, model, optimizer scheduler <span class="op">=</span> (</span>
+<span id="cb9-4"><a href="#cb9-4"></a>    accelerator.prepare(</span>
+<span id="cb9-5"><a href="#cb9-5"></a>        dataloader, model, optimizer, scheduler</span>
+<span id="cb9-6"><a href="#cb9-6"></a>    )</span>
+<span id="cb9-7"><a href="#cb9-7"></a>)</span>
+<span id="cb9-8"><a href="#cb9-8"></a></span>
+<span id="cb9-9"><a href="#cb9-9"></a><span class="cf">for</span> batch <span class="kw">in</span> dataloader:</span>
+<span id="cb9-10"><a href="#cb9-10"></a>    optimizer.zero_grad()</span>
+<span id="cb9-11"><a href="#cb9-11"></a>    inputs, targets <span class="op">=</span> batch</span>
+<span id="cb9-12"><a href="#cb9-12"></a>    <span class="co"># inputs = inputs.to(device)</span></span>
+<span id="cb9-13"><a href="#cb9-13"></a>    <span class="co"># targets = targets.to(device)</span></span>
+<span id="cb9-14"><a href="#cb9-14"></a>    outputs <span class="op">=</span> model(inputs)</span>
+<span id="cb9-15"><a href="#cb9-15"></a>    loss <span class="op">=</span> loss_function(outputs, targets)</span>
+<span id="cb9-16"><a href="#cb9-16"></a>    accelerator.backward(loss) <span class="co"># loss.backward()</span></span>
+<span id="cb9-17"><a href="#cb9-17"></a>    optimizer.step()</span>
+<span id="cb9-18"><a href="#cb9-18"></a>    scheduler.step()</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
 </div>
 </div>
 </section>
+<section id="a-training-library-how-scaling-works" class="slide level2">
+<h2>A Training Library: How Scaling Works</h2>
+<ul>
+<li>Accelerate’s DataLoaders and schedulers work off of a sharding mindset</li>
+<li>Rather than repeating the same data across <code>n</code> nodes, we instead split it</li>
+<li>Speeds up training linearly</li>
+<li>Given a batch size of 16 on a single GPU, to recreate this across 8 GPUs you would use a batch size of 2</li>
+<li>This also means the scheduler will be stepped <code>n</code> GPUs at a time per “global step”</li>
+</ul>
 </section>
 <section id="a-training-library-mixed-precision" class="slide level2">
+<h2>A Training Library: Mixed Precision</h2>
+<ul>
+<li>This may be a bit different than your “normal” idea of mixed precision.</li>
+<li>We do <strong>not</strong> convert the model weights to BF16/FP16</li>
+<li>Instead we <strong>wrap the forward pass</strong> with <code>autocast</code> to convert the gradients automatically</li>
+<li>This preserves the original precision of the weights, which leads to stable training and better fine-tuning later on.</li>
+<li><strong>If you use <code>.bf16()</code> weights, you are STUCK in bf16 perminantly</strong></li>
+</ul>
 </section>
+<section id="a-training-library-mixed-precision-1" class="slide level2">
+<h2>A Training Library: Mixed Precision</h2>
 <ul>
+<li>Let’s tie that back up to the model estimator with neat tools like NVIDIA’s TransformerEngine</li>
 </ul>
+<div style="font-size: 60%;">
+<table style="width:100%;">
+<colgroup>
+<col style="width: 14%">
+<col style="width: 14%">
+<col style="width: 14%">
+<col style="width: 14%">
+<col style="width: 14%">
+<col style="width: 14%">
+<col style="width: 14%">
+</colgroup>
+<thead>
+<tr class="header">
+<th>Optimization Level</th>
+<th>Computation (GEMM)</th>
+<th>Comm</th>
+<th>Weight</th>
+<th>Master Weight</th>
+<th>Weight Gradient</th>
+<th>Optimizer States</th>
+</tr>
+</thead>
+<tbody>
+<tr class="odd">
+<td>FP16 AMP</td>
+<td>FP16</td>
+<td>FP32</td>
+<td>FP32</td>
+<td>N/A</td>
+<td>FP32</td>
+<td>FP32+FP32</td>
+</tr>
+<tr class="even">
+<td>Nvidia TE</td>
+<td>FP8</td>
+<td>FP32</td>
+<td>FP32</td>
+<td>N/A</td>
+<td>FP32</td>
+<td>FP32+FP32</td>
+</tr>
+<tr class="odd">
+<td>MS-AMP O1</td>
+<td>FP8</td>
+<td>FP8</td>
+<td>FP16</td>
+<td>N/A</td>
+<td>FP8</td>
+<td>FP32+FP32</td>
+</tr>
+<tr class="even">
+<td>MS-AMP O2</td>
+<td>FP8</td>
+<td>FP8</td>
+<td>FP16</td>
+<td>N/A</td>
+<td>FP8</td>
+<td>FP8+FP16</td>
+</tr>
+<tr class="odd">
+<td>MS-AMP O3</td>
+<td>FP8</td>
+<td>FP8</td>
+<td>FP8</td>
+<td>FP16</td>
+<td>FP8</td>
+<td>FP8+FP16</td>
+</tr>
+</tbody>
+</table>
 </div>
+<aside class="notes">
+<p>What is actually happening: * Linear Layers and other certain compatible layers are wrapped in a special version that allows for FP8 computation * The general forward pass is wrapped around BF16 * This means that the most memory saved is done during the gradients of the model, <em>not</em> the model itself. * With tools like <code>MS-AMP</code> we can convert more chunks into lower precision, but again like before stable training occurs when the models weights are in full precision and the backprop happens in full precision too.</p>
+<style type="text/css">
+        span.MJX_Assistive_MathML {
+          position:absolute!important;
+          clip: rect(1px, 1px, 1px, 1px);
+          padding: 1px 0 0 0!important;
+          border: 0!important;
+          height: 1px!important;
+          width: 1px!important;
+          overflow: hidden!important;
+          display:block!important;
+      }</style></aside>
 </section>
+<section id="deepspeed-vs-fully-sharded-data-parallelism" class="slide level2">
+<h2>DeepSpeed vs Fully Sharded Data Parallelism</h2>
+<ul>
+<li>Extremely similar, however mostly used different naming conventions for items and slight tweaks in the implementation</li>
+</ul>
+<div style="font-size: 50%;">
+<table style="width:100%;">
+<colgroup>
+<col style="width: 16%">
+<col style="width: 16%">
+<col style="width: 16%">
+<col style="width: 16%">
+<col style="width: 16%">
+<col style="width: 16%">
+</colgroup>
+<thead>
+<tr class="header">
+<th>Framework</th>
+<th>Model Loading (<code>torch_dtype</code>)</th>
+<th>Mixed Precision</th>
+<th>Preparation (Local)</th>
+<th>Training</th>
+<th>Optimizer (Local)</th>
+</tr>
+</thead>
+<tbody>
+<tr class="odd">
+<td>FSDP</td>
+<td>bf16</td>
+<td>default (none)</td>
+<td>bf16</td>
+<td>bf16</td>
+<td>bf16</td>
+</tr>
+<tr class="even">
+<td>FSDP</td>
+<td>bf16</td>
+<td>bf16</td>
+<td>fp32</td>
+<td>bf16</td>
+<td>fp32</td>
+</tr>
+<tr class="odd">
+<td>DeepSpeed</td>
+<td>bf16</td>
+<td>bf16</td>
+<td>fp32</td>
+<td>bf16</td>
+<td>fp32</td>
+</tr>
+</tbody>
+</table>
 </div>
+<p>To learn more, check out the <a href="https://huggingface.co/docs/accelerate/concept_guides/fsdp_and_deepspeed">documentation</a> or join my office hours</p>
 </section>
+<section id="key-takeaways" class="slide level2">
+<h2>Key Takeaways:</h2>
+<ul>
+<li>You can scale out training with <code>accelerate</code>, FSDP, and DeepSpeed across multiple GPUs to train bigger models</li>
+<li>Techniques like <code>FP8</code> can help speed up training some and reduce computational overhead</li>
+<li>Comes at a cost of end-precision and locking model weights for futher fine-tunes if not careful</li>
+</ul>
 </section>
 <section id="some-handy-resources" class="slide level2">
 <h2>Some Handy Resources</h2>
 <li><a href="https://huggingface.co/docs/accelerate/usage_guides/big_modeling">Big Model Inference tutorial</a></li>
 <li><a href="https://huggingface.co/docs/accelerate/usage_guides/deepspeed">DeepSpeed and 🤗 Accelerate</a></li>
 <li><a href="https://huggingface.co/docs/accelerate/usage_guides/fsdp">Fully Sharded Data Parallelism and 🤗 Accelerate</a></li>
+<li><a href="https://huggingface.co/docs/accelerate/concept_guides/fsdp_and_deepspeed">FSDP vs DeepSpeed In-Depth</a></li>
 </ul>
 <div class="footer footer-default">
 </div>
 </section></section>
     </div>
   </div>
   <script>window.backupDefine = window.define; window.define = undefined;</script>
+  <script src="llm_conf_files/libs/revealjs/dist/reveal.js"></script>
   <!-- reveal.js plugins -->
+  <script src="llm_conf_files/libs/revealjs/plugin/quarto-line-highlight/line-highlight.js"></script>
+  <script src="llm_conf_files/libs/revealjs/plugin/pdf-export/pdfexport.js"></script>
+  <script src="llm_conf_files/libs/revealjs/plugin/reveal-menu/menu.js"></script>
+  <script src="llm_conf_files/libs/revealjs/plugin/reveal-menu/quarto-menu.js"></script>
+  <script src="llm_conf_files/libs/revealjs/plugin/quarto-support/support.js"></script>
+  <script src="llm_conf_files/libs/revealjs/plugin/notes/notes.js"></script>
+  <script src="llm_conf_files/libs/revealjs/plugin/search/search.js"></script>
+  <script src="llm_conf_files/libs/revealjs/plugin/zoom/zoom.js"></script>
+  <script src="llm_conf_files/libs/revealjs/plugin/math/math.js"></script>
   <script>window.define = window.backupDefine; window.backupDefine = undefined;</script>
   <script>
       Reveal.initialize({
 'controlsAuto': true,
 'previewLinksAuto': false,
 'pdfSeparateFragments': false,
 'autoAnimateEasing': "ease",
 'autoAnimateDuration': 1,
 'autoAnimateUnmatched': true,
+'menu': {"side":"left","useTextContentForMissingTitles":true,"markers":false,"loadIcons":false,"custom":[{"title":"Tools","icon":"<i class=\"fas fa-gear\"></i>","content":"<ul class=\"slide-menu-items\">\n<li class=\"slide-tool-item active\" data-item=\"0\"><a href=\"#\" onclick=\"RevealMenuToolHandlers.fullscreen(event)\"><kbd>f</kbd> Fullscreen</a></li>\n<li class=\"slide-tool-item\" data-item=\"1\"><a href=\"#\" onclick=\"RevealMenuToolHandlers.speakerMode(event)\"><kbd>s</kbd> Speaker View</a></li>\n<li class=\"slide-tool-item\" data-item=\"2\"><a href=\"#\" onclick=\"RevealMenuToolHandlers.overview(event)\"><kbd>o</kbd> Slide Overview</a></li>\n<li class=\"slide-tool-item\" data-item=\"3\"><a href=\"#\" onclick=\"RevealMenuToolHandlers.togglePdfExport(event)\"><kbd>e</kbd> PDF Export Mode</a></li>\n<li class=\"slide-tool-item\" data-item=\"4\"><a href=\"#\" onclick=\"RevealMenuToolHandlers.keyboardHelp(event)\"><kbd>?</kbd> Keyboard Help</a></li>\n</ul>"}],"openButton":true},
 'smaller': false,
         // Display controls in the bottom right corner
       tabsets.forEach(function(tabset) {
         const tabby = new Tabby('#' + tabset.id);
       });
+      const isCodeAnnotation = (el) => {
+        for (const clz of el.classList) {
+          if (clz.startsWith('code-annotation-')) {
+            return true;
+          }
+        }
+        return false;
+      }
       const clipboard = new window.ClipboardJS('.code-copy-button', {
+        text: function(trigger) {
+          const codeEl = trigger.previousElementSibling.cloneNode(true);
+          for (const childEl of codeEl.children) {
+            if (isCodeAnnotation(childEl)) {
+              childEl.remove();
+            }
+          }
+          return codeEl.innerText;
         }
       });
       clipboard.on('success', function(e) {
         // clear code selection
         e.clearSelection();
       });
+      function tippyHover(el, contentFn, onTriggerFn, onUntriggerFn) {
         const config = {
           allowHTML: true,
           maxWidth: 500,
           delay: 100,
           arrow: false,
           },
           interactive: true,
           interactiveBorder: 10,
+          theme: 'light-border',
+          placement: 'bottom-start',
         };
+        if (contentFn) {
+          config.content = contentFn;
+        }
+        if (onTriggerFn) {
+          config.onTrigger = onTriggerFn;
+        }
+        if (onUntriggerFn) {
+          config.onUntrigger = onUntriggerFn;
+        }
           config['offset'] = [0,0];
           config['maxWidth'] = 700;
         window.tippy(el, config);

llm_conf.html DELETED Viewed

@@ -1,1337 +0,0 @@
-<!DOCTYPE html>
-<html lang="en"><head>
-<script src="llm_conf_files/libs/clipboard/clipboard.min.js"></script>
-<script src="llm_conf_files/libs/quarto-html/tabby.min.js"></script>
-<script src="llm_conf_files/libs/quarto-html/popper.min.js"></script>
-<script src="llm_conf_files/libs/quarto-html/tippy.umd.min.js"></script>
-<link href="llm_conf_files/libs/quarto-html/tippy.css" rel="stylesheet">
-<link href="llm_conf_files/libs/quarto-html/light-border.css" rel="stylesheet">
-<link href="llm_conf_files/libs/quarto-html/quarto-html.min.css" rel="stylesheet" data-mode="light">
-<link href="llm_conf_files/libs/quarto-html/quarto-syntax-highlighting-dark.css" rel="stylesheet" id="quarto-text-highlighting-styles"><meta charset="utf-8">
-  <meta name="generator" content="quarto-99.9.9">
-  <title>Scaling Model Training with More Compute, How Do They Do It?</title>
-  <meta name="apple-mobile-web-app-capable" content="yes">
-  <meta name="apple-mobile-web-app-status-bar-style" content="black-translucent">
-  <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no, minimal-ui">
-  <link rel="stylesheet" href="llm_conf_files/libs/revealjs/dist/reset.css">
-  <link rel="stylesheet" href="llm_conf_files/libs/revealjs/dist/reveal.css">
-  <style>
-    code{white-space: pre-wrap;}
-    span.smallcaps{font-variant: small-caps;}
-    div.columns{display: flex; gap: min(4vw, 1.5em);}
-    div.column{flex: auto; overflow-x: auto;}
-    div.hanging-indent{margin-left: 1.5em; text-indent: -1.5em;}
-    ul.task-list{list-style: none;}
-    ul.task-list li input[type="checkbox"] {
-      width: 0.8em;
-      margin: 0 0.8em 0.2em -1em; /* quarto-specific, see https://github.com/quarto-dev/quarto-cli/issues/4556 */
-      vertical-align: middle;
-    }
-    /* CSS for syntax highlighting */
-    pre > code.sourceCode { white-space: pre; position: relative; }
-    pre > code.sourceCode > span { line-height: 1.25; }
-    pre > code.sourceCode > span:empty { height: 1.2em; }
-    .sourceCode { overflow: visible; }
-    code.sourceCode > span { color: inherit; text-decoration: inherit; }
-    div.sourceCode { margin: 1em 0; }
-    pre.sourceCode { margin: 0; }
-    @media screen {
-    div.sourceCode { overflow: auto; }
-    }
-    @media print {
-    pre > code.sourceCode { white-space: pre-wrap; }
-    pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; }
-    }
-    pre.numberSource code
-      { counter-reset: source-line 0; }
-    pre.numberSource code > span
-      { position: relative; left: -4em; counter-increment: source-line; }
-    pre.numberSource code > span > a:first-child::before
-      { content: counter(source-line);
-        position: relative; left: -1em; text-align: right; vertical-align: baseline;
-        border: none; display: inline-block;
-        -webkit-touch-callout: none; -webkit-user-select: none;
-        -khtml-user-select: none; -moz-user-select: none;
-        -ms-user-select: none; user-select: none;
-        padding: 0 4px; width: 4em;
-      }
-    pre.numberSource { margin-left: 3em;  padding-left: 4px; }
-    div.sourceCode
-      { color: #f8f8f2;  }
-    @media screen {
-    pre > code.sourceCode > span > a:first-child::before { text-decoration: underline; }
-    }
-    code span { color: #f8f8f2; } /* Normal */
-    code span.al { color: #f07178; background-color: #2a0f15; font-weight: bold; } /* Alert */
-    code span.an { color: #d4d0ab; } /* Annotation */
-    code span.at { color: #00e0e0; } /* Attribute */
-    code span.bn { color: #d4d0ab; } /* BaseN */
-    code span.bu { color: #abe338; } /* BuiltIn */
-    code span.cf { color: #ffa07a; font-weight: bold; } /* ControlFlow */
-    code span.ch { color: #abe338; } /* Char */
-    code span.cn { color: #ffd700; } /* Constant */
-    code span.co { color: #f8f8f2; font-style: italic; } /* Comment */
-    code span.cv { color: #ffd700; } /* CommentVar */
-    code span.do { color: #f8f8f2; } /* Documentation */
-    code span.dt { color: #ffa07a; } /* DataType */
-    code span.dv { color: #d4d0ab; } /* DecVal */
-    code span.er { color: #f07178; text-decoration: underline; } /* Error */
-    code span.ex { color: #00e0e0; font-weight: bold; } /* Extension */
-    code span.fl { color: #d4d0ab; } /* Float */
-    code span.fu { color: #ffa07a; } /* Function */
-    code span.im { color: #abe338; } /* Import */
-    code span.in { color: #d4d0ab; } /* Information */
-    code span.kw { color: #ffa07a; font-weight: bold; } /* Keyword */
-    code span.op { color: #ffa07a; } /* Operator */
-    code span.ot { color: #00e0e0; } /* Other */
-    code span.pp { color: #dcc6e0; } /* Preprocessor */
-    code span.re { color: #00e0e0; background-color: #f8f8f2; } /* RegionMarker */
-    code span.sc { color: #abe338; } /* SpecialChar */
-    code span.ss { color: #abe338; } /* SpecialString */
-    code span.st { color: #abe338; } /* String */
-    code span.va { color: #00e0e0; } /* Variable */
-    code span.vs { color: #abe338; } /* VerbatimString */
-    code span.wa { color: #dcc6e0; } /* Warning */
-  </style>
-  <link rel="stylesheet" href="llm_conf_files/libs/revealjs/dist/theme/quarto.css">
-  <link href="llm_conf_files/libs/revealjs/plugin/quarto-line-highlight/line-highlight.css" rel="stylesheet">
-  <link href="llm_conf_files/libs/revealjs/plugin/reveal-menu/menu.css" rel="stylesheet">
-  <link href="llm_conf_files/libs/revealjs/plugin/reveal-menu/quarto-menu.css" rel="stylesheet">
-  <link href="llm_conf_files/libs/revealjs/plugin/quarto-support/footer.css" rel="stylesheet">
-  <style type="text/css">
-  .callout {
-    margin-top: 1em;
-    margin-bottom: 1em;
-    border-radius: .25rem;
-  }
-  .callout.callout-style-simple {
-    padding: 0em 0.5em;
-    border-left: solid #acacac .3rem;
-    border-right: solid 1px silver;
-    border-top: solid 1px silver;
-    border-bottom: solid 1px silver;
-    display: flex;
-  }
-  .callout.callout-style-default {
-    border-left: solid #acacac .3rem;
-    border-right: solid 1px silver;
-    border-top: solid 1px silver;
-    border-bottom: solid 1px silver;
-  }
-  .callout .callout-body-container {
-    flex-grow: 1;
-  }
-  .callout.callout-style-simple .callout-body {
-    font-size: 1rem;
-    font-weight: 400;
-  }
-  .callout.callout-style-default .callout-body {
-    font-size: 0.9rem;
-    font-weight: 400;
-  }
-  .callout.callout-titled.callout-style-simple .callout-body {
-    margin-top: 0.2em;
-  }
-  .callout:not(.callout-titled) .callout-body {
-      display: flex;
-  }
-  .callout:not(.no-icon).callout-titled.callout-style-simple .callout-content {
-    padding-left: 1.6em;
-  }
-  .callout.callout-titled .callout-header {
-    padding-top: 0.2em;
-    margin-bottom: -0.2em;
-  }
-  .callout.callout-titled .callout-title  p {
-    margin-top: 0.5em;
-    margin-bottom: 0.5em;
-  }
-  .callout.callout-titled.callout-style-simple .callout-content  p {
-    margin-top: 0;
-  }
-  .callout.callout-titled.callout-style-default .callout-content  p {
-    margin-top: 0.7em;
-  }
-  .callout.callout-style-simple div.callout-title {
-    border-bottom: none;
-    font-size: .9rem;
-    font-weight: 600;
-    opacity: 75%;
-  }
-  .callout.callout-style-default  div.callout-title {
-    border-bottom: none;
-    font-weight: 600;
-    opacity: 85%;
-    font-size: 0.9rem;
-    padding-left: 0.5em;
-    padding-right: 0.5em;
-  }
-  .callout.callout-style-default div.callout-content {
-    padding-left: 0.5em;
-    padding-right: 0.5em;
-  }
-  .callout.callout-style-simple .callout-icon::before {
-    height: 1rem;
-    width: 1rem;
-    display: inline-block;
-    content: "";
-    background-repeat: no-repeat;
-    background-size: 1rem 1rem;
-  }
-  .callout.callout-style-default .callout-icon::before {
-    height: 0.9rem;
-    width: 0.9rem;
-    display: inline-block;
-    content: "";
-    background-repeat: no-repeat;
-    background-size: 0.9rem 0.9rem;
-  }
-  .callout-title {
-    display: flex
-  }
-  .callout-icon::before {
-    margin-top: 1rem;
-    padding-right: .5rem;
-  }
-  .callout.no-icon::before {
-    display: none !important;
-  }
-  .callout.callout-titled .callout-body > .callout-content > :last-child {
-    padding-bottom: 0.5rem;
-    margin-bottom: 0;
-  }
-  .callout.callout-titled .callout-icon::before {
-    margin-top: .5rem;
-    padding-right: .5rem;
-  }
-  .callout:not(.callout-titled) .callout-icon::before {
-    margin-top: 1rem;
-    padding-right: .5rem;
-  }
-  /* Callout Types */
-  div.callout-note {
-    border-left-color: #4582ec !important;
-  }
-  div.callout-note .callout-icon::before {
-    background-image: url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAAAXNSR0IArs4c6QAAAERlWElmTU0AKgAAAAgAAYdpAAQAAAABAAAAGgAAAAAAA6ABAAMAAAABAAEAAKACAAQAAAABAAAAIKADAAQAAAABAAAAIAAAAACshmLzAAAEU0lEQVRYCcVXTWhcVRQ+586kSUMMxkyaElstCto2SIhitS5Ek8xUKV2poatCcVHtUlFQk8mbaaziwpWgglJwVaquitBOfhQXFlqlzSJpFSpIYyXNjBNiTCck7x2/8/LeNDOZxDuEkgOXe++553zfefee+/OYLOXFk3+1LLrRdiO81yNqZ6K9cG0P3MeFaMIQjXssE8Z1JzLO9ls20MBZX7oG8w9GxB0goaPrW5aNMp1yOZIa7Wv6o2ykpLtmAPs/vrG14Z+6d4jpbSKuhdcSyq9wGMPXjonwmESXrriLzFGOdDBLB8Y6MNYBu0dRokSygMA/mrun8MGFN3behm6VVAwg4WR3i6FvYK1T7MHo9BK7ydH+1uurECoouk5MPRyVSBrBHMYwVobG2aOXM07sWrn5qgB60rc6mcwIDJtQrnrEr44kmy+UO9r0u9O5/YbkS9juQckLed3DyW2XV/qWBBB3ptvI8EUY3I9p/67OW+g967TNr3Sotn3IuVlfMLVnsBwH4fsnebJvyGm5GeIUA3jljERmrv49SizPYuq+z7c2H/jlGC+Ghhupn/hcapqmcudB9jwJ/3jvnvu6vu5lVzF1fXyZuZZ7U8nRmVzytvT+H3kilYvH09mLWrQdwFSsFEsxFVs5fK7A0g8gMZjbif4ACpKbjv7gNGaD8bUrlk8x+KRflttr22JEMRUbTUwwDQScyzPgedQHZT0xnx7ujw2jfVfExwYHwOsDTjLdJ2ebmeQIlJ7neo41s/DrsL3kl+W2lWvAga0tR3zueGr6GL78M3ifH0rGXrBC2aAR8uYcIA5gwV8zIE8onoh8u0Fca/ciF7j1uOzEnqcIm59sEXoGc0+z6+H45V1CvAvHcD7THztu669cnp+L0okAeIc6zjbM/24LgGM1gZk7jnRu1aQWoU9sfUOuhrmtaPIO3YY1KLLWZaEO5TKUbMY5zx8W9UJ6elpLwKXbsaZ4EFl7B4bMtDv0iRipKoDQT2sNQI9b1utXFdYisi+wzZ/ri/1m7QfDgEuvgUUEIJPq3DhX/5DWNqIXDOweC2wvIR90Oq3lDpdMIgD2r0dXvGdsEW5H6x6HLRJYU7C69VefO1x8Gde1ZFSJLfWS1jbCnhtOPxmpfv2LXOA2Xk2tvnwKKPFuZ/oRmwBwqRQDcKNeVQkYcOjtWVBuM/JuYw5b6isojIkYxyYAFn5K7ZBF10fea52y8QltAg6jnMqNHFBmGkQ1j+U43HMi2xMar1Nv0zGsf1s8nUsmUtPOOrbFIR8bHFDMB5zL13Gmr/kGlCkUzedTzzmzsaJXhYawnA3UmARpiYj5ooJZiUoxFRtK3X6pgNPv+IZVPcnwbOl6f+aBaO1CNvPW9n9LmCp01nuSaTRF2YxHqZ8DYQT6WsXT+RD6eUztwYLZ8rM+rcPxamv1VQzFUkzFXvkiVrySGQgJNvXHJAxiU3/NwiC03rSf05VBaPtu/Z7/B8Yn/w7eguloAAAAAElFTkSuQmCC');
-  }
-  div.callout-note.callout-style-default .callout-title {
-    background-color: #dae6fb
-  }
-  div.callout-important {
-    border-left-color: #d9534f !important;
-  }
-  div.callout-important .callout-icon::before {
-    background-image: url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAAAXNSR0IArs4c6QAAAERlWElmTU0AKgAAAAgAAYdpAAQAAAABAAAAGgAAAAAAA6ABAAMAAAABAAEAAKACAAQAAAABAAAAIKADAAQAAAABAAAAIAAAAACshmLzAAAEKklEQVRYCcVXTWhcVRS+575MJym48A+hSRFr00ySRQhURRfd2HYjk2SSTokuBCkU2o0LoSKKraKIBTcuFCoidGFD08nkBzdREbpQ1EDNIv8qSGMFUboImMSZd4/f9zJv8ibJMC8xJQfO3HPPPef7zrvvvnvviIkpC9nsw0UttFunbUhpFzFtarSd6WJkStVMw5xyVqYTvkwfzuf/5FgtkVoB0729j1rjXwThS7Vio+Mo6DNnvLfahoZ+i/o32lULuJ3NNiz7q6+pyAUkJaFF6JwaM2lUJlV0MlnQn5aTRbEu0SEqHUa0A4AdiGuB1kFXRfVyg5d87+Dg4DL6m2TLAub60ilj7A1Ec4odSAc8X95sHh7+ZRPCFo6Fnp7HfU/fBng/hi10CjCnWnJjsxvDNxWw0NfV6Rv5GgP3I3jGWXumdTD/3cbEOP2ZbOZp69yniG3FQ9z1jD7bnBu9Fc2tKGC2q+uAJOQHBDRiZX1x36o7fWBs7J9ownbtO+n0/qWkvW7UPIfc37WgT6ZGR++EOJyeQDSb9UB+DZ1G6DdLDzyS+b/kBCYGsYgJbSQHuThGKRcw5xdeQf8YdNHsc6ePXrlSYMBuSIAFTGAtQo+VuALo4BX83N190NWZWbynBjhOHsmNfFWLeL6v+ynsA58zDvvAC8j5PkbOcXCMg2PZFk3q8MjI7WAG/Dp9AwP7jdGBOOQkAvlFUB+irtm16I1Zw9YBcpGTGXYmk3kQIC/Cds55l+iMI3jqhjAuaoe+am2Jw5GT3Nbz3CkE12NavmzN5+erJW7046n/CH1RO/RVa8lBLozXk9uqykkGAyRXLWlLv5jyp4RFsG5vGVzpDLnIjTWgnRy2Rr+tDKvRc7Y8AyZq10jj8DqXdnIRNtFZb+t/ZRtXcDiVnzpqx8mPcDWxgARUqx0W1QB9MeUZiNrV4qP+Ehc+BpNgATsTX8ozYKL2NtFYAHc84fG7ndxUPr+AR/iQSns7uSUufAymwDOb2+NjK27lEFocm/EE2WpyIy/Hi66MWuMKJn8RvxIcj87IM5Vh9663ziW36kR0HNenXuxmfaD8JC7tfKbrhFr7LiZCrMjrzTeGx+PmkosrkNzW94ObzwocJ7A1HokLolY+AvkTiD/q1H0cN48c5EL8Crkttsa/AXQVDmutfyku0E7jShx49XqV3MFK8IryDhYVbj7Sj2P2eBxwcXoe8T8idsKKPRcnZw1b+slFTubwUwhktrfnAt7J++jwQtLZcm3sr9LQrjRzz6cfMv9aLvgmnAGvpoaGLxM4mAEaLV7iAzQ3oU0IvD5x9ix3yF2RAAuYAOO2f7PEFWCXZ4C9Pb2UsgDeVnFSpbFK7/IWu7TPTvBqzbGdCHOJQSxiEjt6IyZmxQyEJHv6xyQsYk//moVFsN2zP6fRImjfq7/n/wFDguUQFNEwugAAAABJRU5ErkJggg==');
-  }
-  div.callout-important.callout-style-default .callout-title {
-    background-color: #f7dddc
-  }
-  div.callout-warning {
-    border-left-color: #f0ad4e !important;
-  }
-  div.callout-warning .callout-icon::before {
-    background-image: url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAAAXNSR0IArs4c6QAAAERlWElmTU0AKgAAAAgAAYdpAAQAAAABAAAAGgAAAAAAA6ABAAMAAAABAAEAAKACAAQAAAABAAAAIKADAAQAAAABAAAAIAAAAACshmLzAAAETklEQVRYCeVWW2gcVRg+58yaTUnizqbipZeX4uWhBEniBaoUX1Ioze52t7sRq6APio9V9MEaoWlVsFasRq0gltaAPuxms8lu0gcviE/FFOstVbSIxgcv6SU7EZqmdc7v9+9mJtNks51NTUH84ed889/PP+cmxP+d5FIbMJmNbpREu4WUkiTtCicKny0l1pIKmBzovF2S+hIJHX8iEu3hZJ5lNZGqyRrGSIQpq15AzF28jgpeY6yk6GVdrfFqdrD6Iw+QlB8g0YS2g7dyQmXM/IDhBhT0UCiRf59lfqmmDvzRt6kByV/m4JjtzuaujMUM2c5Z2d6JdKrRb3K2q6mA+oYVz8JnDdKPmmNthzkAk/lN63sYPgevrguc72aZX/L9C6x09GYyxBgCX4NlvyGUHOKELlm5rXeR1kchuChJt4SSwyddZRXgvwMGvYo4QSlk3/zkHD8UHxwVJA6zjZZqP8v8kK8OWLnIZtLyCAJagYC4rTGW/9Pqj92N/c+LUaAj27movwbi19tk/whRCIE7Q9vyI6yvRpftAKVTdUjOW40X3h5OXsKCdmFcx0xlLJoSuQngnrJe7Kcjm4OMq9FlC7CMmScQANuNvjfP3PjGXDBaUQmbp296S5L4DrpbrHN1T87ZVEZVCzg1FF0Ft+dKrlLukI+/c9ENo+TvlTDbYFvuKPtQ9+l052rXrgKoWkDAFnvh0wTOmYn8R5f4k/jN/fZiCM1tQx9jQQ4ANhqG4hiL0qIFTGViG9DKB7GYzgubnpofgYRwO+DFjh0Zin2m4b/97EDkXkc+f6xYAPX0KK2I/7fUQuwzuwo/L3AkcjugPNixC8cHf0FyPjWlItmLxWw4Ou9YsQCr5fijMGoD/zpdRy95HRysyXA74MWOnscpO4j2y3HAVisw85hX5+AFBRSHt4ShfLFkIMXTqyKFc46xdzQM6XbAi702a7sy04J0+feReMFKp5q9esYLCqAZYw/k14E/xcLLsFElaornTuJB0svMuJINy8xkIYuL+xPAlWRceH6+HX7THJ0djLUom46zREu7tTkxwmf/FdOZ/sh6Q8qvEAiHpm4PJ4a/doJe0gH1t+aHRgCzOvBvJedEK5OFE5jpm4AGP2a8Dxe3gGJ/pAutug9Gp6he92CsSsWBaEcxGx0FHytmIpuqGkOpldqNYQK8cSoXvd+xLxXADw0kf6UkJNFtdo5MOgaLjiQOQHcn+A6h5NuL2s0qsC2LOM75PcF3yr5STuBSAcGG+meA14K/CI21HcS4LBT6tv0QAh8Dr5l93AhZzG5ZJ4VxAqdZUEl9z7WJ4aN+svMvwHHL21UKTd1mqvChH7/Za5xzXBBKrUcB0TQ+Ulgkfbi/H/YT5EptrGzsEK7tR1B7ln9BBwckYfMiuSqklSznIuoIIOM42MQO+QnduCoFCI0bpkzjCjddHPN/F+2Yu+sd9bKNpVwHhbS3LluK/0zgfwD0xYI5dXuzlQAAAABJRU5ErkJggg==');
-  }
-  div.callout-warning.callout-style-default .callout-title {
-    background-color: #fcefdc
-  }
-  div.callout-tip {
-    border-left-color: #02b875 !important;
-  }
-  div.callout-tip .callout-icon::before {
-    background-image: url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAAAXNSR0IArs4c6QAAAERlWElmTU0AKgAAAAgAAYdpAAQAAAABAAAAGgAAAAAAA6ABAAMAAAABAAEAAKACAAQAAAABAAAAIKADAAQAAAABAAAAIAAAAACshmLzAAADr0lEQVRYCe1XTWgTQRj9ZjZV8a9SPIkKgj8I1bMHsUWrqYLVg4Ue6v9BwZOxSYsIerFao7UiUryIqJcqgtpimhbBXoSCVxUFe9CTiogUrUp2Pt+3aUI2u5vdNh4dmMzOzHvvezuz8xNFM0mjnbXaNu1MvFWRXkXEyE6aYOYJpdW4IXuA4r0fo8qqSMDBU0v1HJUgVieAXxzCsdE/YJTdFcVIZQNMyhruOMJKXYFoLfIfIvVIMWdsrd+Rpd86ZmyzzjJmLStqRn0v8lzkb4rVIXvnpScOJuAn2ACC65FkPzEdEy4TPWRLJ2h7z4cArXzzaOdKlbOvKKX25Wl00jSnrwVxAg3o4dRxhO13RBSdNvH0xSARv3adTXbBdTf64IWO2vH0LT+cv4GR1DJt+DUItaQogeBX/chhbTBxEiZ6gftlDNXTrvT7co4ub5A6gp9HIcHvzTa46OS5fBeP87Qm0fQkr4FsYgVQ7Qg+ZayaDg9jhg1GkWj8RG6lkeSacrrHgDaxdoBiZPg+NXV/KifMuB6//JmYH4CntVEHy/keA6x4h4CU5oFy8GzrBS18cLJMXcljAKB6INjWsRcuZBWVaS3GDrqB7rdapVIeA+isQ57Eev9eCqzqOa81CY05VLd6SamW2wA2H3SiTbnbSxmzfp7WtKZkqy4mdyAlGx7ennghYf8voqp9cLSgKdqNfa6RdRsAAkPwRuJZNbpByn+RrJi1RXTwdi8RQF6ymDwGMAtZ6TVE+4uoKh+MYkcLsT0Hk8eAienbiGdjJHZTpmNjlbFJNKDVAp2fJlYju6IreQxQ08UJDNYdoLSl6AadO+fFuCQqVMB1NJwPm69T04Wv5WhfcWyfXQB+wXRs1pt+nCknRa0LVzSA/2B+a9+zQJadb7IyyV24YAxKp2Jqs3emZTuNnKxsah+uabKbMk7CbTgJx/zIgQYErIeTKRQ9yD9wxVof5YolPHqaWo7TD6tJlh7jQnK5z2n3+fGdggIOx2kaa2YI9QWarc5Ce1ipNWMKeSG4DysFF52KBmTNMmn5HqCFkwy34rDg05gDwgH3bBi+sgFhN/e8QvRn8kbamCOhgrZ9GJhFDgfcMHzFb6BAtjKpFhzTjwv1KCVuxHvCbsSiEz4CANnj84cwHdFXAbAOJ4LTSAawGWFn5tDhLMYz6nWeU2wJfIhmIJBefcd/A5FWQWGgrWzyORZ3Q6HuV+Jf0Bj+BTX69fm1zWgK7By1YTXchFDORywnfQ7GpzOo6S+qECrsx2ifVQAAAABJRU5ErkJggg==');
-  }
-  div.callout-tip.callout-style-default .callout-title {
-    background-color: #ccf1e3
-  }
-  div.callout-caution {
-    border-left-color: #fd7e14 !important;
-  }
-  div.callout-caution .callout-icon::before {
-    background-image: url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAAAXNSR0IArs4c6QAAAERlWElmTU0AKgAAAAgAAYdpAAQAAAABAAAAGgAAAAAAA6ABAAMAAAABAAEAAKACAAQAAAABAAAAIKADAAQAAAABAAAAIAAAAACshmLzAAACV0lEQVRYCdVWzWoUQRCuqp2ICBLJXgITZL1EfQDBW/bkzUMUD7klD+ATSHBEfAIfQO+iXsWDxJsHL96EHAwhgzlkg8nBg25XWb0zIb0zs9muYYWkoKeru+vn664fBqElyZNuyh167NXJ8Ut8McjbmEraKHkd7uAnAFku+VWdb3reSmRV8PKSLfZ0Gjn3a6Xlcq9YGb6tADjn+lUfTXtVmaZ1KwBIvFI11rRXlWlatwIAAv2asaa9mlB9wwygiDX26qaw1yYPzFXg2N1GgG0FMF8Oj+VIx7E/03lHx8UhvYyNZLN7BwSPgekXXLribw7w5/c8EF+DBK5idvDVYtEEwMeYefjjLAdEyQ3M9nfOkgnPTEkYU+sxMq0BxNR6jExrAI31H1rzvLEfRIdgcv1XEdj6QTQAS2wtstEALLG1yEZ3QhH6oDX7ExBSFEkFINXH98NTrme5IOaaA7kIfiu2L8A3qhH9zRbukdCqdsA98TdElyeMe5BI8Rs2xHRIsoTSSVFfCFCWGPn9XHb4cdobRIWABNf0add9jakDjQJpJ1bTXOJXnnRXHRf+dNL1ZV1MBRCXhMbaHqGI1JkKIL7+i8uffuP6wVQAzO7+qVEbF6NbS0LJureYcWXUUhH66nLR5rYmva+2tjRFtojkM2aD76HEGAD3tPtKM309FJg5j/K682ywcWJ3PASCcycH/22u+Bh7Aa0ehM2Fu4z0SAE81HF9RkB21c5bEn4Dzw+/qNOyXr3DCTQDMBOdhi4nAgiFDGCinIa2owCEChUwD8qzd03PG+qdW/4fDzjUMcE1ZpIAAAAASUVORK5CYII=');
-  }
-  div.callout-caution.callout-style-default .callout-title {
-    background-color: #ffe5d0
-  }
-  </style>
-  <style type="text/css">
-    .reveal div.sourceCode {
-      margin: 0;
-      overflow: auto;
-    }
-    .reveal div.hanging-indent {
-      margin-left: 1em;
-      text-indent: -1em;
-    }
-    .reveal .slide:not(.center) {
-      height: 100%;
-    }
-    .reveal .slide.scrollable {
-      overflow-y: auto;
-    }
-    .reveal .footnotes {
-      height: 100%;
-      overflow-y: auto;
-    }
-    .reveal .slide .absolute {
-      position: absolute;
-      display: block;
-    }
-    .reveal .footnotes ol {
-      counter-reset: ol;
-      list-style-type: none;
-      margin-left: 0;
-    }
-    .reveal .footnotes ol li:before {
-      counter-increment: ol;
-      content: counter(ol) ". ";
-    }
-    .reveal .footnotes ol li > p:first-child {
-      display: inline-block;
-    }
-    .reveal .slide ul,
-    .reveal .slide ol {
-      margin-bottom: 0.5em;
-    }
-    .reveal .slide ul li,
-    .reveal .slide ol li {
-      margin-top: 0.4em;
-      margin-bottom: 0.2em;
-    }
-    .reveal .slide ul[role="tablist"] li {
-      margin-bottom: 0;
-    }
-    .reveal .slide ul li > *:first-child,
-    .reveal .slide ol li > *:first-child {
-      margin-block-start: 0;
-    }
-    .reveal .slide ul li > *:last-child,
-    .reveal .slide ol li > *:last-child {
-      margin-block-end: 0;
-    }
-    .reveal .slide .columns:nth-child(3) {
-      margin-block-start: 0.8em;
-    }
-    .reveal blockquote {
-      box-shadow: none;
-    }
-    .reveal .tippy-content>* {
-      margin-top: 0.2em;
-      margin-bottom: 0.7em;
-    }
-    .reveal .tippy-content>*:last-child {
-      margin-bottom: 0.2em;
-    }
-    .reveal .slide > img.stretch.quarto-figure-center,
-    .reveal .slide > img.r-stretch.quarto-figure-center {
-      display: block;
-      margin-left: auto;
-      margin-right: auto;
-    }
-    .reveal .slide > img.stretch.quarto-figure-left,
-    .reveal .slide > img.r-stretch.quarto-figure-left  {
-      display: block;
-      margin-left: 0;
-      margin-right: auto;
-    }
-    .reveal .slide > img.stretch.quarto-figure-right,
-    .reveal .slide > img.r-stretch.quarto-figure-right  {
-      display: block;
-      margin-left: auto;
-      margin-right: 0;
-    }
-  </style>
-  <script src="llm_conf_files/libs/quarto-diagram/mermaid.min.js"></script>
-  <script src="llm_conf_files/libs/quarto-diagram/mermaid-init.js"></script>
-  <link href="llm_conf_files/libs/quarto-diagram/mermaid.css" rel="stylesheet">
-</head>
-<body class="quarto-dark">
-  <div class="reveal">
-    <div class="slides">
-<section id="title-slide" class="quarto-title-block center">
-  <h1 class="title">Scaling Model Training with More Compute, How Do They Do It?</h1>
-<div class="quarto-title-authors">
-</div>
-</section>
-<section id="who-am-i" class="slide level2">
-<h2>Who am I?</h2>
-<ul>
-<li>Zachary Mueller</li>
-<li>Technical Lead for the 🤗 Accelerate project</li>
-<li>API design geek</li>
-</ul>
-</section>
-<section id="understanding-gpu-usage" class="slide level2">
-<h2>Understanding GPU Usage</h2>
-<ul>
-<li>We can somewhat estimate the memory usage in vanilla full-fine-tuning of models</li>
-<li>Requires certain assumptions (that I’ll be covering):
-<ul>
-<li>Adam optimizer</li>
-<li>Batch size of 1</li>
-</ul></li>
-</ul>
-</section>
-<section id="understanding-gpu-usage-1" class="slide level2">
-<h2>Understanding GPU Usage</h2>
-<p>General estimate (<code>bert-base-cased</code>, 108M params):</p>
-<ul>
-<li>Each parameter is 4 bytes</li>
-<li>Backward ~= 2x the model size</li>
-<li>The optimizer step ~= 4x the model size (1x model, 1x gradients, 2x optimizer):</li>
-</ul>
-<div style="font-size: 50%;">
-<table>
-<thead>
-<tr class="header">
-<th>dtype</th>
-<th style="text-align: left;">Model</th>
-<th style="text-align: center;">Gradients</th>
-<th style="text-align: center;">Backward pass</th>
-<th style="text-align: center;">Optimizer step</th>
-<th style="text-align: center;">Highest</th>
-</tr>
-</thead>
-<tbody>
-<tr class="odd">
-<td>float32</td>
-<td style="text-align: left;">413.18 MB</td>
-<td style="text-align: center;">413.18 MB</td>
-<td style="text-align: center;">826.36 MB</td>
-<td style="text-align: center;">1.61 GB</td>
-<td style="text-align: center;">1.61 GB</td>
-</tr>
-<tr class="even">
-<td>float16</td>
-<td style="text-align: left;">413.18 MB*</td>
-<td style="text-align: center;">619.77 MB</td>
-<td style="text-align: center;">826.36 MB</td>
-<td style="text-align: center;">826.36 MB</td>
-<td style="text-align: center;">826.36 MB</td>
-</tr>
-</tbody>
-</table>
-<p>*All estimations were based off the <a href="https://huggingface.co/spaces/hf-accelerate/model-memory-usage">Model Estimator Tool</a></p>
-</div>
-</section>
-<section id="understanding-gpu-usage-2" class="slide level2">
-<h2>Understanding GPU Usage</h2>
-<p>This works fine for small models, we have cards with anywhere from 12-24GB of GPU memory (on the GPU-poor side).</p>
-<p>But what happens as we scale?</p>
-<p>Here’s <code>llama-3-8B</code> (8.03B parameters)</p>
-<div style="font-size: 50%;">
-<table>
-<thead>
-<tr class="header">
-<th>dtype</th>
-<th style="text-align: left;">Model</th>
-<th style="text-align: center;">Gradients</th>
-<th style="text-align: center;">Backward pass</th>
-<th style="text-align: center;">Optimizer step</th>
-<th style="text-align: center;">Highest</th>
-</tr>
-</thead>
-<tbody>
-<tr class="odd">
-<td>float32</td>
-<td style="text-align: left;">28.21 GB</td>
-<td style="text-align: center;">28.21 GB</td>
-<td style="text-align: center;">56.43 GB</td>
-<td style="text-align: center;">112.84 GB</td>
-<td style="text-align: center;">112.84 GB</td>
-</tr>
-<tr class="even">
-<td>float16</td>
-<td style="text-align: left;">28.21 GB*</td>
-<td style="text-align: center;">42.32 GB</td>
-<td style="text-align: center;">56.43 GB</td>
-<td style="text-align: center;">56.43 GB</td>
-<td style="text-align: center;">56.43 GB</td>
-</tr>
-</tbody>
-</table>
-</div>
-<p>Well, <em>I</em> don’t have 56GB of GPU memory in a single card, let alone 112GB.</p>
-<p>What can we do?</p>
-</section>
-<section>
-<section id="distributed-training" class="title-slide slide level1 center">
-<h1>Distributed Training</h1>
-</section>
-<section id="kinds-of-training" class="slide level2">
-<h2>Kinds of Training</h2>
-<ul>
-<li>Single GPU:
-<ul>
-<li>No distributed techniques at play</li>
-</ul></li>
-<li>DDP:
-<ul>
-<li>A full copy of the model exists on each device, but data is chunked between each GPU</li>
-</ul></li>
-<li>FSDP &amp; DeepSpeed:
-<ul>
-<li>Split chunks of the model and optimizer states across GPUs, allowing for training bigger models on smaller (multiple) GPUs</li>
-</ul></li>
-</ul>
-</section></section>
-<section>
-<section id="fully-sharded-data-parallelism" class="title-slide slide level1 center">
-<h1>Fully Sharded Data Parallelism</h1>
-</section>
-<section id="fully-sharded-data-parallelism-1" class="slide level2">
-<h2>Fully Sharded Data Parallelism</h2>
-<img data-src="fsdp.png" id="fig-539a35d47e664c97a50115a146a7f1bd-1" class="r-stretch quarto-figure-center"><aside class="notes">
-<ul>
-<li>Take the model and split it across <code>n</code> GPUs</li>
-<li>Each GPU computes the shard’s gradients</li>
-<li>At the end, all gradients are synchronized and the final full model gradient is calculated</li>
-<li>The backward pass can then be performed</li>
-</ul>
-<style type="text/css">
-        span.MJX_Assistive_MathML {
-          position:absolute!important;
-          clip: rect(1px, 1px, 1px, 1px);
-          padding: 1px 0 0 0!important;
-          border: 0!important;
-          height: 1px!important;
-          width: 1px!important;
-          overflow: hidden!important;
-          display:block!important;
-      }</style></aside>
-</section>
-<section id="fsdp-getting-parameter-specific" class="slide level2">
-<h2>FSDP: Getting parameter specific</h2>
-<ul>
-<li>Different parameters can dicatate how much memory is needed for total GPU training across multiple GPUs</li>
-<li>These include how model weights are sharded, gradients, and more.</li>
-<li>I’ll cover some important ones I needed when doing a Full-Fine-Tune of Llama-3-8B <em>without PEFT</em> on 2x4090’s</li>
-</ul>
-</section>
-<section id="sharding_strategy" class="slide level2">
-<h2><code>sharding_strategy</code></h2>
-<ul>
-<li>Dictates the level of divving resources to perform
-<ul>
-<li><code>FULL_SHARD</code>: Includes optimizer states, gradients, and parameters</li>
-<li><code>SHARD_GRAD_OP</code>: Includes optimizer states and gradients</li>
-<li><code>NO_SHARD</code>: Normal DDP</li>
-<li><code>HYBRID_SHARD</code>: Includes optimizer states, gradients, and parameters but each node has the full model</li>
-</ul>
-<aside class="notes">
-<pre><code>FULL_SHARD:
-  Parameters, Gradients, Optimizer States: All are sharded.
-  Parameters Handling: Unshard before forward pass, reshard after forward pass, unshard before backward pass, reshard after backward pass.
-  Gradients Handling: Synchronize and shard after backward pass.
-  Optimizer States: Updated locally per rank.</code></pre>
-<p>SHARD_GRAD_OP: Gradients and Optimizer States: Sharded during computation. Parameters: Unshard before forward pass, remain unsharded during forward pass, reshard after backward pass. Inside no_sync(): Parameters are not resharded after backward computation. Optimizer States: Updated locally per rank.</p>
-<p>NO_SHARD: Parameters, Gradients, Optimizer States: Not sharded, replicated across ranks. Gradients Handling: Synchronized via all-reduce after backward pass. Optimizer States: Updated locally per rank.</p>
-<p>HYBRID_SHARD: Parameters, Gradients, Optimizer States: Combines FULL_SHARD within a node and replicates parameters across nodes. Communication: Expensive operations like all-gathers and reduce-scatters are limited to within a node, enhancing performance for medium-sized models.</p>
-<style type="text/css">
-        span.MJX_Assistive_MathML {
-          position:absolute!important;
-          clip: rect(1px, 1px, 1px, 1px);
-          padding: 1px 0 0 0!important;
-          border: 0!important;
-          height: 1px!important;
-          width: 1px!important;
-          overflow: hidden!important;
-          display:block!important;
-      }</style></aside></li>
-</ul>
-</section>
-<section id="auto_wrap_policy" class="slide level2">
-<h2><code>auto_wrap_policy</code>:</h2>
-<ul>
-<li>How the model should be split</li>
-<li>Can be either <code>TRANSFORMER_BASED_WRAP</code> or <code>SIZE_BASED_WRAP</code></li>
-<li><code>TRANSFORMER</code>/<code>fsdp_transformers_layer_cls_to_wrap</code>:
-<ul>
-<li>Need to declare the layer</li>
-<li>Generally <code>transformers</code> has good defaults</li>
-</ul></li>
-<li><code>SIZE</code>/<code>fsdp_min_num_param</code>:
-<ul>
-<li>Number of total parameters in a shard</li>
-</ul></li>
-</ul>
-</section>
-<section id="offload_params" class="slide level2">
-<h2><code>offload_params</code>:</h2>
-<ul>
-<li>Offloads the parameters and gradients to the CPU if they can’t fit into memory</li>
-<li>Allows you to train much larger models locally, but will be much slower</li>
-</ul>
-<blockquote>
-<p>Case: FFT of Llama-3-8B with <code>fsdp_offload_params</code> on 2x4090 GPUs was 72hrs, vs ~an hour or two when using 1xH100</p>
-</blockquote>
-</section>
-<section id="cpu_ram_efficient_loading-and-sync_module_states" class="slide level2">
-<h2><code>cpu_ram_efficient_loading</code> and <code>sync_module_states</code></h2>
-<ul>
-<li>Uses the idea behind big model inference/the <code>meta</code> device to load in the model to the GPU in a low-ram scenario</li>
-<li>Rather than needing <code>model_size</code> * <code>n_gpus</code> RAM, we can load the model on a single node and then send the weights directly to each shard when the time is right via <code>sync_module_states</code></li>
-</ul>
-</section></section>
-<section>
-<section id="tying-this-to-accelerate" class="title-slide slide level1 center">
-<h1>Tying this to 🤗 Accelerate</h1>
-</section>
-<section id="tying-this-to-accelerate-1" class="slide level2">
-<h2>Tying this to 🤗 Accelerate</h2>
-<ul>
-<li>So far we’ve covered the theory, but how do we put it into practice</li>
-<li>By using a library that’s at the heart of the entire open-source ecosystem</li>
-</ul>
-<div style="font-size: 60%;padding-left:10%;padding-top:0%;">
-<ul>
-<li>Nearly all of 🤗</li>
-<li><code>axolotl</code></li>
-<li><code>fastai</code></li>
-<li><code>FastChat</code></li>
-<li><code>lucidrains</code></li>
-<li><code>kornia</code></li>
-</ul>
-</div>
-<p>Are you using it and you don’t even know?</p>
-</section>
-<section id="what-is-accelerate" class="slide level2">
-<h2>What is 🤗 Accelerate?</h2>
-<div class="cell" data-reveal="true" data-fig-height="6">
-<div class="cell-output-display">
-<div>
-<div>
-<pre class="mermaid mermaid-js">graph LR
-    A(("🤗 Accelerate#32;"))
-    A --&gt; B["CLI Interface#32;"]
-    A --&gt; C["Training Library#32;"]
-    A --&gt; D["Big Model&lt;br&gt;Inference#32;"]
-</pre>
-</div>
-</div>
-</div>
-</div>
-</section>
-<section id="a-cli-interface" class="slide level2">
-<h2>A CLI Interface</h2>
-<ul>
-<li><code>accelerate config</code>
-<ul>
-<li>Configure the environment</li>
-</ul></li>
-<li><code>accelerate estimate-memory</code>
-<ul>
-<li>How to guess vRAM requirements</li>
-</ul></li>
-<li><code>accelerate launch</code>
-<ul>
-<li>How to run your script</li>
-</ul></li>
-</ul>
-</section>
-<section id="launching-distributed-training-is-hard" class="slide level2">
-<h2>Launching distributed training is hard</h2>
-<ul>
-<li><div class="sourceCode" id="cb2"><pre class="sourceCode numberSource bash number-lines code-with-copy"><code class="sourceCode bash"><span id="cb2-1"><a href="#cb2-1"></a><span class="ex">python</span> script.py</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div></li>
-<li><div class="sourceCode" id="cb3"><pre class="sourceCode numberSource bash number-lines code-with-copy"><code class="sourceCode bash"><span id="cb3-1"><a href="#cb3-1"></a><span class="ex">torchrun</span> <span class="at">--nnodes</span><span class="op">=</span>1 <span class="at">--nproc_per_node</span><span class="op">=</span>2 script.py</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div></li>
-<li><div class="sourceCode" id="cb4"><pre class="sourceCode numberSource bash number-lines code-with-copy"><code class="sourceCode bash"><span id="cb4-1"><a href="#cb4-1"></a><span class="ex">deepspeed</span> <span class="at">--num_gpus</span><span class="op">=</span>2 script.py</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div></li>
-</ul>
-<p>How can we make this better?</p>
-</section>
-<section id="accelerate-launch" class="slide level2">
-<h2><code>accelerate launch</code></h2>
-<div class="sourceCode" id="cb5"><pre class="sourceCode numberSource bash number-lines code-with-copy"><code class="sourceCode bash"><span id="cb5-1"><a href="#cb5-1"></a><span class="ex">accelerate</span> launch script.py</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
-</section>
-<section id="accelerate-config" class="slide level2">
-<h2><code>accelerate config</code></h2>
-<ul>
-<li>Rely on <code>config.yaml</code> files</li>
-<li>Choose to either running <code>accelerate config</code> or write your own:</li>
-</ul>
-<div class="columns" style="font-size: 50%;padding-left:10%;">
-<div class="column" style="width:40%;">
-<div class="code-with-filename">
-<div class="code-with-filename-file">
-<pre><strong>ddp_config.yaml</strong></pre>
-</div>
-<div class="sourceCode" id="cb6" data-filename="ddp_config.yaml"><pre class="sourceCode numberSource yaml number-lines code-with-copy"><code class="sourceCode yaml"><span id="cb6-1"><a href="#cb6-1"></a><span class="fu">compute_environment</span><span class="kw">:</span><span class="at"> LOCAL_MACHINE</span></span>
-<span id="cb6-2"><a href="#cb6-2"></a><span class="fu">distributed_type</span><span class="kw">:</span><span class="at"> MULTI_GPU</span></span>
-<span id="cb6-3"><a href="#cb6-3"></a><span class="fu">main_training_function</span><span class="kw">:</span><span class="at"> main</span></span>
-<span id="cb6-4"><a href="#cb6-4"></a><span class="fu">mixed_precision</span><span class="kw">:</span><span class="at"> bf16</span></span>
-<span id="cb6-5"><a href="#cb6-5"></a><span class="fu">num_machines</span><span class="kw">:</span><span class="at"> </span><span class="dv">1</span></span>
-<span id="cb6-6"><a href="#cb6-6"></a><span class="fu">num_processes</span><span class="kw">:</span><span class="at"> </span><span class="dv">8</span></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
-</div>
-</div><div class="column" style="width:40%;">
-<div class="code-with-filename">
-<div class="code-with-filename-file">
-<pre><strong>fsdp_config.yaml</strong></pre>
-</div>
-<div class="sourceCode" id="cb7" data-filename="fsdp_config.yaml"><pre class="sourceCode numberSource yaml number-lines code-with-copy"><code class="sourceCode yaml"><span id="cb7-1"><a href="#cb7-1"></a><span class="fu">compute_environment</span><span class="kw">:</span><span class="at"> LOCAL_MACHINE</span></span>
-<span id="cb7-2"><a href="#cb7-2"></a><span class="fu">distributed_type</span><span class="kw">:</span><span class="at"> FSDP</span></span>
-<span id="cb7-3"><a href="#cb7-3"></a><span class="fu">fsdp_config</span><span class="kw">:</span></span>
-<span id="cb7-4"><a href="#cb7-4"></a><span class="at">  </span><span class="fu">fsdp_auto_wrap_policy</span><span class="kw">:</span><span class="at"> TRANSFORMER_BASED_WRAP</span></span>
-<span id="cb7-5"><a href="#cb7-5"></a><span class="at">  </span><span class="fu">fsdp_backward_prefetch</span><span class="kw">:</span><span class="at"> BACKWARD_PRE</span></span>
-<span id="cb7-6"><a href="#cb7-6"></a><span class="at">  </span><span class="fu">fsdp_cpu_ram_efficient_loading</span><span class="kw">:</span><span class="at"> </span><span class="ch">true</span></span>
-<span id="cb7-7"><a href="#cb7-7"></a><span class="at">  </span><span class="fu">fsdp_forward_prefetch</span><span class="kw">:</span><span class="at"> </span><span class="ch">false</span></span>
-<span id="cb7-8"><a href="#cb7-8"></a><span class="at">  </span><span class="fu">fsdp_offload_params</span><span class="kw">:</span><span class="at"> </span><span class="ch">false</span></span>
-<span id="cb7-9"><a href="#cb7-9"></a><span class="at">  </span><span class="fu">fsdp_sharding_strategy</span><span class="kw">:</span><span class="at"> FULL_SHARD</span></span>
-<span id="cb7-10"><a href="#cb7-10"></a><span class="at">  </span><span class="fu">fsdp_state_dict_type</span><span class="kw">:</span><span class="at"> SHARDED_STATE_DICT</span></span>
-<span id="cb7-11"><a href="#cb7-11"></a><span class="at">  </span><span class="fu">fsdp_sync_module_states</span><span class="kw">:</span><span class="at"> </span><span class="ch">true</span></span>
-<span id="cb7-12"><a href="#cb7-12"></a><span class="at">  </span><span class="fu">fsdp_use_orig_params</span><span class="kw">:</span><span class="at"> </span><span class="ch">false</span></span>
-<span id="cb7-13"><a href="#cb7-13"></a><span class="fu">main_training_function</span><span class="kw">:</span><span class="at"> main</span></span>
-<span id="cb7-14"><a href="#cb7-14"></a><span class="fu">mixed_precision</span><span class="kw">:</span><span class="at"> bf16</span></span>
-<span id="cb7-15"><a href="#cb7-15"></a><span class="fu">num_machines</span><span class="kw">:</span><span class="at"> </span><span class="dv">1</span></span>
-<span id="cb7-16"><a href="#cb7-16"></a><span class="fu">num_processes</span><span class="kw">:</span><span class="at"> </span><span class="dv">8</span></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
-</div>
-</div>
-</div>
-</section></section>
-<section>
-<section id="a-training-library" class="title-slide slide level1 center">
-<h1>A Training Library</h1>
-</section>
-<section id="a-training-library-the-code" class="slide level2">
-<h2>A Training Library: The Code</h2>
-<div class="columns" style="font-size: 50%;">
-<div class="column">
-<p><br><br><br></p>
-<div class="sourceCode" id="cb8" data-code-line-numbers="5-6,9"><pre class="sourceCode numberSource python number-lines code-with-copy"><code class="sourceCode python"><span id="cb8-1"><a href="#cb8-1"></a><span class="co"># For alignment purposes</span></span>
-<span id="cb8-2"><a href="#cb8-2"></a><span class="cf">for</span> batch <span class="kw">in</span> dataloader:</span>
-<span id="cb8-3"><a href="#cb8-3"></a>    optimizer.zero_grad()</span>
-<span id="cb8-4"><a href="#cb8-4"></a>    inputs, targets <span class="op">=</span> batch</span>
-<span id="cb8-5"><a href="#cb8-5"></a>    inputs <span class="op">=</span> inputs.to(device)</span>
-<span id="cb8-6"><a href="#cb8-6"></a>    targets <span class="op">=</span> targets.to(device)</span>
-<span id="cb8-7"><a href="#cb8-7"></a>    outputs <span class="op">=</span> model(inputs)</span>
-<span id="cb8-8"><a href="#cb8-8"></a>    loss <span class="op">=</span> loss_function(outputs, targets)</span>
-<span id="cb8-9"><a href="#cb8-9"></a>    loss.backward()</span>
-<span id="cb8-10"><a href="#cb8-10"></a>    optimizer.step()</span>
-<span id="cb8-11"><a href="#cb8-11"></a>    scheduler.step()</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
-</div><div class="column">
-<div class="sourceCode" id="cb9" data-code-line-numbers="1-7,12-13,16"><pre class="sourceCode numberSource python number-lines code-with-copy"><code class="sourceCode python"><span id="cb9-1"><a href="#cb9-1"></a><span class="im">from</span> accelerate <span class="im">import</span> Accelerator</span>
-<span id="cb9-2"><a href="#cb9-2"></a>accelerator <span class="op">=</span> Accelerator()</span>
-<span id="cb9-3"><a href="#cb9-3"></a>dataloader, model, optimizer scheduler <span class="op">=</span> (</span>
-<span id="cb9-4"><a href="#cb9-4"></a>    accelerator.prepare(</span>
-<span id="cb9-5"><a href="#cb9-5"></a>        dataloader, model, optimizer, scheduler</span>
-<span id="cb9-6"><a href="#cb9-6"></a>    )</span>
-<span id="cb9-7"><a href="#cb9-7"></a>)</span>
-<span id="cb9-8"><a href="#cb9-8"></a></span>
-<span id="cb9-9"><a href="#cb9-9"></a><span class="cf">for</span> batch <span class="kw">in</span> dataloader:</span>
-<span id="cb9-10"><a href="#cb9-10"></a>    optimizer.zero_grad()</span>
-<span id="cb9-11"><a href="#cb9-11"></a>    inputs, targets <span class="op">=</span> batch</span>
-<span id="cb9-12"><a href="#cb9-12"></a>    <span class="co"># inputs = inputs.to(device)</span></span>
-<span id="cb9-13"><a href="#cb9-13"></a>    <span class="co"># targets = targets.to(device)</span></span>
-<span id="cb9-14"><a href="#cb9-14"></a>    outputs <span class="op">=</span> model(inputs)</span>
-<span id="cb9-15"><a href="#cb9-15"></a>    loss <span class="op">=</span> loss_function(outputs, targets)</span>
-<span id="cb9-16"><a href="#cb9-16"></a>    accelerator.backward(loss) <span class="co"># loss.backward()</span></span>
-<span id="cb9-17"><a href="#cb9-17"></a>    optimizer.step()</span>
-<span id="cb9-18"><a href="#cb9-18"></a>    scheduler.step()</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
-</div>
-</div>
-</section>
-<section id="a-training-library-how-scaling-works" class="slide level2">
-<h2>A Training Library: How Scaling Works</h2>
-<ul>
-<li>Accelerate’s DataLoaders and schedulers work off of a sharding mindset</li>
-<li>Rather than repeating the same data across <code>n</code> nodes, we instead split it</li>
-<li>Speeds up training linearly</li>
-<li>Given a batch size of 16 on a single GPU, to recreate this across 8 GPUs you would use a batch size of 2</li>
-<li>This also means the scheduler will be stepped <code>n</code> GPUs at a time per “global step”</li>
-</ul>
-</section>
-<section id="a-training-library-mixed-precision" class="slide level2">
-<h2>A Training Library: Mixed Precision</h2>
-<ul>
-<li>This may be a bit different than your “normal” idea of mixed precision.</li>
-<li>We do <strong>not</strong> convert the model weights to BF16/FP16</li>
-<li>Instead we <strong>wrap the forward pass</strong> with <code>autocast</code> to convert the gradients automatically</li>
-<li>This preserves the original precision of the weights, which leads to stable training and better fine-tuning later on.</li>
-<li><strong>If you use <code>.bf16()</code> weights, you are STUCK in bf16 perminantly</strong></li>
-</ul>
-</section>
-<section id="a-training-library-mixed-precision-1" class="slide level2">
-<h2>A Training Library: Mixed Precision</h2>
-<ul>
-<li>Let’s tie that back up to the model estimator with neat tools like NVIDIA’s TransformerEngine</li>
-</ul>
-<div style="font-size: 60%;">
-<table style="width:100%;">
-<colgroup>
-<col style="width: 14%">
-<col style="width: 14%">
-<col style="width: 14%">
-<col style="width: 14%">
-<col style="width: 14%">
-<col style="width: 14%">
-<col style="width: 14%">
-</colgroup>
-<thead>
-<tr class="header">
-<th>Optimization Level</th>
-<th>Computation (GEMM)</th>
-<th>Comm</th>
-<th>Weight</th>
-<th>Master Weight</th>
-<th>Weight Gradient</th>
-<th>Optimizer States</th>
-</tr>
-</thead>
-<tbody>
-<tr class="odd">
-<td>FP16 AMP</td>
-<td>FP16</td>
-<td>FP32</td>
-<td>FP32</td>
-<td>N/A</td>
-<td>FP32</td>
-<td>FP32+FP32</td>
-</tr>
-<tr class="even">
-<td>Nvidia TE</td>
-<td>FP8</td>
-<td>FP32</td>
-<td>FP32</td>
-<td>N/A</td>
-<td>FP32</td>
-<td>FP32+FP32</td>
-</tr>
-<tr class="odd">
-<td>MS-AMP O1</td>
-<td>FP8</td>
-<td>FP8</td>
-<td>FP16</td>
-<td>N/A</td>
-<td>FP8</td>
-<td>FP32+FP32</td>
-</tr>
-<tr class="even">
-<td>MS-AMP O2</td>
-<td>FP8</td>
-<td>FP8</td>
-<td>FP16</td>
-<td>N/A</td>
-<td>FP8</td>
-<td>FP8+FP16</td>
-</tr>
-<tr class="odd">
-<td>MS-AMP O3</td>
-<td>FP8</td>
-<td>FP8</td>
-<td>FP8</td>
-<td>FP16</td>
-<td>FP8</td>
-<td>FP8+FP16</td>
-</tr>
-</tbody>
-</table>
-</div>
-<aside class="notes">
-<p>What is actually happening: * Linear Layers and other certain compatible layers are wrapped in a special version that allows for FP8 computation * The general forward pass is wrapped around BF16 * This means that the most memory saved is done during the gradients of the model, <em>not</em> the model itself. * With tools like <code>MS-AMP</code> we can convert more chunks into lower precision, but again like before stable training occurs when the models weights are in full precision and the backprop happens in full precision too.</p>
-<style type="text/css">
-        span.MJX_Assistive_MathML {
-          position:absolute!important;
-          clip: rect(1px, 1px, 1px, 1px);
-          padding: 1px 0 0 0!important;
-          border: 0!important;
-          height: 1px!important;
-          width: 1px!important;
-          overflow: hidden!important;
-          display:block!important;
-      }</style></aside>
-</section>
-<section id="deepspeed-vs-fully-sharded-data-parallelism" class="slide level2">
-<h2>DeepSpeed vs Fully Sharded Data Parallelism</h2>
-<ul>
-<li>Extremely similar, however mostly used different naming conventions for items and slight tweaks in the implementation</li>
-</ul>
-<div style="font-size: 50%;">
-<table style="width:100%;">
-<colgroup>
-<col style="width: 16%">
-<col style="width: 16%">
-<col style="width: 16%">
-<col style="width: 16%">
-<col style="width: 16%">
-<col style="width: 16%">
-</colgroup>
-<thead>
-<tr class="header">
-<th>Framework</th>
-<th>Model Loading (<code>torch_dtype</code>)</th>
-<th>Mixed Precision</th>
-<th>Preparation (Local)</th>
-<th>Training</th>
-<th>Optimizer (Local)</th>
-</tr>
-</thead>
-<tbody>
-<tr class="odd">
-<td>FSDP</td>
-<td>bf16</td>
-<td>default (none)</td>
-<td>bf16</td>
-<td>bf16</td>
-<td>bf16</td>
-</tr>
-<tr class="even">
-<td>FSDP</td>
-<td>bf16</td>
-<td>bf16</td>
-<td>fp32</td>
-<td>bf16</td>
-<td>fp32</td>
-</tr>
-<tr class="odd">
-<td>DeepSpeed</td>
-<td>bf16</td>
-<td>bf16</td>
-<td>fp32</td>
-<td>bf16</td>
-<td>fp32</td>
-</tr>
-</tbody>
-</table>
-</div>
-<p>To learn more, check out the <a href="https://huggingface.co/docs/accelerate/concept_guides/fsdp_and_deepspeed">documentation</a> or join my office hours</p>
-</section>
-<section id="key-takeaways" class="slide level2">
-<h2>Key Takeaways:</h2>
-<ul>
-<li>You can scale out training with <code>accelerate</code>, FSDP, and DeepSpeed across multiple GPUs to train bigger models</li>
-<li>Techniques like <code>FP8</code> can help speed up training some and reduce computational overhead</li>
-<li>Comes at a cost of end-precision and locking model weights for futher fine-tunes if not careful</li>
-</ul>
-</section>
-<section id="some-handy-resources" class="slide level2">
-<h2>Some Handy Resources</h2>
-<ul>
-<li><a href="https://hf.co/docs/accelerate">🤗 Accelerate documentation</a></li>
-<li><a href="https://huggingface.co/docs/accelerate/basic_tutorials/launch">Launching distributed code</a></li>
-<li><a href="https://huggingface.co/docs/accelerate/basic_tutorials/notebook">Distributed code and Jupyter Notebooks</a></li>
-<li><a href="https://huggingface.co/docs/accelerate/basic_tutorials/migration">Migrating to 🤗 Accelerate easily</a></li>
-<li><a href="https://huggingface.co/docs/accelerate/usage_guides/big_modeling">Big Model Inference tutorial</a></li>
-<li><a href="https://huggingface.co/docs/accelerate/usage_guides/deepspeed">DeepSpeed and 🤗 Accelerate</a></li>
-<li><a href="https://huggingface.co/docs/accelerate/usage_guides/fsdp">Fully Sharded Data Parallelism and 🤗 Accelerate</a></li>
-<li><a href="https://huggingface.co/docs/accelerate/concept_guides/fsdp_and_deepspeed">FSDP vs DeepSpeed In-Depth</a></li>
-</ul>
-<div class="footer footer-default">
-</div>
-</section></section>
-    </div>
-  </div>
-  <script>window.backupDefine = window.define; window.define = undefined;</script>
-  <script src="llm_conf_files/libs/revealjs/dist/reveal.js"></script>
-  <!-- reveal.js plugins -->
-  <script src="llm_conf_files/libs/revealjs/plugin/quarto-line-highlight/line-highlight.js"></script>
-  <script src="llm_conf_files/libs/revealjs/plugin/pdf-export/pdfexport.js"></script>
-  <script src="llm_conf_files/libs/revealjs/plugin/reveal-menu/menu.js"></script>
-  <script src="llm_conf_files/libs/revealjs/plugin/reveal-menu/quarto-menu.js"></script>
-  <script src="llm_conf_files/libs/revealjs/plugin/quarto-support/support.js"></script>
-  <script src="llm_conf_files/libs/revealjs/plugin/notes/notes.js"></script>
-  <script src="llm_conf_files/libs/revealjs/plugin/search/search.js"></script>
-  <script src="llm_conf_files/libs/revealjs/plugin/zoom/zoom.js"></script>
-  <script src="llm_conf_files/libs/revealjs/plugin/math/math.js"></script>
-  <script>window.define = window.backupDefine; window.backupDefine = undefined;</script>
-  <script>
-      // Full list of configuration options available at:
-      // https://revealjs.com/config/
-      Reveal.initialize({
-'controlsAuto': true,
-'previewLinksAuto': false,
-'pdfSeparateFragments': false,
-'autoAnimateEasing': "ease",
-'autoAnimateDuration': 1,
-'autoAnimateUnmatched': true,
-'menu': {"side":"left","useTextContentForMissingTitles":true,"markers":false,"loadIcons":false,"custom":[{"title":"Tools","icon":"<i class=\"fas fa-gear\"></i>","content":"<ul class=\"slide-menu-items\">\n<li class=\"slide-tool-item active\" data-item=\"0\"><a href=\"#\" onclick=\"RevealMenuToolHandlers.fullscreen(event)\"><kbd>f</kbd> Fullscreen</a></li>\n<li class=\"slide-tool-item\" data-item=\"1\"><a href=\"#\" onclick=\"RevealMenuToolHandlers.speakerMode(event)\"><kbd>s</kbd> Speaker View</a></li>\n<li class=\"slide-tool-item\" data-item=\"2\"><a href=\"#\" onclick=\"RevealMenuToolHandlers.overview(event)\"><kbd>o</kbd> Slide Overview</a></li>\n<li class=\"slide-tool-item\" data-item=\"3\"><a href=\"#\" onclick=\"RevealMenuToolHandlers.togglePdfExport(event)\"><kbd>e</kbd> PDF Export Mode</a></li>\n<li class=\"slide-tool-item\" data-item=\"4\"><a href=\"#\" onclick=\"RevealMenuToolHandlers.keyboardHelp(event)\"><kbd>?</kbd> Keyboard Help</a></li>\n</ul>"}],"openButton":true},
-'smaller': false,
-        // Display controls in the bottom right corner
-        controls: false,
-        // Help the user learn the controls by providing hints, for example by
-        // bouncing the down arrow when they first encounter a vertical slide
-        controlsTutorial: false,
-        // Determines where controls appear, "edges" or "bottom-right"
-        controlsLayout: 'edges',
-        // Visibility rule for backwards navigation arrows; "faded", "hidden"
-        // or "visible"
-        controlsBackArrows: 'faded',
-        // Display a presentation progress bar
-        progress: true,
-        // Display the page number of the current slide
-        slideNumber: false,
-        // 'all', 'print', or 'speaker'
-        showSlideNumber: 'all',
-        // Add the current slide number to the URL hash so that reloading the
-        // page/copying the URL will return you to the same slide
-        hash: true,
-        // Start with 1 for the hash rather than 0
-        hashOneBasedIndex: false,
-        // Flags if we should monitor the hash and change slides accordingly
-        respondToHashChanges: true,
-        // Push each slide change to the browser history
-        history: true,
-        // Enable keyboard shortcuts for navigation
-        keyboard: true,
-        // Enable the slide overview mode
-        overview: true,
-        // Disables the default reveal.js slide layout (scaling and centering)
-        // so that you can use custom CSS layout
-        disableLayout: false,
-        // Vertical centering of slides
-        center: false,
-        // Enables touch navigation on devices with touch input
-        touch: true,
-        // Loop the presentation
-        loop: false,
-        // Change the presentation direction to be RTL
-        rtl: false,
-        // see https://revealjs.com/vertical-slides/#navigation-mode
-        navigationMode: 'linear',
-        // Randomizes the order of slides each time the presentation loads
-        shuffle: false,
-        // Turns fragments on and off globally
-        fragments: true,
-        // Flags whether to include the current fragment in the URL,
-        // so that reloading brings you to the same fragment position
-        fragmentInURL: false,
-        // Flags if the presentation is running in an embedded mode,
-        // i.e. contained within a limited portion of the screen
-        embedded: false,
-        // Flags if we should show a help overlay when the questionmark
-        // key is pressed
-        help: true,
-        // Flags if it should be possible to pause the presentation (blackout)
-        pause: true,
-        // Flags if speaker notes should be visible to all viewers
-        showNotes: false,
-        // Global override for autoplaying embedded media (null/true/false)
-        autoPlayMedia: null,
-        // Global override for preloading lazy-loaded iframes (null/true/false)
-        preloadIframes: null,
-        // Number of milliseconds between automatically proceeding to the
-        // next slide, disabled when set to 0, this value can be overwritten
-        // by using a data-autoslide attribute on your slides
-        autoSlide: 0,
-        // Stop auto-sliding after user input
-        autoSlideStoppable: true,
-        // Use this method for navigation when auto-sliding
-        autoSlideMethod: null,
-        // Specify the average time in seconds that you think you will spend
-        // presenting each slide. This is used to show a pacing timer in the
-        // speaker view
-        defaultTiming: null,
-        // Enable slide navigation via mouse wheel
-        mouseWheel: false,
-        // The display mode that will be used to show slides
-        display: 'block',
-        // Hide cursor if inactive
-        hideInactiveCursor: true,
-        // Time before the cursor is hidden (in ms)
-        hideCursorTime: 5000,
-        // Opens links in an iframe preview overlay
-        previewLinks: false,
-        // Transition style (none/fade/slide/convex/concave/zoom)
-        transition: 'none',
-        // Transition speed (default/fast/slow)
-        transitionSpeed: 'default',
-        // Transition style for full page slide backgrounds
-        // (none/fade/slide/convex/concave/zoom)
-        backgroundTransition: 'none',
-        // Number of slides away from the current that are visible
-        viewDistance: 3,
-        // Number of slides away from the current that are visible on mobile
-        // devices. It is advisable to set this to a lower number than
-        // viewDistance in order to save resources.
-        mobileViewDistance: 2,
-        // The "normal" size of the presentation, aspect ratio will be preserved
-        // when the presentation is scaled to fit different resolutions. Can be
-        // specified using percentage units.
-        width: 1050,
-        height: 700,
-        // Factor of the display size that should remain empty around the content
-        margin: 0.1,
-        math: {
-          mathjax: 'https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js',
-          config: 'TeX-AMS_HTML-full',
-          tex2jax: {
-            inlineMath: [['\\(','\\)']],
-            displayMath: [['\\[','\\]']],
-            balanceBraces: true,
-            processEscapes: false,
-            processRefs: true,
-            processEnvironments: true,
-            preview: 'TeX',
-            skipTags: ['script','noscript','style','textarea','pre','code'],
-            ignoreClass: 'tex2jax_ignore',
-            processClass: 'tex2jax_process'
-          },
-        },
-        // reveal.js plugins
-        plugins: [QuartoLineHighlight, PdfExport, RevealMenu, QuartoSupport,
-          RevealMath,
-          RevealNotes,
-          RevealSearch,
-          RevealZoom
-        ]
-      });
-    </script>
-    <script id="quarto-html-after-body" type="application/javascript">
-    window.document.addEventListener("DOMContentLoaded", function (event) {
-      const toggleBodyColorMode = (bsSheetEl) => {
-        const mode = bsSheetEl.getAttribute("data-mode");
-        const bodyEl = window.document.querySelector("body");
-        if (mode === "dark") {
-          bodyEl.classList.add("quarto-dark");
-          bodyEl.classList.remove("quarto-light");
-        } else {
-          bodyEl.classList.add("quarto-light");
-          bodyEl.classList.remove("quarto-dark");
-        }
-      }
-      const toggleBodyColorPrimary = () => {
-        const bsSheetEl = window.document.querySelector("link#quarto-bootstrap");
-        if (bsSheetEl) {
-          toggleBodyColorMode(bsSheetEl);
-        }
-      }
-      toggleBodyColorPrimary();
-      const tabsets =  window.document.querySelectorAll(".panel-tabset-tabby")
-      tabsets.forEach(function(tabset) {
-        const tabby = new Tabby('#' + tabset.id);
-      });
-      const isCodeAnnotation = (el) => {
-        for (const clz of el.classList) {
-          if (clz.startsWith('code-annotation-')) {
-            return true;
-          }
-        }
-        return false;
-      }
-      const clipboard = new window.ClipboardJS('.code-copy-button', {
-        text: function(trigger) {
-          const codeEl = trigger.previousElementSibling.cloneNode(true);
-          for (const childEl of codeEl.children) {
-            if (isCodeAnnotation(childEl)) {
-              childEl.remove();
-            }
-          }
-          return codeEl.innerText;
-        }
-      });
-      clipboard.on('success', function(e) {
-        // button target
-        const button = e.trigger;
-        // don't keep focus
-        button.blur();
-        // flash "checked"
-        button.classList.add('code-copy-button-checked');
-        var currentTitle = button.getAttribute("title");
-        button.setAttribute("title", "Copied!");
-        let tooltip;
-        if (window.bootstrap) {
-          button.setAttribute("data-bs-toggle", "tooltip");
-          button.setAttribute("data-bs-placement", "left");
-          button.setAttribute("data-bs-title", "Copied!");
-          tooltip = new bootstrap.Tooltip(button,
-            { trigger: "manual",
-              customClass: "code-copy-button-tooltip",
-              offset: [0, -8]});
-          tooltip.show();
-        }
-        setTimeout(function() {
-          if (tooltip) {
-            tooltip.hide();
-            button.removeAttribute("data-bs-title");
-            button.removeAttribute("data-bs-toggle");
-            button.removeAttribute("data-bs-placement");
-          }
-          button.setAttribute("title", currentTitle);
-          button.classList.remove('code-copy-button-checked');
-        }, 1000);
-        // clear code selection
-        e.clearSelection();
-      });
-      function tippyHover(el, contentFn, onTriggerFn, onUntriggerFn) {
-        const config = {
-          allowHTML: true,
-          maxWidth: 500,
-          delay: 100,
-          arrow: false,
-          appendTo: function(el) {
-              return el.closest('section.slide') || el.parentElement;
-          },
-          interactive: true,
-          interactiveBorder: 10,
-          theme: 'light-border',
-          placement: 'bottom-start',
-        };
-        if (contentFn) {
-          config.content = contentFn;
-        }
-        if (onTriggerFn) {
-          config.onTrigger = onTriggerFn;
-        }
-        if (onUntriggerFn) {
-          config.onUntrigger = onUntriggerFn;
-        }
-          config['offset'] = [0,0];
-          config['maxWidth'] = 700;
-        window.tippy(el, config);
-      }
-      const noterefs = window.document.querySelectorAll('a[role="doc-noteref"]');
-      for (var i=0; i<noterefs.length; i++) {
-        const ref = noterefs[i];
-        tippyHover(ref, function() {
-          // use id or data attribute instead here
-          let href = ref.getAttribute('data-footnote-href') || ref.getAttribute('href');
-          try { href = new URL(href).hash; } catch {}
-          const id = href.replace(/^#\/?/, "");
-          const note = window.document.getElementById(id);
-          return note.innerHTML;
-        });
-      }
-      const findCites = (el) => {
-        const parentEl = el.parentElement;
-        if (parentEl) {
-          const cites = parentEl.dataset.cites;
-          if (cites) {
-            return {
-              el,
-              cites: cites.split(' ')
-            };
-          } else {
-            return findCites(el.parentElement)
-          }
-        } else {
-          return undefined;
-        }
-      };
-      var bibliorefs = window.document.querySelectorAll('a[role="doc-biblioref"]');
-      for (var i=0; i<bibliorefs.length; i++) {
-        const ref = bibliorefs[i];
-        const citeInfo = findCites(ref);
-        if (citeInfo) {
-          tippyHover(citeInfo.el, function() {
-            var popup = window.document.createElement('div');
-            citeInfo.cites.forEach(function(cite) {
-              var citeDiv = window.document.createElement('div');
-              citeDiv.classList.add('hanging-indent');
-              citeDiv.classList.add('csl-entry');
-              var biblioDiv = window.document.getElementById('ref-' + cite);
-              if (biblioDiv) {
-                citeDiv.innerHTML = biblioDiv.innerHTML;
-              }
-              popup.appendChild(citeDiv);
-            });
-            return popup.innerHTML;
-          });
-        }
-      }
-    });
-    </script>
-</body></html>