Spaces:

SUSTech
/

tlem

Running

File size: 46,490 Bytes

507319c
72ccff7

<!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"><head>

<meta charset="utf-8">
<meta name="generator" content="quarto-1.3.450">

<meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes">


<title>Transparent LLMs Evaluation Metrics</title>
<style>
code{white-space: pre-wrap;}
span.smallcaps{font-variant: small-caps;}
div.columns{display: flex; gap: min(4vw, 1.5em);}
div.column{flex: auto; overflow-x: auto;}
div.hanging-indent{margin-left: 1.5em; text-indent: -1.5em;}
ul.task-list{list-style: none;}
ul.task-list li input[type="checkbox"] {
  width: 0.8em;
  margin: 0 0.8em 0.2em -1em; /* quarto-specific, see https://github.com/quarto-dev/quarto-cli/issues/4556 */ 
  vertical-align: middle;
}
/* CSS for syntax highlighting */
pre > code.sourceCode { white-space: pre; position: relative; }
pre > code.sourceCode > span { display: inline-block; line-height: 1.25; }
pre > code.sourceCode > span:empty { height: 1.2em; }
.sourceCode { overflow: visible; }
code.sourceCode > span { color: inherit; text-decoration: inherit; }
div.sourceCode { margin: 1em 0; }
pre.sourceCode { margin: 0; }
@media screen {
div.sourceCode { overflow: auto; }
}
@media print {
pre > code.sourceCode { white-space: pre-wrap; }
pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; }
}
pre.numberSource code
  { counter-reset: source-line 0; }
pre.numberSource code > span
  { position: relative; left: -4em; counter-increment: source-line; }
pre.numberSource code > span > a:first-child::before
  { content: counter(source-line);
    position: relative; left: -1em; text-align: right; vertical-align: baseline;
    border: none; display: inline-block;
    -webkit-touch-callout: none; -webkit-user-select: none;
    -khtml-user-select: none; -moz-user-select: none;
    -ms-user-select: none; user-select: none;
    padding: 0 4px; width: 4em;
  }
pre.numberSource { margin-left: 3em;  padding-left: 4px; }
div.sourceCode
  {   }
@media screen {
pre > code.sourceCode > span > a:first-child::before { text-decoration: underline; }
}
</style>


<script src="README_files/libs/clipboard/clipboard.min.js"></script>
<script src="README_files/libs/quarto-html/quarto.js"></script>
<script src="README_files/libs/quarto-html/popper.min.js"></script>
<script src="README_files/libs/quarto-html/tippy.umd.min.js"></script>
<script src="README_files/libs/quarto-html/anchor.min.js"></script>
<link href="README_files/libs/quarto-html/tippy.css" rel="stylesheet">
<link href="README_files/libs/quarto-html/quarto-syntax-highlighting.css" rel="stylesheet" id="quarto-text-highlighting-styles">
<script src="README_files/libs/bootstrap/bootstrap.min.js"></script>
<link href="README_files/libs/bootstrap/bootstrap-icons.css" rel="stylesheet">
<link href="README_files/libs/bootstrap/bootstrap.min.css" rel="stylesheet" id="quarto-bootstrap" data-mode="light">
<script src="https://cdnjs.cloudflare.com/ajax/libs/require.js/2.3.6/require.min.js" integrity="sha512-c3Nl8+7g4LMSTdrm621y7kf9v3SDPnhxLNhcjFJbKECVnmZHTdo+IRO05sNLTH/D3vA6u1X32ehoLC7WFVdheg==" crossorigin="anonymous"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.5.1/jquery.min.js" integrity="sha512-bLT0Qm9VnAYZDflyKcBaQ2gg0hSYNQrJ8RilYldYQ1FxQYoCLtUjuuRuZo+fjqhx/qtq/1itJ0C2ejDxltZVFg==" crossorigin="anonymous"></script>
<script type="application/javascript">define('jquery', [],function() {return window.jQuery;})</script>
<script src="https://unpkg.com/@jupyter-widgets/html-manager@*/dist/embed-amd.js" crossorigin="anonymous"></script>


</head>

<body class="fullcontent">

<div id="quarto-content" class="page-columns page-rows-contents page-layout-article">

<main class="content" id="quarto-document-content">

<header id="title-block-header" class="quarto-title-block default">
<div class="quarto-title">
<h1 class="title">Transparent LLMs Evaluation Metrics</h1>
</div>



<div class="quarto-title-meta">

    
  
    
  </div>
  

</header>

<section id="introducing-tlem-the-future-of-language-model-evaluation" class="level1">
<h1>Introducing TLEM: The Future of Language Model Evaluation 🌐✨</h1>
<p>In an era where the globe is racing to train and launch increasingly sophisticated language models, there’s a pressing need for a unified standard to gauge their effectiveness. That’s where TLEM, or Transparent LLMs Evaluation Metrics – a nod to the French phrase <strong>tout le monde</strong> (everyone) – steps in. TLEM is not just another framework; it’s a revolution in the way we assess language models. Its name embodies our commitment to transparency and decentralization in the evaluation of large language models.</p>
<section id="why-tlem-heres-why" class="level2">
<h2 class="anchored" data-anchor-id="why-tlem-heres-why">🌟 Why TLEM? Here’s Why!</h2>
<ul>
<li><p>Universal Standardization: With the international community eager to develop and unveil large language models, TLEM offers a much-needed standardized criterion to differentiate the good from the great.</p></li>
<li><p>Developer &amp; User-Friendly: Existing open-source implementations often suffer from deep encapsulation, posing challenges for both developers and users. TLEM changes the game by being incredibly user-friendly and accessible.</p></li>
<li><p>Addressing the Self-Evaluation Bias: A common hurdle in the current landscape is the tendency of models to self-evaluate, leading to a reliance on their own assessments and only referencing open-source evaluations. This has resulted in redundant efforts and reduced reproducibility within the open-source community. TLEM tackles this issue head-on.</p></li>
<li><p>Designed for Ease and Decentralization: TLEM stands out with its extreme ease of use. Forget the hassle of manually pulling repositories and installing – TLEM simplifies it all. Moreover, its metrics are designed to be decentralized, empowering users to extend and contribute new evaluation metrics, fostering a community-driven approach.</p></li>
</ul>
</section>
<section id="join-the-tlem-movement" class="level2">
<h2 class="anchored" data-anchor-id="join-the-tlem-movement">🚀 Join the TLEM Movement!</h2>
<p>TLEM is more than a framework; it’s a movement towards a more transparent, decentralized, and community-driven future in language model evaluation. Be a part of this exciting journey. Dive into the world of TLEM, where every contribution counts, and every evaluation brings us closer to excellence in language model development.</p>
<p>Let’s shape the future together with TLEM! 🌟💻🔍</p>
</section>
</section>
<section id="usage" class="level1">
<h1>Usage</h1>
<section id="start-evaluating-your-model-in-3-line" class="level2">
<h2 class="anchored" data-anchor-id="start-evaluating-your-model-in-3-line">Start evaluating your model in 3 line</h2>
<p>You can start evaluating your model with TLEM in 3 lines, tlem is designed to work without installing.</p>
<div class="cell" data-executetime="{&quot;end_time&quot;:&quot;2023-12-02T17:55:47.835488Z&quot;,&quot;start_time&quot;:&quot;2023-12-02T17:55:44.915535Z&quot;}" data-execution_count="2">
<div class="sourceCode cell-code" id="cb1"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb1-1"><a href="#cb1-1" aria-hidden="true" tabindex="-1"></a>suite <span class="op">=</span> evaluate.EvaluationSuite.load(<span class="st">"SUSTech/tlem"</span>, download_mode<span class="op">=</span><span class="st">"force_redownload"</span>)</span>
<span id="cb1-2"><a href="#cb1-2" aria-hidden="true" tabindex="-1"></a>suite.load(<span class="st">"gsm8k"</span>)  <span class="co"># You can check the available datasets by suite.supported_datasets</span></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
</div>
<div class="cell" data-executetime="{&quot;end_time&quot;:&quot;2023-12-02T17:55:55.391135Z&quot;,&quot;start_time&quot;:&quot;2023-12-02T17:55:55.266737Z&quot;}" data-execution_count="4">
<div class="sourceCode cell-code" id="cb2"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb2-1"><a href="#cb2-1" aria-hidden="true" tabindex="-1"></a>suite.run(pipe <span class="op">:=</span> <span class="kw">lambda</span> x: x)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
<div class="cell-output cell-output-display">
<script type="application/vnd.jupyter.widget-view+json">
{"model_id":"bf385f9d1496433daadde6b638f81f15","version_major":2,"version_minor":0,"quarto_mimetype":"application/vnd.jupyter.widget-view+json"}
</script>
</div>
<div class="cell-output cell-output-display" data-execution_count="4">
<pre><code>{'gsm8k': 0.022744503411675512}</code></pre>
</div>
</div>
<p>The lambda function indicate a model pipeline which takes a list of string as input and return a list of string as output. You can use any model you want, as long as it can be wrapped in this way. We use the most popular VLLM and Openai API as an example:</p>
<div class="cell" data-executetime="{&quot;end_time&quot;:&quot;2023-12-02T17:55:55.433370Z&quot;,&quot;start_time&quot;:&quot;2023-12-02T17:55:55.392566Z&quot;}" data-execution_count="5">
<div class="sourceCode cell-code" id="cb4"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb4-1"><a href="#cb4-1" aria-hidden="true" tabindex="-1"></a>session <span class="op">=</span> aiohttp.ClientSession(timeout<span class="op">=</span>aiohttp.ClientTimeout(total<span class="op">=</span><span class="dv">60</span> <span class="op">*</span> <span class="dv">60</span> <span class="op">*</span> <span class="dv">24</span> <span class="op">*</span> <span class="dv">7</span>))</span>
<span id="cb4-2"><a href="#cb4-2" aria-hidden="true" tabindex="-1"></a>url <span class="op">=</span> <span class="st">"xxx"</span></span>
<span id="cb4-3"><a href="#cb4-3" aria-hidden="true" tabindex="-1"></a>client <span class="op">=</span> AsyncOpenAI(<span class="op">**</span>{<span class="st">"base_url"</span>: <span class="ss">f"http://</span><span class="sc">{</span>url<span class="sc">}</span><span class="ss">/v1/"</span>, <span class="st">"api_key"</span>: <span class="st">"EMPTY"</span>})</span>
<span id="cb4-4"><a href="#cb4-4" aria-hidden="true" tabindex="-1"></a></span>
<span id="cb4-5"><a href="#cb4-5" aria-hidden="true" tabindex="-1"></a></span>
<span id="cb4-6"><a href="#cb4-6" aria-hidden="true" tabindex="-1"></a><span class="at">@suite.utils.async_pipe</span></span>
<span id="cb4-7"><a href="#cb4-7" aria-hidden="true" tabindex="-1"></a><span class="cf">async</span> <span class="kw">def</span> chatgpt(msg):</span>
<span id="cb4-8"><a href="#cb4-8" aria-hidden="true" tabindex="-1"></a>    <span class="bu">input</span> <span class="op">=</span> <span class="ss">f"### Human: </span><span class="sc">{</span>msg<span class="sc">}</span><span class="ch">\n\n</span><span class="ss">### Assistant: "</span></span>
<span id="cb4-9"><a href="#cb4-9" aria-hidden="true" tabindex="-1"></a>    <span class="cf">try</span>:</span>
<span id="cb4-10"><a href="#cb4-10" aria-hidden="true" tabindex="-1"></a>        resp <span class="op">=</span> <span class="cf">await</span> client.completions.create(</span>
<span id="cb4-11"><a href="#cb4-11" aria-hidden="true" tabindex="-1"></a>            model<span class="op">=</span><span class="st">"gpt-3.5-turbo"</span>,</span>
<span id="cb4-12"><a href="#cb4-12" aria-hidden="true" tabindex="-1"></a>            max_tokens<span class="op">=</span><span class="va">None</span>,</span>
<span id="cb4-13"><a href="#cb4-13" aria-hidden="true" tabindex="-1"></a>            prompt<span class="op">=</span><span class="bu">input</span>,</span>
<span id="cb4-14"><a href="#cb4-14" aria-hidden="true" tabindex="-1"></a>            temperature<span class="op">=</span><span class="dv">0</span>,</span>
<span id="cb4-15"><a href="#cb4-15" aria-hidden="true" tabindex="-1"></a>        )</span>
<span id="cb4-16"><a href="#cb4-16" aria-hidden="true" tabindex="-1"></a>        <span class="cf">return</span> resp.choices[<span class="dv">0</span>].text</span>
<span id="cb4-17"><a href="#cb4-17" aria-hidden="true" tabindex="-1"></a>    <span class="cf">except</span> <span class="pp">Exception</span> <span class="im">as</span> e:</span>
<span id="cb4-18"><a href="#cb4-18" aria-hidden="true" tabindex="-1"></a>        <span class="cf">return</span> <span class="st">"OpenAI Error"</span></span>
<span id="cb4-19"><a href="#cb4-19" aria-hidden="true" tabindex="-1"></a></span>
<span id="cb4-20"><a href="#cb4-20" aria-hidden="true" tabindex="-1"></a></span>
<span id="cb4-21"><a href="#cb4-21" aria-hidden="true" tabindex="-1"></a><span class="at">@suite.utils.async_pipe</span></span>
<span id="cb4-22"><a href="#cb4-22" aria-hidden="true" tabindex="-1"></a><span class="cf">async</span> <span class="kw">def</span> vllm(msg):</span>
<span id="cb4-23"><a href="#cb4-23" aria-hidden="true" tabindex="-1"></a>    <span class="bu">input</span> <span class="op">=</span> <span class="ss">f"### Human: </span><span class="sc">{</span>msg<span class="sc">}</span><span class="ch">\n\n</span><span class="ss">### Assistant: "</span></span>
<span id="cb4-24"><a href="#cb4-24" aria-hidden="true" tabindex="-1"></a>    data <span class="op">=</span> {</span>
<span id="cb4-25"><a href="#cb4-25" aria-hidden="true" tabindex="-1"></a>        <span class="st">"prompt"</span>: <span class="bu">input</span>,</span>
<span id="cb4-26"><a href="#cb4-26" aria-hidden="true" tabindex="-1"></a>        <span class="st">"max_tokens"</span>: <span class="dv">4096</span>,</span>
<span id="cb4-27"><a href="#cb4-27" aria-hidden="true" tabindex="-1"></a>        <span class="st">"n"</span>: <span class="dv">1</span>,</span>
<span id="cb4-28"><a href="#cb4-28" aria-hidden="true" tabindex="-1"></a>        <span class="st">"temperature"</span>: <span class="dv">0</span>,</span>
<span id="cb4-29"><a href="#cb4-29" aria-hidden="true" tabindex="-1"></a>    }</span>
<span id="cb4-30"><a href="#cb4-30" aria-hidden="true" tabindex="-1"></a></span>
<span id="cb4-31"><a href="#cb4-31" aria-hidden="true" tabindex="-1"></a>    <span class="cf">try</span>:</span>
<span id="cb4-32"><a href="#cb4-32" aria-hidden="true" tabindex="-1"></a>        <span class="cf">async</span> <span class="cf">with</span> session.post(<span class="ss">f"http://</span><span class="sc">{</span>url<span class="sc">}</span><span class="ss">/generate"</span>, json<span class="op">=</span>data) <span class="im">as</span> response:</span>
<span id="cb4-33"><a href="#cb4-33" aria-hidden="true" tabindex="-1"></a>            response_json <span class="op">=</span> <span class="cf">await</span> response.json()</span>
<span id="cb4-34"><a href="#cb4-34" aria-hidden="true" tabindex="-1"></a>            <span class="cf">return</span> response_json[<span class="st">"text"</span>][<span class="dv">0</span>][<span class="bu">len</span>(<span class="bu">input</span>) :]</span>
<span id="cb4-35"><a href="#cb4-35" aria-hidden="true" tabindex="-1"></a>    <span class="cf">except</span> <span class="pp">Exception</span> <span class="im">as</span> e:</span>
<span id="cb4-36"><a href="#cb4-36" aria-hidden="true" tabindex="-1"></a>        <span class="cf">return</span> <span class="st">"Vllm Error"</span></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
</div>
</section>
</section>
<section id="hackable" class="level1">
<h1>Hackable</h1>
</section>
<section id="section" class="level1">
<h1></h1>
<p>TLEM is designed to be hackable. Every tlem is a <code>task</code> in the <code>suite</code>, <code>suite.run</code> just run all the tasks in the suite. For each task, you can check it’s input, label and output by</p>
<div class="cell" data-executetime="{&quot;end_time&quot;:&quot;2023-12-02T17:55:55.443078Z&quot;,&quot;start_time&quot;:&quot;2023-12-02T17:55:55.434617Z&quot;}" data-execution_count="6">
<div class="sourceCode cell-code" id="cb5"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb5-1"><a href="#cb5-1" aria-hidden="true" tabindex="-1"></a>task <span class="op">=</span> suite[<span class="dv">0</span>]</span>
<span id="cb5-2"><a href="#cb5-2" aria-hidden="true" tabindex="-1"></a><span class="co"># task.outputs is avaliable after suite.run or task.run</span></span>
<span id="cb5-3"><a href="#cb5-3" aria-hidden="true" tabindex="-1"></a>pd.DataFrame({<span class="st">"input"</span>: task.samples, <span class="st">"label"</span>: task.labels, <span class="st">"output"</span>: task.outputs})</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
<div class="cell-output cell-output-display" data-execution_count="6">
<div>


<table class="dataframe table table-sm table-striped small" data-quarto-postprocess="true" data-border="1">
<thead>
<tr class="header">
<th data-quarto-table-cell-role="th"></th>
<th data-quarto-table-cell-role="th">input</th>
<th data-quarto-table-cell-role="th">label</th>
<th data-quarto-table-cell-role="th">output</th>
</tr>
</thead>
<tbody>
<tr class="odd">
<td data-quarto-table-cell-role="th">0</td>
<td>Janet’s ducks lay 16 eggs per day. She eats th...</td>
<td>Janet sells 16 - 3 - 4 = &lt;&lt;16-3-4=9&gt;&gt;9 duck eg...</td>
<td>Janet’s ducks lay 16 eggs per day. She eats th...</td>
</tr>
<tr class="even">
<td data-quarto-table-cell-role="th">1</td>
<td>A robe takes 2 bolts of blue fiber and half th...</td>
<td>It takes 2/2=&lt;&lt;2/2=1&gt;&gt;1 bolt of white fiber\nS...</td>
<td>A robe takes 2 bolts of blue fiber and half th...</td>
</tr>
<tr class="odd">
<td data-quarto-table-cell-role="th">2</td>
<td>Josh decides to try flipping a house. He buys...</td>
<td>The cost of the house and repairs came out to ...</td>
<td>Josh decides to try flipping a house. He buys...</td>
</tr>
<tr class="even">
<td data-quarto-table-cell-role="th">3</td>
<td>James decides to run 3 sprints 3 times a week....</td>
<td>He sprints 3*3=&lt;&lt;3*3=9&gt;&gt;9 times\nSo he runs 9*...</td>
<td>James decides to run 3 sprints 3 times a week....</td>
</tr>
<tr class="odd">
<td data-quarto-table-cell-role="th">4</td>
<td>Every day, Wendi feeds each of her chickens th...</td>
<td>If each chicken eats 3 cups of feed per day, t...</td>
<td>Every day, Wendi feeds each of her chickens th...</td>
</tr>
<tr class="even">
<td data-quarto-table-cell-role="th">...</td>
<td>...</td>
<td>...</td>
<td>...</td>
</tr>
<tr class="odd">
<td data-quarto-table-cell-role="th">1314</td>
<td>John had a son James when he was 19. James is...</td>
<td>Dora is 12-3=&lt;&lt;12-3=9&gt;&gt;9\nSo James is 9*2=&lt;&lt;9*...</td>
<td>John had a son James when he was 19. James is...</td>
</tr>
<tr class="even">
<td data-quarto-table-cell-role="th">1315</td>
<td>There are some oranges in a basket. Ana spends...</td>
<td>There are 60 minutes in an hour. Ana peels an ...</td>
<td>There are some oranges in a basket. Ana spends...</td>
</tr>
<tr class="odd">
<td data-quarto-table-cell-role="th">1316</td>
<td>Mark's car breaks down and he needs to get a n...</td>
<td>The discount on the radiator was 400*.8=$&lt;&lt;400...</td>
<td>Mark's car breaks down and he needs to get a n...</td>
</tr>
<tr class="even">
<td data-quarto-table-cell-role="th">1317</td>
<td>Farmer Brown has 20 animals on his farm, all e...</td>
<td>Let C be the number of chickens.\nThere are 20...</td>
<td>Farmer Brown has 20 animals on his farm, all e...</td>
</tr>
<tr class="odd">
<td data-quarto-table-cell-role="th">1318</td>
<td>Henry and 3 of his friends order 7 pizzas for ...</td>
<td>There are 7*8=&lt;&lt;7*8=56&gt;&gt;56 slices in total.\nT...</td>
<td>Henry and 3 of his friends order 7 pizzas for ...</td>
</tr>
</tbody>
</table>

<p>1319 rows × 3 columns</p>
</div>
</div>
</div>
<p>and you can verify our metric by</p>
<div class="cell" data-executetime="{&quot;end_time&quot;:&quot;2023-12-02T17:55:55.499005Z&quot;,&quot;start_time&quot;:&quot;2023-12-02T17:55:55.444459Z&quot;}" data-execution_count="7">
<div class="sourceCode cell-code" id="cb6"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb6-1"><a href="#cb6-1" aria-hidden="true" tabindex="-1"></a>task.metric(task.labels, task.labels)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
<div class="cell-output cell-output-display" data-execution_count="7">
<pre><code>{'gsm8k': 1.0}</code></pre>
</div>
</div>
<div class="cell" data-executetime="{&quot;end_time&quot;:&quot;2023-12-02T17:55:55.541161Z&quot;,&quot;start_time&quot;:&quot;2023-12-02T17:55:55.500362Z&quot;}" data-execution_count="8">
<div class="sourceCode cell-code" id="cb8"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb8-1"><a href="#cb8-1" aria-hidden="true" tabindex="-1"></a>task.metric(task.outputs, task.labels)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
<div class="cell-output cell-output-display" data-execution_count="8">
<pre><code>{'gsm8k': 0.022744503411675512}</code></pre>
</div>
</div>
</section>
<section id="contribution" class="level1">
<h1>Contribution</h1>
</section>
<section id="section-1" class="level1">
<h1></h1>
<p>You can easily add your own task by inheriting the <code>Task</code> class. For example, if you want to add a task to evaluate the model’s ability to generate a specific type of text, you can do it in this way:</p>
<div class="cell" data-executetime="{&quot;end_time&quot;:&quot;2023-12-02T17:55:59.542396Z&quot;,&quot;start_time&quot;:&quot;2023-12-02T17:55:55.542504Z&quot;}" data-execution_count="9">
<div class="sourceCode cell-code" id="cb10"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb10-1"><a href="#cb10-1" aria-hidden="true" tabindex="-1"></a>task <span class="op">=</span> suite.task_class(</span>
<span id="cb10-2"><a href="#cb10-2" aria-hidden="true" tabindex="-1"></a>    dataset_name<span class="op">=</span>(<span class="st">"gsm8k"</span>, <span class="st">"main"</span>),</span>
<span id="cb10-3"><a href="#cb10-3" aria-hidden="true" tabindex="-1"></a>    input_column<span class="op">=</span><span class="st">"question"</span>,</span>
<span id="cb10-4"><a href="#cb10-4" aria-hidden="true" tabindex="-1"></a>    label_column<span class="op">=</span><span class="st">"answer"</span>,</span>
<span id="cb10-5"><a href="#cb10-5" aria-hidden="true" tabindex="-1"></a>    metric_name<span class="op">=</span><span class="st">"evaluate-metric/competition_math"</span>,</span>
<span id="cb10-6"><a href="#cb10-6" aria-hidden="true" tabindex="-1"></a>)</span>
<span id="cb10-7"><a href="#cb10-7" aria-hidden="true" tabindex="-1"></a>task.run(pipe)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
<div class="cell-output cell-output-stdout">
<pre><code>&lt;class 'evaluate_modules.metrics.evaluate-metric--competition_math.b85814e0172dae97fa4bd6eff6f33caba2ff9547860acabd50222c6dee474a24.competition_math.CompetitionMathMetric'&gt;</code></pre>
</div>
<div class="cell-output cell-output-display" data-execution_count="9">
<pre><code>{'accuracy': 0.0}</code></pre>
</div>
</div>
<p>where the metric can be put in any huggingface space, TLEM is designed to be decentralized, allowing you to run evaluations on private datasets without the need to contribute your code back to TLEM. You can also define the metric locally:</p>
<div class="cell" data-executetime="{&quot;end_time&quot;:&quot;2023-12-02T17:55:59.551341Z&quot;,&quot;start_time&quot;:&quot;2023-12-02T17:55:59.544629Z&quot;}" data-execution_count="10">
<div class="sourceCode cell-code" id="cb13"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb13-1"><a href="#cb13-1" aria-hidden="true" tabindex="-1"></a><span class="kw">def</span> my_metric(responses, references):</span>
<span id="cb13-2"><a href="#cb13-2" aria-hidden="true" tabindex="-1"></a>    <span class="co"># return .99</span></span>
<span id="cb13-3"><a href="#cb13-3" aria-hidden="true" tabindex="-1"></a>    scores <span class="op">=</span> [random.choices([<span class="dv">0</span>, <span class="dv">1</span>]) <span class="cf">for</span> resp, ans <span class="kw">in</span> <span class="bu">zip</span>(responses, references)]</span>
<span id="cb13-4"><a href="#cb13-4" aria-hidden="true" tabindex="-1"></a>    <span class="cf">return</span> np.mean(scores)</span>
<span id="cb13-5"><a href="#cb13-5" aria-hidden="true" tabindex="-1"></a></span>
<span id="cb13-6"><a href="#cb13-6" aria-hidden="true" tabindex="-1"></a></span>
<span id="cb13-7"><a href="#cb13-7" aria-hidden="true" tabindex="-1"></a>task.metric <span class="op">=</span> my_metric</span>
<span id="cb13-8"><a href="#cb13-8" aria-hidden="true" tabindex="-1"></a>task.run(pipe)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
<div class="cell-output cell-output-display" data-execution_count="10">
<pre><code>0.5034116755117514</code></pre>
</div>
</div>
<section id="tlem-leaderboard" class="level2">
<h2 class="anchored" data-anchor-id="tlem-leaderboard">TLEM Leaderboard</h2>
<p>If you wish to add your model results to the TLEM leaderboard, you are required to provide the code used for running TLEM and its outcomes in your model card. We do not actively replicate your code; you are responsible for the accuracy of your results.</p>
<div class="cell" data-executetime="{&quot;end_time&quot;:&quot;2023-12-03T19:07:31.609321Z&quot;,&quot;start_time&quot;:&quot;2023-12-03T19:07:29.116542Z&quot;}" data-execution_count="110">
<div id="fig-polar" class="quarto-figure quarto-figure-center anchored">
<figure class="figure">
<div>


<table class="dataframe table table-sm table-striped small" data-quarto-postprocess="true" data-border="1">
<thead>
<tr class="header">
<th data-quarto-table-cell-role="th"></th>
<th data-quarto-table-cell-role="th">mmlu-chat</th>
<th data-quarto-table-cell-role="th">cmmlu-chat</th>
<th data-quarto-table-cell-role="th">ceval-chat</th>
<th data-quarto-table-cell-role="th">gsm8k</th>
<th data-quarto-table-cell-role="th">BBH</th>
<th data-quarto-table-cell-role="th">MATH</th>
<th data-quarto-table-cell-role="th">average</th>
</tr>
<tr class="odd">
<th data-quarto-table-cell-role="th">model</th>
<th data-quarto-table-cell-role="th"></th>
<th data-quarto-table-cell-role="th"></th>
<th data-quarto-table-cell-role="th"></th>
<th data-quarto-table-cell-role="th"></th>
<th data-quarto-table-cell-role="th"></th>
<th data-quarto-table-cell-role="th"></th>
<th data-quarto-table-cell-role="th"></th>
</tr>
</thead>
<tbody>
<tr class="odd">
<td data-quarto-table-cell-role="th">SUS-Chat-34B</td>
<td>77.35</td>
<td>78.68</td>
<td>82.42</td>
<td>80.06</td>
<td>67.62</td>
<td>28.80</td>
<td>69.155000</td>
</tr>
<tr class="even">
<td data-quarto-table-cell-role="th">Qwen-72B-Chat</td>
<td>74.52</td>
<td>77.02</td>
<td>77.22</td>
<td>76.57</td>
<td>72.63</td>
<td>35.90</td>
<td>68.976667</td>
</tr>
<tr class="odd">
<td data-quarto-table-cell-role="th">DeepSeek-67B-Chat</td>
<td>69.43</td>
<td>48.51</td>
<td>59.70</td>
<td>74.45</td>
<td>69.73</td>
<td>29.56</td>
<td>58.563333</td>
</tr>
<tr class="even">
<td data-quarto-table-cell-role="th">Yi-34B-Chat</td>
<td>66.96</td>
<td>55.16</td>
<td>77.16</td>
<td>63.76</td>
<td>61.54</td>
<td>10.02</td>
<td>55.766667</td>
</tr>
<tr class="odd">
<td data-quarto-table-cell-role="th">OrionStar-34B</td>
<td>68.51</td>
<td>66.88</td>
<td>65.13</td>
<td>54.36</td>
<td>62.88</td>
<td>12.80</td>
<td>55.093333</td>
</tr>
</tbody>
</table>

</div>
<figcaption class="figure-caption">Figure&nbsp;1: TLEM leaderboard</figcaption>
</figure>
</div>
</div>
<div class="cell" data-executetime="{&quot;end_time&quot;:&quot;2023-12-03T19:07:00.676753Z&quot;,&quot;start_time&quot;:&quot;2023-12-03T19:07:00.662376Z&quot;}" data-execution_count="109">
<div class="cell-output cell-output-display">

<div>                            <div id="54d40fc8-bbd7-4d1e-82b9-a30e0c32b0e7" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("54d40fc8-bbd7-4d1e-82b9-a30e0c32b0e7")) {                    Plotly.newPlot(                        "54d40fc8-bbd7-4d1e-82b9-a30e0c32b0e7",                        [{"fill":"toself","name":"OrionStar-34B","r":[68.51,66.88,65.13,54.36,62.88,12.8,55.09333333333333],"theta":["mmlu-chat","cmmlu-chat","ceval-chat","gsm8k","BBH","MATH"],"type":"scatterpolar"},{"fill":"toself","name":"Yi-34B-Chat","r":[66.96,55.16,77.16,63.76,61.54,10.02,55.76666666666666],"theta":["mmlu-chat","cmmlu-chat","ceval-chat","gsm8k","BBH","MATH"],"type":"scatterpolar"},{"fill":"toself","name":"Qwen-72B-Chat","r":[74.52,77.02,77.22,76.57,72.63,35.9,68.97666666666666],"theta":["mmlu-chat","cmmlu-chat","ceval-chat","gsm8k","BBH","MATH"],"type":"scatterpolar"},{"fill":"toself","name":"DeepSeek-67B-Chat","r":[69.43,48.51,59.7,74.45,69.73,29.56,58.56333333333333],"theta":["mmlu-chat","cmmlu-chat","ceval-chat","gsm8k","BBH","MATH"],"type":"scatterpolar"},{"fill":"toself","name":"SUS-Chat-34B","r":[77.35,78.68,82.42,80.06,67.62,28.8,69.155],"theta":["mmlu-chat","cmmlu-chat","ceval-chat","gsm8k","BBH","MATH"],"type":"scatterpolar"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"polar":{"radialaxis":{"visible":true,"range":[0,82.42]}},"showlegend":true},                        {"responsive": true}                    ).then(function(){
                            
var gd = document.getElementById('54d40fc8-bbd7-4d1e-82b9-a30e0c32b0e7');
var x = new MutationObserver(function (mutations, observer) {{
        var display = window.getComputedStyle(gd).display;
        if (!display || display === 'none') {{
            console.log([gd, 'removed!']);
            Plotly.purge(gd);
            observer.disconnect();
        }}
}});

// Listen for the removal of the full notebook cells
var notebookContainer = gd.closest('#notebook-container');
if (notebookContainer) {{
    x.observe(notebookContainer, {childList: true});
}}

// Listen for the clearing of the current output cell
var outputEl = gd.closest('.output');
if (outputEl) {{
    x.observe(outputEl, {childList: true});
}}

                        })                };                });            </script>        </div>
<p>TLEM leaderboard</p>
</div>
</div>
<p>Embrace the change. Embrace TLEM.</p>
</section>
</section>

</main>
<!-- /main column -->
<script id="quarto-html-after-body" type="application/javascript">
window.document.addEventListener("DOMContentLoaded", function (event) {
  const toggleBodyColorMode = (bsSheetEl) => {
    const mode = bsSheetEl.getAttribute("data-mode");
    const bodyEl = window.document.querySelector("body");
    if (mode === "dark") {
      bodyEl.classList.add("quarto-dark");
      bodyEl.classList.remove("quarto-light");
    } else {
      bodyEl.classList.add("quarto-light");
      bodyEl.classList.remove("quarto-dark");
    }
  }
  const toggleBodyColorPrimary = () => {
    const bsSheetEl = window.document.querySelector("link#quarto-bootstrap");
    if (bsSheetEl) {
      toggleBodyColorMode(bsSheetEl);
    }
  }
  toggleBodyColorPrimary();  
  const icon = "";
  const anchorJS = new window.AnchorJS();
  anchorJS.options = {
    placement: 'right',
    icon: icon
  };
  anchorJS.add('.anchored');
  const isCodeAnnotation = (el) => {
    for (const clz of el.classList) {
      if (clz.startsWith('code-annotation-')) {                     
        return true;
      }
    }
    return false;
  }
  const clipboard = new window.ClipboardJS('.code-copy-button', {
    text: function(trigger) {
      const codeEl = trigger.previousElementSibling.cloneNode(true);
      for (const childEl of codeEl.children) {
        if (isCodeAnnotation(childEl)) {
          childEl.remove();
        }
      }
      return codeEl.innerText;
    }
  });
  clipboard.on('success', function(e) {
    // button target
    const button = e.trigger;
    // don't keep focus
    button.blur();
    // flash "checked"
    button.classList.add('code-copy-button-checked');
    var currentTitle = button.getAttribute("title");
    button.setAttribute("title", "Copied!");
    let tooltip;
    if (window.bootstrap) {
      button.setAttribute("data-bs-toggle", "tooltip");
      button.setAttribute("data-bs-placement", "left");
      button.setAttribute("data-bs-title", "Copied!");
      tooltip = new bootstrap.Tooltip(button, 
        { trigger: "manual", 
          customClass: "code-copy-button-tooltip",
          offset: [0, -8]});
      tooltip.show();    
    }
    setTimeout(function() {
      if (tooltip) {
        tooltip.hide();
        button.removeAttribute("data-bs-title");
        button.removeAttribute("data-bs-toggle");
        button.removeAttribute("data-bs-placement");
      }
      button.setAttribute("title", currentTitle);
      button.classList.remove('code-copy-button-checked');
    }, 1000);
    // clear code selection
    e.clearSelection();
  });
  function tippyHover(el, contentFn) {
    const config = {
      allowHTML: true,
      content: contentFn,
      maxWidth: 500,
      delay: 100,
      arrow: false,
      appendTo: function(el) {
          return el.parentElement;
      },
      interactive: true,
      interactiveBorder: 10,
      theme: 'quarto',
      placement: 'bottom-start'
    };
    window.tippy(el, config); 
  }
  const noterefs = window.document.querySelectorAll('a[role="doc-noteref"]');
  for (var i=0; i<noterefs.length; i++) {
    const ref = noterefs[i];
    tippyHover(ref, function() {
      // use id or data attribute instead here
      let href = ref.getAttribute('data-footnote-href') || ref.getAttribute('href');
      try { href = new URL(href).hash; } catch {}
      const id = href.replace(/^#\/?/, "");
      const note = window.document.getElementById(id);
      return note.innerHTML;
    });
  }
      let selectedAnnoteEl;
      const selectorForAnnotation = ( cell, annotation) => {
        let cellAttr = 'data-code-cell="' + cell + '"';
        let lineAttr = 'data-code-annotation="' +  annotation + '"';
        const selector = 'span[' + cellAttr + '][' + lineAttr + ']';
        return selector;
      }
      const selectCodeLines = (annoteEl) => {
        const doc = window.document;
        const targetCell = annoteEl.getAttribute("data-target-cell");
        const targetAnnotation = annoteEl.getAttribute("data-target-annotation");
        const annoteSpan = window.document.querySelector(selectorForAnnotation(targetCell, targetAnnotation));
        const lines = annoteSpan.getAttribute("data-code-lines").split(",");
        const lineIds = lines.map((line) => {
          return targetCell + "-" + line;
        })
        let top = null;
        let height = null;
        let parent = null;
        if (lineIds.length > 0) {
            //compute the position of the single el (top and bottom and make a div)
            const el = window.document.getElementById(lineIds[0]);
            top = el.offsetTop;
            height = el.offsetHeight;
            parent = el.parentElement.parentElement;
          if (lineIds.length > 1) {
            const lastEl = window.document.getElementById(lineIds[lineIds.length - 1]);
            const bottom = lastEl.offsetTop + lastEl.offsetHeight;
            height = bottom - top;
          }
          if (top !== null && height !== null && parent !== null) {
            // cook up a div (if necessary) and position it 
            let div = window.document.getElementById("code-annotation-line-highlight");
            if (div === null) {
              div = window.document.createElement("div");
              div.setAttribute("id", "code-annotation-line-highlight");
              div.style.position = 'absolute';
              parent.appendChild(div);
            }
            div.style.top = top - 2 + "px";
            div.style.height = height + 4 + "px";
            let gutterDiv = window.document.getElementById("code-annotation-line-highlight-gutter");
            if (gutterDiv === null) {
              gutterDiv = window.document.createElement("div");
              gutterDiv.setAttribute("id", "code-annotation-line-highlight-gutter");
              gutterDiv.style.position = 'absolute';
              const codeCell = window.document.getElementById(targetCell);
              const gutter = codeCell.querySelector('.code-annotation-gutter');
              gutter.appendChild(gutterDiv);
            }
            gutterDiv.style.top = top - 2 + "px";
            gutterDiv.style.height = height + 4 + "px";
          }
          selectedAnnoteEl = annoteEl;
        }
      };
      const unselectCodeLines = () => {
        const elementsIds = ["code-annotation-line-highlight", "code-annotation-line-highlight-gutter"];
        elementsIds.forEach((elId) => {
          const div = window.document.getElementById(elId);
          if (div) {
            div.remove();
          }
        });
        selectedAnnoteEl = undefined;
      };
      // Attach click handler to the DT
      const annoteDls = window.document.querySelectorAll('dt[data-target-cell]');
      for (const annoteDlNode of annoteDls) {
        annoteDlNode.addEventListener('click', (event) => {
          const clickedEl = event.target;
          if (clickedEl !== selectedAnnoteEl) {
            unselectCodeLines();
            const activeEl = window.document.querySelector('dt[data-target-cell].code-annotation-active');
            if (activeEl) {
              activeEl.classList.remove('code-annotation-active');
            }
            selectCodeLines(clickedEl);
            clickedEl.classList.add('code-annotation-active');
          } else {
            // Unselect the line
            unselectCodeLines();
            clickedEl.classList.remove('code-annotation-active');
          }
        });
      }
  const findCites = (el) => {
    const parentEl = el.parentElement;
    if (parentEl) {
      const cites = parentEl.dataset.cites;
      if (cites) {
        return {
          el,
          cites: cites.split(' ')
        };
      } else {
        return findCites(el.parentElement)
      }
    } else {
      return undefined;
    }
  };
  var bibliorefs = window.document.querySelectorAll('a[role="doc-biblioref"]');
  for (var i=0; i<bibliorefs.length; i++) {
    const ref = bibliorefs[i];
    const citeInfo = findCites(ref);
    if (citeInfo) {
      tippyHover(citeInfo.el, function() {
        var popup = window.document.createElement('div');
        citeInfo.cites.forEach(function(cite) {
          var citeDiv = window.document.createElement('div');
          citeDiv.classList.add('hanging-indent');
          citeDiv.classList.add('csl-entry');
          var biblioDiv = window.document.getElementById('ref-' + cite);
          if (biblioDiv) {
            citeDiv.innerHTML = biblioDiv.innerHTML;
          }
          popup.appendChild(citeDiv);
        });
        return popup.innerHTML;
      });
    }
  }
});
</script>
</div> <!-- /content -->



</body></html>