Added basics

Dockerfile

@@ -0,0 +1,11 @@
+FROM python:3.9
+
+WORKDIR /code
+
+COPY ./requirements.txt /code/requirements.txt
+
+RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
+
+COPY . .
+
+CMD ["marimo", "run", "--host", "0.0.0.0", "--port", "7860", "basics.py"]

basics.py

@@ -0,0 +1,446 @@
+import marimo
+
+__generated_with = "0.6.8"
+app = marimo.App(app_title="SLMs basics")
+
+
+@app.cell
+def __():
+    import marimo as mo
+    from pprint import pformat
+    from collections import defaultdict
+    import utils as U
+
+    U.init_output
+    return U, defaultdict, mo, pformat
+
+
+@app.cell
+def __(mo):
+    mo.md(
+        r"""
+        # Small language models
+
+        ## Happy birthday
+        ---
+        To get started, we analyze lyrics of perhaps the most popular song in the world.
+        You may be familiar with the lyrics:
+        """
+    )
+    return
+
+
+@app.cell
+def __():
+    corpus_text = """
+     Happy birthday to you 
+     Happy birthday to you 
+     Happy birthday dear Dave 
+     Happy birthday to you 
+    """
+    corpus_text
+    return corpus_text,
+
+
+@app.cell
+def __(mo):
+    mo.md(
+        r"""
+        To work with text, we usually want to split it to some shorter pieces, such
+        as words. In general, such pieces are called **tokens**, but we'll start with just
+        words. Our lyrics split into words become:
+        """
+    )
+    return
+
+
+@app.cell
+def __(U, corpus_text):
+    corpus_words = corpus_text.split(' ')
+    U.python_out(corpus_words)
+    return corpus_words,
+
+
+@app.cell
+def __(mo):
+    mo.md(
+        rf"""
+        (The here `'\n'` means that we start a new line. While not really a word, we treat it as such for now.)
+
+        We can also build our **vocabulary**, which is just all individual words that is in our lyrics:
+        """
+    )
+    return
+
+
+@app.cell
+def __(U, corpus_words):
+    # Using dict instead of set to keep the order
+    _vocabulary = {w: None for w in corpus_words}.keys()
+    U.python_out(list(_vocabulary))
+    return
+
+
+@app.cell
+def __(mo):
+    mo.md(
+        r"""
+        The currently popular large language models (LLMs) -- such as GPT, Llama and Mistral -- are based on predicting what token becomes after
+        some number of tokens.
+
+        In our case, for example, the word `'Happy'` is followerd by the word `'birthday'` and the
+        word `'birthday'` is followed by the word `'to'`.
+
+        In fact, to make an extremely simple language model, we can just list what words are followed by each
+        word. For our lyrics this becomes:
+        """
+    )
+    return
+
+
+@app.cell
+def __(U, corpus_words):
+    next_words = {}
+    for i in range(len(corpus_words)-1):
+        word = corpus_words[i]
+        next_word = corpus_words[i+1]
+        if word not in next_words:
+            next_words[word] = []
+        next_words[word].append(next_word)
+    U.python_out(next_words)
+    return i, next_word, next_words, word
+
+
+@app.cell
+def __(mo):
+    mo.md(r"Or as a visual graph format:")
+    return
+
+
+@app.cell
+def __(U, next_words):
+    U.plot_follower_graph(next_words)
+    return
+
+
+@app.cell
+def __(mo):
+    mo.md(
+        r"""
+        We can see that after a new line `'\n'` we always get the word `'Happy'`, and `'Happy'` is always followed by
+        `'birthday'`. Somewhat more interestingly, the word `'birthday'` was followed three times by `'to'` but also
+        once by `'dear'`.
+
+        With this model, we are ready to generate new lyrics! Select the next word from the dropdown
+        to add it into the lyrics.
+        """
+    )
+    return
+
+
+@app.cell
+def __(corpus_words, mo):
+    initial_lyrics_birthday = tuple(corpus_words[:2])
+    get_lyrics_birthday, set_lyrics_birthday = mo.state(initial_lyrics_birthday, allow_self_loops=True)
+    return get_lyrics_birthday, initial_lyrics_birthday, set_lyrics_birthday
+
+
+@app.cell
+def __(mo):
+    def dropdown_generate(next_words, lyrics_state, initial_lyrics):
+        get_lyrics, set_lyrics = lyrics_state
+        lyrics = get_lyrics()
+        options = set(next_words[lyrics[-1]])
+        def update(value):
+            new_lyrics = (*get_lyrics(), value)
+            set_lyrics((*get_lyrics(), value))
+
+        lyrics_text = ' ' + ' '.join(get_lyrics())
+        optvals = {repr(o): o for o in options}
+        dropdown = mo.ui.dropdown(options=optvals, on_change=update)
+        reset = mo.ui.button(
+            label="Reset lyrics",
+            on_change=lambda *args: set_lyrics(initial_lyrics)
+        )
+
+        #lyrics_el = mo.Html(f"<pre>{lyrics_text} {dropdown}</pre>")
+        return dropdown, reset
+    return dropdown_generate,
+
+
+@app.cell
+def __(
+    dropdown_generate,
+    get_lyrics_birthday,
+    initial_lyrics_birthday,
+    mo,
+    next_words,
+    set_lyrics_birthday,
+):
+    # These have to be globals for the events to be triggered.
+    # Marimo has some ways to go to enable modular code
+    dropdown_birthday, reset_birthday = dropdown_generate(next_words, (get_lyrics_birthday, set_lyrics_birthday), initial_lyrics_birthday)
+    _text = ' '.join(get_lyrics_birthday())
+    _lyrics_el = mo.Html(f"<pre>{_text} {dropdown_birthday}</pre>")
+
+    mo.hstack([_lyrics_el, reset_birthday])
+    return dropdown_birthday, reset_birthday
+
+
+@app.cell
+def __(mo):
+    mo.md(
+        rf"""
+        ## Blowin' in the wind
+        ---
+
+        The previous looked only one word at the time. However, we can easily use more than one word to predict the next one. How many words (or tokens) we use to predict the next one, is known as the **context length**. The context length of the previous example was 1.
+
+        With the very simple lyrics context length more than 1 does not make much sense, so let's pick something a bit more complicated:
+        """
+    )
+    return
+
+
+@app.cell
+def __():
+    blowin_text = """
+     Yes, and how many roads must a man walk down, before you call him a man? 
+     And how many seas must a white dove sail, before she sleeps in the sand? 
+     Yes, and how many times must the cannonballs fly, before they're forever banned? 
+
+     Yes, and how many years must a mountain exist, before it is washed to the sea? 
+     And how many years can some people exist, before they're allowed to be free? 
+     Yes, and how many times can a man turn his head, and pretend that he just doesn't see? 
+
+     Yes, and how many times must a man look up, before he can see the sky? 
+     And how many ears must one man have, before he can hear people cry? 
+     Yes, and how many deaths will it take 'til he knows, that too many people have died? 
+    """
+    blowin_text
+    return blowin_text,
+
+
+@app.cell
+def __(mo):
+    mo.md(
+        rf"""
+        You may recognize the lyrics. They're the verses of the Bob Dylan's song [Blowin' in the Wind](https://www.youtube.com/watch?v=MMFj8uDubsE).
+
+        We proceed like before, first splitting the lyrics into words:
+        """
+    )
+    return
+
+
+@app.cell
+def __(U, blowin_text):
+    blowin_words = blowin_text.split(' ')
+    U.python_out(blowin_words)
+    return blowin_words,
+
+
+@app.cell
+def __(mo):
+    mo.md(
+        rf"""
+        Note that we now have punctuation included in the ''words'', like the comma in `'Yes,'` the question mark in `'man?'`. We also treat two newlines `'\n\n'` as one ''word''. This comes handy, as it separates the verses.
+
+        We now have quite a bit larger vocabulary:
+        """
+    )
+    return
+
+
+@app.cell
+def __(U, blowin_words):
+    U.python_out(list(U.corpus_to_vocabulary(blowin_words)))
+    return
+
+
+@app.cell
+def __(mo):
+    mo.md(
+        rf"""
+        ### More context
+        ---
+        We build a simple language model again with these lyrics. These simple models are usually called ''Markov Chain text generators''. This is a bit misleading, as even the next-token-predicting LLMs are Markov chains. We won't discuss what Markov chains really are and what makes a model such, but Wikipedia has a [rather good article](https://en.wikipedia.org/wiki/Markov_chain) of these if you're interested. 
+
+        Previously in the ''Happy Birthday'' example the model looked only one word at the time. However, we can easily use more than one word to predict the next one. How many words (or tokens) we use to predict the next one, is known as the **context length**. The context length of the previous example was 1.
+
+        For lyrics as simple as in ''Happy Birthday'' using a context length more than 1 didn't make much sense. However, with the more complicated lyrics we can see how the model behavior changes with different context lengths.
+
+        You can select the context length with the slider and see how the model changes.
+        """
+    )
+    return
+
+
+@app.cell
+def __(context_length_slider, mo):
+    mo.md(f"The context length is {context_length_slider.value}")
+    return
+
+
+@app.cell
+def __(mo):
+    # TODO: Display context length value
+    context_length_slider = mo.ui.slider(start=1, stop=8, full_width=True)
+    context_length_slider
+    return context_length_slider,
+
+
+@app.cell
+def __(blowin_words, context_length_slider, defaultdict):
+    #blowin_context_length = 2
+    blowin_context_length = context_length_slider.value
+    # Doing this more succintly now
+    def get_ngrams(tokens, n):
+        for i in range(len(tokens) - n + 1):
+            yield tokens[i:i+n]
+
+    blowin_next_words1 = defaultdict(list) 
+    for *_context, _next_word in get_ngrams(blowin_words, blowin_context_length + 1):
+        blowin_next_words1[tuple(_context)].append(_next_word)
+
+    #python_out(dict(blowin_next_words1))
+    return blowin_context_length, blowin_next_words1, get_ngrams
+
+
+@app.cell
+def __():
+    #plot_follower_graph(blowin_next_words1)
+    return
+
+
+@app.cell
+def __(mo):
+    mo.md(rf"We can now generate some lyrics with the model. Here's some machine generated ones, you can do your own below.")
+    return
+
+
+@app.cell
+def __():
+    import random
+    random.seed(3)
+    return random,
+
+
+@app.cell
+def __(mo):
+    regen_blowin1_btn = mo.ui.button(label="Generate new verse")
+    regen_blowin1_btn
+    return regen_blowin1_btn,
+
+
+@app.cell
+def genblow1_1(U, blowin_next_words1, random, regen_blowin1_btn):
+    # TODO: Keep the seed constant across generations
+
+    regen_blowin1_btn
+
+    def _generate(next_words):
+        context = next(iter(next_words.keys()))
+        yield from context
+
+        while True:
+            choices = next_words[context]
+            if not choices: return
+            next_word = random.choice(choices)
+            if next_word == '\n\n': return
+            yield next_word
+            context = (*context[1:], next_word)
+
+    _generated = list(_generate(blowin_next_words1))
+    U.pre_box(' '.join(_generated))
+    return
+
+
+@app.cell
+def __(U, blowin_next_words1, mo):
+    mo.accordion({
+        "Next word table": U.python_out(dict(blowin_next_words1)),
+        "Next word graph": U.plot_follower_graph(blowin_next_words1)
+    })
+    return
+
+
+@app.cell
+def __(mo):
+    mo.md(
+        rf"""
+        With a short context length the lyrics dont make much sense. With a longer context length it starts to just copy the originals. Try to find a context length that seems to make a nice tradeoff between these. As a hint, you can get something quite silly with some context lengths.
+
+        Try to be such a language model yourself! This time the generated lyrics are hidden. Don't peek at them before you're done, and pretend you don't remember what you picked before!
+        """
+    )
+    return
+
+
+@app.cell
+def __(blowin_context_length, blowin_words, mo):
+    initial_lyrics_blowin = blowin_words[:blowin_context_length + 1]
+    get_lyrics_blowin1, set_lyrics_blowin1 = mo.state(initial_lyrics_blowin, allow_self_loops=True)
+    return get_lyrics_blowin1, initial_lyrics_blowin, set_lyrics_blowin1
+
+
+@app.cell
+def __(
+    blowin_context_length,
+    blowin_next_words1,
+    get_lyrics_blowin1,
+    initial_lyrics_blowin,
+    mo,
+    set_lyrics_blowin1,
+):
+    def dropdown_generate_blowin(next_words, lyrics_state, initial_lyrics):
+        get_lyrics, set_lyrics = lyrics_state
+        lyrics = get_lyrics()
+        context = tuple(lyrics[-blowin_context_length:])
+        options = set(next_words[context])
+        def update(value):
+            new_lyrics = (*get_lyrics(), value)
+            set_lyrics((*get_lyrics(), value))
+
+        lyrics_text = ' ' + ' '.join(get_lyrics())
+        optvals = {repr(o): o for o in options}
+        dropdown = mo.ui.dropdown(options=optvals, on_change=update)
+        reset = mo.ui.button(
+            label="Reset lyrics",
+            on_change=lambda *args: set_lyrics(initial_lyrics)
+        )
+
+        #lyrics_el = mo.Html(f"<pre>{lyrics_text} {dropdown}</pre>")
+        return dropdown, reset
+
+    dropdown_blowin1, reset_blowin1 = dropdown_generate_blowin(blowin_next_words1, (get_lyrics_blowin1, set_lyrics_blowin1), initial_lyrics_blowin)
+    _ctx = ', '.join(map(repr, get_lyrics_blowin1()[-blowin_context_length:]))
+    _lyrics_el = mo.Html(f"<pre>{_ctx} {dropdown_blowin1}</pre>")
+
+    _lyrics_el
+    return dropdown_blowin1, dropdown_generate_blowin, reset_blowin1
+
+
+@app.cell
+def __(get_lyrics_blowin1, mo, reset_blowin1):
+    _lyrics = ' '.join(get_lyrics_blowin1())
+    _spoiler = mo.accordion({'Your generated lyrics. SPOILER!': mo.Html(f"<pre>{_lyrics}</pre>")})
+    mo.vstack([_spoiler, reset_blowin1])
+    return
+
+
+@app.cell
+def __(mo):
+    mo.md(
+        rf"""
+        ---
+        In the next notebook, we'll take a closer look at **tokenization**, i.e. how we split the text for processing.
+
+        [Continue to Tokenization >](?file=tokenization.py)
+        """
+    )
+    return
+
+
+if __name__ == "__main__":
+    app.run()

requirements.txt

@@ -0,0 +1,5 @@
+marimo
+transformers
+pydot
+matplotlib
+