Add all .py files

neural_networks.py

@@ -0,0 +1,31 @@
+import marimo
+
+__generated_with = "0.6.8"
+app = marimo.App()
+
+
+@app.cell
+def __():
+    import marimo as mo
+    return mo,
+
+
+@app.cell
+def __(mo):
+    mo.md(
+        rf"""
+        # Neural network models
+
+        **TODO**, please check back later.
+        """
+    )
+    return
+
+
+@app.cell
+def __():
+    return
+
+
+if __name__ == "__main__":
+    app.run()

tokenization.py

@@ -0,0 +1,273 @@
+import marimo
+
+__generated_with = "0.6.8"
+app = marimo.App()
+
+
+@app.cell
+def __():
+    import marimo as mo
+
+    import marimo as mo
+    from pprint import pformat
+    from collections import defaultdict
+    import utils as U
+
+    U.init_output
+    return U, defaultdict, mo, pformat
+
+
+@app.cell
+def __(mo):
+    mo.md(
+        rf"""
+        # Tokenization
+        ---
+        In the [previous notebook](?file=basics.py) we split the text roughly to words. But the models don't care about what the pieces of text are, and we can split them any way we want. In this notebook we can try out different tokenizations and see how they affect the model's behavior.
+
+        Now the text will separate tokens with alternate colored backgrounds, so they can be distinguished easier. We also print out the newline characters and show spaces as underlined.
+
+        Last time we also did a trick where spaces `' '` were not tokens. In the following we'll treat them as separate tokens too.
+
+        Now you should see some familiar lyrics tokenized like this:
+        """
+    )
+    return
+
+
+@app.cell
+def __(U):
+    class Tokenizer:
+        def tokens_to_strings(self, tokens):
+            return map(self.token_to_string, tokens)
+
+        def detokenize(self, tokens):
+            strings = self.tokens_to_strings(tokens)
+            return ''.join(strings)
+
+        def token_to_string(self, s):
+            return s
+
+    class HackyWordTokenizer(Tokenizer):
+        def __call__(self, s):
+            return s.split(' ')
+
+        def tokens_to_strings(self, tokens):
+            for token in tokens:
+                yield token
+                # TODO: Shouldn't yield last space
+                yield ' '
+
+    import re
+    class WordTokenizer(Tokenizer):
+        def __call__(self, s):
+            out = re.split('( +|\n+)', s)
+            return [t for t in out if t]
+
+    class CharacterTokenizer(Tokenizer):
+        def __call__(self, s):
+            return list(s)
+
+    import transformers
+
+    #_BASE_MODEL="EleutherAI/pythia-14m"
+    _BASE_MODEL="facebook/opt-125m"
+    class SubwordTokenizer(Tokenizer):
+        def __init__(self):
+            self._tok = transformers.AutoTokenizer.from_pretrained(_BASE_MODEL)
+
+        def __call__(self, s):
+            # Using strings instead of ids to avoid confusion
+            token_ids = self._tok(s)['input_ids']
+            return [self._tok.decode([id]) for id in token_ids]
+
+
+    tokenizers = {
+        "Word": WordTokenizer(),
+        "Character": CharacterTokenizer(),
+        "Subword": SubwordTokenizer(),
+    }
+
+    languages = {
+        "English": U.blowin_text,
+        "Finnish": U.blowin_text_finnish,
+        "German": U.blowin_text_german,
+    }
+
+
+    return (
+        CharacterTokenizer,
+        HackyWordTokenizer,
+        SubwordTokenizer,
+        Tokenizer,
+        WordTokenizer,
+        languages,
+        re,
+        tokenizers,
+        transformers,
+    )
+
+
+@app.cell
+def __(languages, mo):
+    language_selector = mo.ui.dropdown(options=languages, value="English", allow_select_none=False)
+
+    random_seed_slider = mo.ui.slider(start=1, value=1, stop=30, full_width=False, show_value=True, label="Variation (random seed)")
+    return language_selector, random_seed_slider
+
+
+@app.cell
+def __():
+    return
+
+
+@app.cell
+def __(mo):
+    #corpus_text_first_line = corpus_text.strip().split('\n')[0]
+
+    tokenizer_texts = {
+        "Word tokenizer": mo.md("""
+        The word tokenizer splits the text into individual words. This tends to generate somewhat legible text even with short context lengths. However, it can't create new words!
+
+        This is not so bad in English that has quite few inflections. However, in synthetic and agglutinative languages like Finnish this is a big problem, as you can form new words that have never been uttered in the history of the world!
+        """),
+        "Character tokenizer": mo.md("""
+        The character tokenizer splits the text into individual characters. With this we can create new words, but especially with shorter context length, it produces total gibberish!
+
+        A tradeoff between word tokenization and character tokenization is **subword tokenization**. Here common strings, like English words and Finnish inflections, are typically represented as a single token, but the tokenization also includes individual characters.
+        """),
+        "Subword tokenizer": mo.md(f"""
+        Subword tokenizer tries to split the text to commonly occuring strings, such as words, but it can "fall back" to smaller strings, including single characters. Typically most common English words are individual tokens. Also subwords like Finnish inflections or syllables may get their own tokens.
+
+        A common method for subword tokenization is [Byte Pair Encoding](https://en.wikipedia.org/wiki/Byte_pair_encoding). The tokenizer in this examples uses that method, and is in fact the same tokenizer that was used for GPT-2.
+
+        You may notice that for English the resulting tokenization is not that different from the word tokenization. A major difference is that spaces are included in the tokens. However, see what happens if you do a Finnish or German translation of the lyrics.
+        """),
+    }
+
+    context_length_slider = mo.ui.slider(start=1, value = 2, stop=10, full_width=False, label="Context length", show_value=True)
+
+    tokenizer_tabs = mo.ui.tabs(
+        tokenizer_texts,
+        value="Word"
+    )
+
+    return context_length_slider, tokenizer_tabs, tokenizer_texts
+
+
+@app.cell
+def __(tokenizer_tabs, tokenizers):
+    tokenizer_type = tokenizer_tabs.value.split()[0]
+    tokenizer = tokenizers[tokenizer_type]
+    return tokenizer, tokenizer_type
+
+
+@app.cell
+def __(
+    U,
+    context_length_slider,
+    language_selector,
+    tokenizer,
+    tokenizer_type,
+):
+    corpus_text = language_selector.value
+    context_length = context_length_slider.value
+    corpus_tokens = tokenizer(corpus_text)
+    print(tokenizer, tokenizer_type, corpus_tokens)
+    vocabulary = U.corpus_to_vocabulary(corpus_tokens)
+
+    return context_length, corpus_text, corpus_tokens, vocabulary
+
+
+@app.cell
+def __():
+    return
+
+
+@app.cell
+def __(U, context_length, corpus_tokens, tokenizer):
+    next_tokens = U.get_next_token_table(corpus_tokens, context_length)
+    U.tokens_out(corpus_tokens, tokenizer)
+    return next_tokens,
+
+
+@app.cell
+def __(mo):
+    mo.md(rf"With the tabs below, you can select different tokenizers. As you change the tokenizer, the results below change automatically. Go through the different tokenizers and observe how they change the results!")
+    return
+
+
+@app.cell
+def __(mo, tokenizer_tabs):
+    mo.md(
+        f"""
+        ## Tokenizer selection
+        ---
+        <div style="height: 20em; overflow: auto;">
+        {tokenizer_tabs}
+        </div>
+        """
+    )
+    return
+
+
+@app.cell
+def __(language_selector, mo):
+    mo.md(rf"Lyrics language {language_selector} (Translation by Google Translate)")
+    return
+
+
+@app.cell
+def __(mo):
+    mo.md(
+        f"""
+        ## Playground (watch this change!)
+        ---
+        """
+    )
+    return
+
+
+@app.cell
+def __(
+    U,
+    context_length_slider,
+    corpus_tokens,
+    mo,
+    next_tokens,
+    random_seed_slider,
+    tokenizer,
+):
+    gen_seed = random_seed_slider.value
+    gen_tokens = U.generate_tokens(next_tokens, seed=gen_seed)
+
+    gen_ui = mo.vstack([
+        U.tokens_out(gen_tokens, tokenizer),
+        mo.hstack([context_length_slider, random_seed_slider])
+    ])
+
+    mo.ui.tabs({
+        "Random generated": gen_ui,
+        "Tokenized original": U.tokens_out(corpus_tokens, tokenizer),
+        #"Follower graph": U.plot_follower_context_graph(next_tokens),
+        #"Vocabulary": U.python_out(vocabulary),
+        #"Next token table": U.python_out(dict(next_tokens)),
+    })
+    return gen_seed, gen_tokens, gen_ui
+
+
+@app.cell
+def __(mo):
+    mo.md(
+        rf"""
+        ---
+        In the next notebook, we'll learn basics of neural networks and how they can be used to create more flexible and scalable language models.
+
+        [Continue to Neural Networks >](?file=tokenization.py)
+        """
+    )
+    return
+
+
+if __name__ == "__main__":
+    app.run()

utils.py

@@ -0,0 +1,203 @@
+import marimo as mo
+
+from pprint import pformat
+from collections import defaultdict, Counter
+import random
+
+_pre_box_height = "10em";
+_font_size = "12px";
+def pre_box(text):
+    return mo.Html(f"""
+<pre class="pre_out_box" style="overflow: auto; height: {_pre_box_height}; font-size: {_font_size};">
+{text}
+
+</pre>""")
+
+def python_out(code):
+    return mo.Html(f"""
+<pre class="python_out_box" style="overflow: auto; height: {_pre_box_height}; font-size: {_font_size};">
+{pformat(code, sort_dicts=False, compact=True)}
+
+</pre>""")
+
+def tokens_out(tokens, tokenizer):
+    out = ""
+    for i, string in enumerate(tokenizer.tokens_to_strings(tokens)):
+        #colors = ["rgb(20, 184, 166)", "rgb(245, 158, 11)"]
+        colors = [
+                "#2b9a66",
+                #"#26997b",
+                "#00749e",
+                "#dc3e42",
+        ]
+        #colors = "#d1f0fa", "#ffcdce"
+        colors = "var(--sky-3)", "var(--red-3)", "var(--amber-3)"
+        color = colors[i%len(colors)]
+        # TODO: Be more general!
+        if string == ' ':
+            decoration = "underline"
+        else:
+            decoration = "none"
+
+        n_newlines = string.count('\n')
+        string = string.replace("\n", "\\n")
+        string += "\n"*n_newlines
+
+        out += f'<span style="background-color: {color}; text-decoration: {decoration}">{string}</span>'
+    out = f'<div style="overflow: auto; height: {_pre_box_height};">{out}</div>'
+    return pre_box(out)
+
+def corpus_to_vocabulary(tokens):
+    # Using dict instead of set to keep the order
+    return list({w: None for w in tokens}.keys())
+
+init_output = mo.Html(f"""
+    <style>
+    .python_out_box {{
+        overflow: auto !important;
+        max_height: {_pre_box_height};
+        font-size: 12px;
+    }}
+
+    .pre_out_box {{
+        overflow: auto !important;
+        height: {_pre_box_height};
+        font-size: 12px;
+    }}
+    </style>
+    """)
+init_output = None
+
+def graph_out(svg):
+    return mo.Html(f"""
+        <div style="overflow: auto; max-height: 32em;">
+        {svg}
+        </div>
+    """)
+
+def plot_follower_graph(next_words):
+    import pydot
+
+    graph = pydot.Dot("follower_graph", ordering="in")
+    def mangle(s):
+        #if isinstance(s, tuple) and len(s) == 1:
+        #    s = s[0]
+        return repr(s).replace(r'\n', r'\\n')
+    for context, followers in next_words.items():
+        graph.add_node(pydot.Node(mangle(context)))
+        for follower in followers:
+            edge = graph.add_edge(pydot.Edge(mangle(context), mangle(follower)))
+            # A bit of a hack
+            #if hasattr(followers, 'get'):
+            #    edge.set_label(followers.get(follower))
+            #else:
+            #    count = None
+            
+    svg = graph.create_svg().decode('utf-8')
+    return graph_out(svg)
+
+def plot_follower_context_graph(next_words):
+    # TODO: This is fugly. Use dot
+    import pydot
+
+    graph = pydot.Dot("follower_graph", ordering="in", strict=True)
+    def mangle(s):
+        #if isinstance(s, tuple) and len(s) == 1:
+        #    s = s[0]
+        return repr(s).replace(r'\n', r'\\n')
+    for context, followers in next_words.items():
+        #graph.add_node(pydot.Node(mangle(context)))
+        for follower in followers:
+            # A bit of a hack
+            #edge = graph.add_edge(pydot.Edge(mangle(context), mangle(follower)))
+            new_context = (*context[1:], follower)
+            for follower in next_words.get(context, []):
+                follower_context = (*context[1:], follower)
+                graph.add_edge(pydot.Edge(
+                    mangle(context),
+                    mangle(follower_context),
+                    label=mangle(follower)
+                ))
+
+    svg = graph.create_svg().decode('utf-8')
+    return graph_out(svg)
+
+def generate_tokens(next_words, context=None, max_tokens=200, seed=3):
+    rng = random.Random(seed)
+
+    if context is None:
+        context = next(iter(next_words.keys()))
+    yield from context
+
+    for i in range(max_tokens):
+        candidates = next_words.get(context, None)
+        if not candidates: return
+
+        choices, counts = zip(*candidates.items())
+        if not choices: return
+        next_word = rng.choice(choices)
+        if next_word == '\n\n': return
+        yield next_word
+        context = (*context[1:], next_word)
+
+# Doing this more succintly now
+def get_ngrams(tokens, n):
+    for i in range(len(tokens) - n + 1):
+        yield tokens[i:i+n]
+
+def get_next_token_table(tokens, context_length, table=None):
+    if table is None:
+        table = defaultdict(Counter)
+    for *context, next_token in get_ngrams(tokens, context_length + 1):
+        table[tuple(context)][next_token] += 1
+    
+    return table
+
+happy_birthday_text = """
+Happy birthday to you 
+Happy birthday to you 
+Happy birthday dear Dave 
+Happy birthday to you 
+"""
+
+blowin_text = """
+Yes, and how many roads must a man walk down, before you call him a man?
+And how many seas must a white dove sail, before she sleeps in the sand?
+Yes, and how many times must the cannonballs fly, before they're forever banned?
+
+Yes, and how many years must a mountain exist, before it is washed to the sea?
+And how many years can some people exist, before they're allowed to be free?
+Yes, and how many times can a man turn his head, and pretend that he just doesn't see?
+
+Yes, and how many times must a man look up, before he can see the sky?
+And how many ears must one man have, before he can hear people cry?
+Yes, and how many deaths will it take 'til he knows, that too many people have died?
+"""
+
+blowin_text_finnish = """
+Niin, ja kuinka monta tietä miehen täytyy kävellä, ennen kuin kutsut häntä mieheksi?
+Ja kuinka monta merta valkoisen kyyhkysen täytyy purjehtia, ennen kuin se nukkuu hiekkaan?
+Kyllä, ja kuinka monta kertaa kanuunankuulat täytyy lentää, ennen kuin ne on ikuisesti kielletty?
+
+Kyllä, ja kuinka monta vuotta vuoren on oltava olemassa, ennen kuin se huuhtoutuu mereen?
+Ja kuinka monta vuotta jotkut ihmiset voivat olla olemassa ennen kuin he saavat olla vapaita?
+Kyllä, ja kuinka monta kertaa ihminen voi kääntää päätään ja teeskennellä, ettei hän vain näe?
+
+Kyllä, ja kuinka monta kertaa miehen täytyy katsoa ylös, ennen kuin hän voi nähdä taivaan?
+Ja kuinka monta korvaa yhdellä ihmisellä pitää olla, ennen kuin hän voi kuulla ihmisten itkevän?
+Kyllä, ja kuinka monta kuolemaa kestää, ennen kuin hän tietää, että liian monta ihmistä on kuollut?
+"""
+
+blowin_text_german = """
+Ja, und wie viele Wege muss ein Mann gehen, bevor man ihn einen Mann nennt?
+Und wie viele Meere muss eine weiße Taube durchsegeln, bevor sie im Sand schläft?
+Ja, und wie oft müssen die Kanonenkugeln fliegen, bevor sie für immer verboten werden?
+
+Ja, und wie viele Jahre muss ein Berg existieren, bevor er ins Meer gespült wird?
+Und wie viele Jahre können manche Menschen existieren, bevor sie frei sein dürfen?
+Ja, und wie oft kann ein Mann den Kopf drehen und so tun, als würde er einfach nichts sehen?
+
+Ja, und wie oft muss ein Mensch nach oben schauen, bevor er den Himmel sehen kann?
+Und wie viele Ohren muss ein Mann haben, bevor er Menschen weinen hören kann?
+Ja, und wie viele Todesfälle wird es dauern, bis er weiß, dass zu viele Menschen gestorben sind?
+"""