Upload 12 files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+configs/silly_tavern/cards/Seraphina.png filter=lfs diff=lfs merge=lfs -text

configs/lmstudio/preset.json

@@ -0,0 +1,11 @@
+{
+  "name": "OpusV1StoryWriting",
+  "inference_params": {
+    "input_prefix": "<|im_end|>\n<|im_start|>user\n",
+    "input_suffix": "<|im_end|>\n<|im_start|>text\n",
+    "antiprompt": ["<|im_start|>", "<|im_end|>"],
+    "pre_prompt_prefix": "<|im_start|>system\n",
+    "pre_prompt_suffix": "",
+    "pre_prompt": "You are an intelligent, skilled, versatile writer.\n\nYour task is to write a story based on the information below.\n\n## Overall plot description:\n\n"
+  }
+}

configs/silly_tavern/v1/context_settings.json

@@ -0,0 +1,11 @@
+{
+  "story_string": "<|im_start|>system\nYou are an intelligent, skilled, versatile writer.\n\nYour task is to write a story based on the information below.\n\n\n## Overall plot description:\n\n{{scenario}}\n\n{{description}}\n\n\n## Characters:\n\n### {{char}}\n{{personality}}\n{{persona}}<|im_end|>",
+  "example_separator": "",
+  "chat_start": "",
+  "use_stop_strings": false,
+  "always_force_name2": false,
+  "trim_sentences": false,
+  "include_newline": false,
+  "single_line": false,
+  "name": "ChatMLOpusV1_ST2"
+}

configs/silly_tavern/v1/instruct_mode_settings.json

@@ -0,0 +1,17 @@
+{
+  "system_prompt": "",
+  "input_sequence": "<|im_start|>text names= {{user}}\n",
+  "output_sequence": "<|im_end|>\n<|im_start|>text names= {{char}}\n",
+  "first_output_sequence": "",
+  "last_output_sequence": "",
+  "system_sequence_prefix": "",
+  "system_sequence_suffix": "",
+  "stop_sequence": "",
+  "separator_sequence": "<|im_end|>\n",
+  "wrap": false,
+  "macro": true,
+  "names": false,
+  "names_force_groups": false,
+  "activation_regex": "",
+  "name": "ChatMLOpusV1_ST1"
+}

configs/silly_tavern/v2/context_settings.json

@@ -0,0 +1,11 @@
+{
+  "story_string": "<|im_start|>system\n{{#if system}}{{system}}\n\n\n{{/if}}## Overall plot description:\n\n{{#if wiBefore}}{{wiBefore}}\n\n{{/if}}{{#if scenario}}{{scenario}}\n\n\n{{/if}}## Characters:\n\n### {{char}}\n{{#if description}}{{description}}\n{{/if}}{{#if personality}}{{personality}}\n\n{{/if}}### {{user}}\n{{#if persona}}{{persona}}\n\n{{/if}}{{#if wiAfter}}{{wiAfter}}\n\n{{/if}}{{#if mesExamples}}## {{char}}'s example message:\n\n{{mesExamples}}{{/if}}",
+  "example_separator": "",
+  "chat_start": "",
+  "use_stop_strings": false,
+  "always_force_name2": false,
+  "trim_sentences": true,
+  "include_newline": false,
+  "single_line": false,
+  "name": "ChatMLOpusV1_ST2"
+}

configs/silly_tavern/v2/instruct_mode_settings.json

@@ -0,0 +1,17 @@
+{
+  "system_prompt": "You are an intelligent, skilled, versatile writer.\n\nYour task is to write a role-play based on the information below.\n\n\n## Style description:\n\nThis role-play is written as a third-person introspective narrative in past tense. Scenes are described vividly, with great detail.",
+  "input_sequence": "<|im_end|>\n<|im_start|>text names= {{user}}\n",
+  "output_sequence": "<|im_end|>\n<|im_start|>text names= {{char}}\n",
+  "first_output_sequence": "",
+  "last_output_sequence": "<|im_end|>\n<|im_start|>user\nLength: 400 words\n{{char}} replies to {{user}} in detailed and elaborate way.<|im_end|>\n<|im_start|>text names= {{char}}\n",
+  "system_sequence_prefix": "",
+  "system_sequence_suffix": "",
+  "stop_sequence": "",
+  "separator_sequence": "",
+  "wrap": false,
+  "macro": true,
+  "names": false,
+  "names_force_groups": false,
+  "activation_regex": "",
+  "name": "ChatMLOpusV1_ST2"
+}

example/interactive.py

@@ -0,0 +1,129 @@
+# python interactive.py
+
+# %%
+
+import fileinput
+
+
+from vllm import LLM, SamplingParams
+
+from prompt.format import (
+    format_opus_v1_prompt,
+    OpusV1Character,
+    OpusV1Prompt,
+    OpusV1StorySystemPrompt,
+    OpusV1Turn,
+)
+
+
+# %%
+
+
+def main():
+    sampling_params = SamplingParams(
+        # I usually stay between 0.0 and 1.0, especially for the Yi models I found lower tends to be better.
+        # For assistant tasks, I usually use 0.0.
+        temperature=0.8,
+        min_p=0.05,
+        presence_penalty=0.1,
+        frequency_penalty=0.1,
+        repetition_penalty=1.1,
+        max_tokens=200,
+        ignore_eos=True,
+        skip_special_tokens=False,
+        spaces_between_special_tokens=False,
+        stop=["<|im_end|>"],
+        include_stop_str_in_output=False,
+    )
+
+    # Set max_model_len to fit in memory.
+    model = LLM(
+        "dreamgen/opus-v1.2-7b",
+        max_model_len=2000,
+        enforce_eager=True,
+        swap_space=0,
+        gpu_memory_utilization=0.85,
+    )
+
+    plot_description = """
+This is a fanfiction from the Harry Potter universe. In this alternate reality, Harry Potter is evil and secretly siding with Slytherin.
+Up until now, Harry was pretending to be friends with Hermione and Ron, that changes when he invites Hermione to his chambers where he tricks her to drink Amorentia, the most powerful love potion.
+"""
+
+    char1 = OpusV1Character(
+        name="Harry Potter",
+        description="""Harry Potter in this fanfiction is secretly a member of Slytherin and is using his powers for evil rather than for good. Up until now, he was pretending to be friends with Hermione and Ron.""",
+    )
+    char2 = OpusV1Character(
+        name="Hermione Granger",
+        description="""Hermione appears just like in the original books.""",
+    )
+
+    story_prompt = OpusV1StorySystemPrompt(
+        plot_description=plot_description,
+        style_description="",
+        characters=[char1, char2],
+    )
+
+    turns = [
+        OpusV1Turn(
+            role="user",
+            content="""Harry invites Hermione into his chamber and offers her water, which Hermione happily accepts and drinks.""".strip(),
+        ),
+        OpusV1Turn(
+            role="text",
+            names=[char1.name],
+            content="""“Come in,” said Harry, waving at the doorway behind Hermione’s back.""".strip(),
+        ),
+    ]
+
+    def run():
+        turns.append(OpusV1Turn(role="text", content="", names=[char2.name], open=True))
+
+        prompt = OpusV1Prompt(story=story_prompt, turns=turns)
+
+        output = model.generate(
+            format_opus_v1_prompt(prompt), sampling_params, use_tqdm=False
+        )
+
+        response = OpusV1Turn(
+            role="text", content=output[0].outputs[0].text.strip(), names=[char2.name]
+        )
+        turns.append(response)
+        print(pretty_turn(response), flush=True)
+        print(f"[{char1.name}]: ", end="", flush=True)
+
+    print("## Plot description:\n")
+    print(plot_description.strip() + "\n\n")
+
+    for turn in turns:
+        print(pretty_turn(turn))
+
+    run()
+
+    for line in fileinput.input():
+        line = line.strip()
+        if line.startswith("/ins"):
+            content = line[4:].strip()
+            role = "user"
+            names = []
+        else:
+            content = line
+            role = "text"
+            names = [char1.name]
+
+        turns.append(OpusV1Turn(role=role, content=content, names=names))
+        run()
+
+
+def pretty_turn(turn):
+    if turn.role == "user":
+        return f"/ins {turn.content.strip()}"
+    else:
+        if len(turn.names) > 0:
+            return f"[{turn.names[0]}]: {turn.content.strip()}"
+        else:
+            return turn.content.strip()
+
+
+main()

example/prompt/format.py

@@ -0,0 +1,96 @@
+# %%
+from typing import Optional, List
+from dataclasses import field, dataclass
+
+
+@dataclass
+class OpusV1Turn:
+    role: str
+    content: str
+    names: List[str] = field(default_factory=list)
+    # If set to true, will not append <|im_end|>, so the model will continue the turn.
+    # In RP you can for example use the following to force a specific character response:
+    # role="text"
+    # names=["Jack"]
+    # open="true"
+    open: bool = False
+
+
+@dataclass
+class OpusV1Character:
+    name: str
+    description: str
+
+
+@dataclass
+class OpusV1StorySystemPrompt:
+    format: str = "prose"
+    plot_description: str = ""
+    style_description: str = ""
+    characters: List[OpusV1Character] = field(default_factory=list)
+
+
+@dataclass
+class OpusV1Prompt:
+    story: Optional[OpusV1StorySystemPrompt] = None
+    turns: List[OpusV1Turn] = field(default_factory=list)
+
+
+def format_opus_v1_prompt(prompt) -> str:
+    turns = prompt.turns
+    if prompt.story is not None:
+        system = format_opus_v1_system_prompt(prompt.story)
+        turns = [OpusV1Turn(role="system", content=system)] + turns
+
+    parts = []
+    for i, turn in enumerate(turns):
+        assert turn.role in ["user", "text", "system", "assistant"]
+        assert turn.role != "system" or i == 0
+
+        is_last = i == len(turns) - 1
+        open = is_last and turn.open
+        parts.append(format_turn(turn.role, turn.content, turn.names, open=open))
+    return "".join(parts)
+
+
+def format_turn(
+    role: str, content: str, names: List[str] = [], open: bool = False
+) -> str:
+    im_start = "<|im_start|>"
+    im_end = "<|im_end|>"
+
+    body = im_start + role
+    if len(names) > 0:
+        body += f" names= {'; '.join(names)}"
+
+    body += "\n"
+    if open:
+        return body + content.lstrip()
+    else:
+        return body + content.strip() + im_end + "\n"
+
+
+def format_opus_v1_system_prompt(prompt) -> str:
+    format_text = "story" if prompt.format == "prose" else "role-play"
+    system = f"""
+You are an intelligent, skilled, versatile writer.
+
+Your task is to write a {format_text} based on the information below.
+
+Write the {format_text} as if it's a book.
+    """.strip()
+
+    if len(prompt.plot_description) > 0:
+        system += "\n\n\n## Plot description:\n\n"
+        system += prompt.plot_description.strip()
+    if len(prompt.style_description) > 0:
+        system += "\n\n\n## Style description:\n\n"
+        system += prompt.style_description.strip()
+    if len(prompt.characters) > 0:
+        system += "\n\n\n## Characters:\n\n"
+        for character in prompt.characters:
+            system += f"### {character.name}\n\n"
+            system += character.description.strip()
+            system += "\n\n"
+
+    return system.strip()

example/simple.py

@@ -0,0 +1,132 @@
+# python simple.py
+
+# %%
+
+from vllm import LLM, SamplingParams
+
+from prompt.format import (
+    format_opus_v1_prompt,
+    OpusV1Character,
+    OpusV1Prompt,
+    OpusV1StorySystemPrompt,
+    OpusV1Turn,
+)
+
+
+# %%
+
+
+def build_story_prompt() -> OpusV1Prompt:
+    plot_description = """
+This is a fanfiction from the Harry Potter universe. In this alternate reality, Harry Potter is evil and secretly siding with Slytherin.
+Up until now, Harry was pretending to be friends with Hermione and Ron, that changes when he invites Hermione to his chambers where he tricks her to drink Amorentia, the most powerful love potion.
+"""
+
+    harry_description = """
+Harry Potter in this fanfiction is secretly a member of Slytherin and is using his powers for evil rather than for good. Up until now, he was pretending to be friends with Hermione and Ron.
+"""
+
+    hermione_description = """
+Hermione appears just like in the original books.
+"""
+
+    story_prompt = OpusV1StorySystemPrompt(
+        plot_description=plot_description,
+        style_description="",
+        characters=[
+            OpusV1Character(name="Harry Potter", description=harry_description),
+            OpusV1Character(name="Hermione Granger", description=hermione_description),
+        ],
+    )
+
+    return OpusV1Prompt(
+        story=story_prompt,
+        turns=[
+            OpusV1Turn(
+                role="user",
+                content="""
+The story starts with Harry welcoming Hermione into his chambers, who he invited there earlier that day. He offers her water to drink, but it contains a love potion.
+    """.strip(),
+            ),
+            OpusV1Turn(
+                role="text",
+                content="""
+“Come in,” said Harry, waving at the doorway behind Hermione’s back.
+
+“Hello?” she said, stepping inside, “what did you want me to come up here for?”
+
+“Well, I thought we could get away from all the noise down there, have a chat about what we plan to do for Christmas…” Harry said, fumbling for words. He had never really been any good with girls. “But anyway, please, take a seat and let me get us some water!” he said, darting over to the sideboard.
+
+He returned quickly with two glasses of water. Hermione took hers and thanked him, taking in a big gulp. As soon as she swallowed, Harry saw her eyes widen as her heart began beating wildly in her chest.
+
+It worked! Harry thought, grinning to himself. Amorentia truly was the world’s best love potion, its effects lasting twice as long and being five times stronger.
+    """.strip(),
+                open=True,
+            ),
+        ],
+    )
+
+
+def build_assistant_prompt() -> OpusV1Prompt:
+    return OpusV1Prompt(
+        turns=[
+            OpusV1Turn(
+                role="system",
+                content="You are an intelligent, knowledgeable, helpful, general-purpose assistant.",
+            ),
+            OpusV1Turn(
+                role="user",
+                content="Give me a sentence where every word begins with 'S'",
+            ),
+        ]
+    )
+
+
+# %%
+
+
+def main():
+    sampling_params = SamplingParams(
+        # I usually stay between 0.0 and 1.0, especially for the Yi models I found lower tends to be better.
+        # For assistant tasks, I usually use 0.0.
+        temperature=0.0,
+        min_p=0.05,
+        presence_penalty=0.1,
+        frequency_penalty=0.1,
+        repetition_penalty=1.1,
+        max_tokens=200,
+        ignore_eos=True,
+        skip_special_tokens=False,
+        spaces_between_special_tokens=False,
+    )
+
+    # Set max_model_len to fit in memory.
+    model = LLM(
+        "dreamgen/opus-v1.2-7b",
+        max_model_len=2000,
+        enforce_eager=True,
+        swap_space=0,
+        gpu_memory_utilization=0.85,
+    )
+
+    story_prompt = build_story_prompt()
+    print(format_opus_v1_prompt(story_prompt))
+
+    output = model.generate(format_opus_v1_prompt(story_prompt), sampling_params)
+    print(output[0].outputs[0].text)
+
+    # Expected:
+    """
+    It would make her fall deeply in love with him, and then he could use her to get what he wanted.
+
+    “Harry, what’s going on? You look so happy!” Hermione asked, smiling at him.
+
+    “Oh, well, I guess I am,” Harry replied, trying not to laugh. “I mean, I’ve always known that you were the one for me.”
+
+    “Really?” Hermione asked, blushing slightly. “I didn’t know that.”
+
+    “Yeah, I’ve always had feelings for you,” Harry said, leaning forward and placing his hand on top of hers. “And now that I’ve got you alone, I can finally tell you how much I care about you.”
+    """
+
+
+main()