new approach

README.md

@@ -1,6 +1,6 @@
 ---
 title: KafkaLM 8x7b German
-emoji: 🏴
+emoji: 🏴🍺
 colorFrom: gray
 colorTo: yellow
 sdk: gradio

app.py

@@ -1,31 +1,34 @@
-from transformers import AutoModelForCausalLM, AutoTokenizer
+import gradio as gr
+from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
 
+# Load the model and tokenizer from Hugging Face
 model_id = "seedboxai/KafkaLM-8x7B-German-V0.1-DPO"
-
-model = AutoModelForCausalLM.from_pretrained(model_id, load_in_4bit=True, trust_remote_code=True)
-
 tokenizer = AutoTokenizer.from_pretrained(model_id)
+model = AutoModelForCausalLM.from_pretrained(model_id, load_in_4bit=True, trust_remote_code=True)
 
-def generate_prompt(input):
-    prompt = ''
-    sys_prompt = "Du bist ein freundlicher und hilfsbereiter KI-Assistent. Du beantwortest Fragen faktenorientiert und präzise, ohne dabei relevante Fakten auszulassen."
+# Define the text generation function
+def generate_text(user_input, system_prompt):
+    # Combine the system prompt and the user input to form the full prompt
+    full_prompt = f"{system_prompt.strip()}\n\n{user_input.strip()}"
     
-    prompt += f"<|system|>\n{sys_prompt.strip()}</s>\n"
-    prompt += f"<|user|>\n{input.strip()}</s>\n"
-    prompt += f"<|assistant|>\n"
-
-    return prompt.strip()
-
-
-generate_text = transformers.pipeline(
-    model=model, tokenizer=tokenizer,
-    return_full_text=True,  
-    task='text-generation',
-    temperature=0.5,  
-    max_new_tokens=512,
-    top_p=0.95,
-    top_k=50,
-    do_sample=True,
-)
-
-print(generate_text(generate_prompt("Wer ist eigentlich dieser Kafka?")))
+    # Initialize the pipeline for text generation
+    text_generator = pipeline('text-generation', model=model, tokenizer=tokenizer,
+                              return_full_text=True, temperature=0.5,
+                              max_new_tokens=512, top_p=0.95, top_k=50, do_sample=True)
+    
+    # Generate text based on the full prompt
+    results = text_generator(full_prompt)
+    generated_text = results[0]['generated_text']
+    
+    return generated_text
+
+# Setup the Gradio interface
+iface = gr.Interface(fn=generate_text,
+                     inputs=[gr.Textbox(lines=2, label="User Prompt"), gr.Textbox(lines=5, label="System Prompt")],
+                     outputs=gr.Textbox(label="Generated Text"),
+                     title="Text Generation with KafkaLM",
+                     description="Enter a user prompt and a system prompt to generate text using the KafkaLM model.")
+
+# Launch the Gradio app
+if __name__ == "__main__":
+    iface.launch()

config.json

@@ -1,31 +0,0 @@
-{
-  "_name_or_path": "/workspace/Mixtral-MoE-8x7B/mixtral_FT",
-  "architectures": [
-    "MixtralForCausalLM"
-  ],
-  "attention_dropout": 0.0,
-  "bos_token_id": 1,
-  "eos_token_id": 2,
-  "hidden_act": "silu",
-  "hidden_size": 4096,
-  "initializer_range": 0.02,
-  "intermediate_size": 14336,
-  "max_position_embeddings": 32768,
-  "model_type": "mixtral",
-  "num_attention_heads": 32,
-  "num_experts_per_tok": 2,
-  "num_hidden_layers": 32,
-  "num_key_value_heads": 8,
-  "num_local_experts": 8,
-  "output_router_logits": false,
-  "rms_norm_eps": 1e-05,
-  "rope_theta": 1000000.0,
-  "router_aux_loss_coef": 0.02,
-  "sliding_window": null,
-  "tie_word_embeddings": false,
-  "torch_dtype": "float32",
-  "transformers_version": "4.38.0.dev0",
-  "use_cache": true,
-  "vocab_size": 32000
-}
-

generation_config.json

@@ -1,6 +0,0 @@
-{
-  "_from_model_config": true,
-  "bos_token_id": 1,
-  "eos_token_id": 2,
-  "transformers_version": "4.38.0.dev0"
-}

model-00001-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:5c990c6ea3bdfcde5d6fa97a20f7bb8489e8c7338644e2b22f8cf25818ce9e11
-size 4920052720

model-00002-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:fe7646c12dd730078c98f2d44297b44458307dd12767b467b4f86a2984f1de47
-size 4865559920

model-00003-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:13f19a49293abe99a576b0ce3929cc07ef41f27d0c9baaa3f52f68970b6720d4
-size 4865559920

model-00004-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:d518bfaaaed4ab6745318fd3547c4a30ad8f5dc9ed3880e23597976d34c68554
-size 4865559920

model-00005-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:7cadcd62f958377f8c6ecd2a8d91789b39d947608ef5966dcbe1c4a1f47049d0
-size 4865559920

model-00006-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:0db04d581cc8843ee905224a263c533aea2bc040df93ebd9e21f7485ea753132
-size 4932504264

model-00007-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:ff5119411700efc0b0d80b0e7b364eddbfb9847f91c3296e0b344acd981fcbd2
-size 4865559912

model-00008-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:f2d6314b97e43abad0e93d8fb3fdf2a1ff8dc7dd1e18d56abf07334ca1b56096
-size 4865559920

model-00009-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:b83d7e18dfbcca682465e23367fcb93655e2d5fe60ff00371b49844cdeb98b8b
-size 4865559920

model-00010-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:040485aa8f9d4d146e0b248ddcf16d2593ab47a156f026543ba659783bee51d8
-size 4865559920

model-00011-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:f9fda815c4e401d63794a75664e4454086808cada111685d56ccf9590e6fe54b
-size 4865559920

model-00012-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:e7ac62fa7b4945beaa9aa2541df84b1b3df5cdce7d1af1accd308384ce7ddc9c
-size 4999646240

model-00013-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:41f87398271754733a0667682eb8ef4d211d1aab970ab86cff5bdca322f8a8df
-size 4798417968

model-00014-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:b1c4e598453fdcc597b6291e8b4788b21cab23be4f1b806e0871ab4140680603
-size 4865559944

model-00015-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:eb037b28ce4cd25ae3f235c22a40735520ec542c14da4c335de353b6155e1835
-size 4865559944

model-00016-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:0e2e178b8bedb7d402569aa1a1dbfa1fe3161944d2854b255eee94c73162015f
-size 4865559944

model-00017-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:76943d899f6924da8272f4cc50d02e8864de99158d355bc35f5ba0e8915a858e
-size 4865559944

model-00018-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:434c588897cde14211182cfd7d2eade857b0addd8c9c409655f72b1d005b910e
-size 4865559944

model-00019-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:c5bdbc41a8f2386b2ecfb6ff9844e26deec4023ada34b3f2dd555506f5ef9d73
-size 4932504280

model-00020-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:d58562d1f02dd718c7ed1da1fd0ca6f73c9e356cb3bffcc5d93106b61a9359a3
-size 4865559944

model-00021-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:26fab439fad7a239ae79a27dad2dbc7ca9ff1c35ffaad6c28922f7a1f91f81b2
-size 4865559944

model-00022-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:f809057c08a390638024321e4c22d68c25dc5de15033bf875ccb4e3f807d1dd9
-size 4865559944

model-00023-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:24141d66049ad3723104e8f55a99b90afb2e1dfe74cf42919ae4ef4c8a117ee0
-size 4865559944

model-00024-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:25d5c910e8270ede8c48a189549fc8818e26913f1a7421ba02b5b95e8e4e22ee
-size 4865559944

model-00025-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:64dec72f25781d1370290a1a67c908fe326e6e088b544e5c3598810d25fa4160
-size 4932504280

model-00026-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:f5c3a336a71bee7c2eb841c8a4a26b05ad034d691124108c7773b4b42809f0be
-size 4865559944

model-00027-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:4878787645006ae7b52fd7c53bbb26ff450d0ebdd179292d0bc532b6586492d2
-size 4865559944

model-00028-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:0b7b39c7adc025284ad8e514487ecd4939bafc7386ae9deea94c528970b145d8
-size 4865559944

model-00029-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:1e588bb3ca105a72e84a20e4230eb799e86539bd5e5e27009446af45ce139dff
-size 4865559944

model-00030-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:58294165ab3abb9ef2a73e7d93aaab5b638bdb419aae60249c50bd7deb088188
-size 4865559944

model-00031-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:c14e641970ea2f713977d63036547814ccd243f2681352ec941570d9df17312a
-size 4932504280

model-00032-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:2b2c94d4d9fd8996474fde5ff1b568fbbdafeb130c7a7027dd48acfbf2413274
-size 4865559944

model-00033-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:03f06fe39f9c0fa4dd8dd18894a9c54ffc0267a5b29c677c8fc65ce7d4542010
-size 4865559944

model-00034-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:f241bf09bf2c482c01267eaca0d609ca76897ebb095f5811d17885bf376584b6
-size 4865559944

model-00035-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:7dc056901ca382f1411315fdb058f5e81d94c84891eccd84f52f7305b55ce2af
-size 4865559944

model-00036-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:906664c2b302d64f59ac8b77f1574f96b4a8e050fd30a127d57ef3af119a68a1
-size 4865559944

model-00037-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:57525a3d3d422788d35b394624fea656a5d8f7d1cc987d8377b7963bf006d34d
-size 4999646264

model-00038-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:1f6c9be67ffd7c39216759c94e4eb835ae455b9a1f15d3d2facd28a90774a0ce
-size 4798417968

model-00039-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:cf35d1c660a0ff1e3715fafc0b3575c26e9ed2bc1a9ba618db23aaab1bd44cb2
-size 1463862216