to test dataset loading

.github/workflows/hugging-face.yaml

@@ -12,8 +12,33 @@ jobs:
         with:
           fetch-depth: 0
           lfs: true
-          
+      - name: Set Git identity
+        run: |
+          git config --global user.email "github-actions-bot@github.com"
+          git config --global user.name "GitHub Actions"
+
+      - name: Update README.md
+        run: |
+          tmp_file=$(mktemp)
+          echo "---" >> $tmp_file
+          echo "title: Yume" >> $tmp_file
+          echo "emoji: ✨" >> $tmp_file
+          echo "colorFrom: green" >> $tmp_file
+          echo "colorTo: blue" >> $tmp_file
+          echo "sdk: streamlit" >> $tmp_file
+          echo "sdk_version: 1.29.0" >> $tmp_file
+          echo "app_file: interface.py" >> $tmp_file
+          echo "pinned: false" >> $tmp_file
+          echo "license: openrail" >> $tmp_file
+          echo "---" >> $tmp_file
+          echo "" >> $tmp_file
+          cat README.md >> $tmp_file
+          mv $tmp_file README.md
+          git add README.md
+          git commit -m "Updated README.md"
+
       - name: Push to hub
         env:
           HF_TOKEN: ${{ secrets.HF_TOKEN }}
-        run: git push https://zaibutcooler:$HF_TOKEN@huggingface.co/spaces/zaibutcooler/yume main
+        run: |
+          git push https://zaibutcooler:$HF_TOKEN@huggingface.co/spaces/zaibutcooler/yume --force main

sampling.py

@@ -1,4 +1,4 @@
-from .yume import Yume,Config
+from .yume import Yume, Config
 
 config = Config()
 

tests/test_datasets.py

@@ -1,5 +1,5 @@
 import unittest
-from yume.dataset import Trainset 
+from yume.dataset import Trainset
 
 
 class TestDatasets(unittest.TestCase):
@@ -19,5 +19,6 @@ class TestDatasets(unittest.TestCase):
         encoded_text = trainset.tokenizer.encode(dummy_text)
         assert trainset.tokenizer.decode(encoded_text) == dummy_text
 
+
 if __name__ == "__main__":
     unittest.main()

tests/test_pretrained.py

@@ -1,5 +1,5 @@
 import unittest
-from yume import Yume,Config
+from yume import Yume, Config
 
 
 class TestPretrained(unittest.TestCase):
@@ -7,7 +7,7 @@ class TestPretrained(unittest.TestCase):
         super().__init__(methodName)
         self.config = Config()
         self.yume = Yume(config=self.config)
-    
+
     def test_download(self):
         self.yume.load_pretrained()
         pass

tests/test_tokenizer.py

@@ -1,12 +1,13 @@
 import unittest
 from yume import Tokenizer
 
+
 class TestTokenizer(unittest.TestCase):
     def __init__(self, methodName: str = "runTest") -> None:
         super().__init__(methodName)
         self.tokenizer = Tokenizer()
         self.dummy_text = "馬鹿なこと言わないでよ"
-    
+
     def test_encode(self):
         pass
 

training.py

@@ -1,4 +1,4 @@
-from .yume import Yume,Trainset,Config
+from .yume import Yume, Trainset, Config
 
 config = Config()
 
@@ -16,6 +16,6 @@ yume.pretrain(dataset.data)
 
 yume.sample()
 
-#optional 
+# optional
 # yume.huggingface_login("your hf tokens")
-# yume.save_pretrained("yume")
+# yume.save_pretrained("yume")

yume/__init__.py

@@ -1,4 +1,4 @@
 from .yume import Yume
 from .dataset import Trainset
-from.tokenizer import Tokenizer
+from .tokenizer import Tokenizer
 from .config import Config

yume/config.py

@@ -9,7 +9,7 @@ class Config:
         n_embd=768,
         dropout=0.0,
         bias=True,
-        lr=0.001
+        lr=0.001,
     ) -> None:
         self.num_epoch = num_epoch
         self.block_sized = 1024

yume/dataset.py

@@ -14,19 +14,17 @@ class Trainset(Dataset):
 
     def __len__(self):
         return len(self.data)
-    
+
     def __getitem__(self, index):
         assert len(self.data) > 10
         return []
 
-
-    def _load_dataset(self,url="zaibutcooler/animanga-vault"):
+    def _load_dataset(self, url="zaibutcooler/animanga-vault"):
         loaded_dataset = load_dataset(url)
-        self.texts = self.loaded_data["train"]["raw"]
-        self.data = self.loaded_data["train"]["data"]
+        self.texts = loaded_dataset["animanga"]["texts"]
         dummy_logger("Successfully loaded the dataset")
-    
-    def _tokenize(self,tiktoken=True):
+
+    def _tokenize(self, tiktoken=True):
         if tiktoken:
             enc = tiktoken.get_encoding("cl100k_base")
             assert enc.decode(enc.encode("hello world")) == "hello world"
@@ -36,4 +34,4 @@ class Trainset(Dataset):
         else:
             self.tokenizer = Tokenizer()
             self.tokenizer.load_pretrained()
-        
+        self.tokenizer.encode(self.texts)

yume/yume.py

@@ -18,17 +18,16 @@ class Yume:
 
     def generate(self):
         pass
-    
+
     def sample(self):
         pass
 
-    def pretrain(self,tokens):
+    def pretrain(self, tokens):
         lr = self.config.lr
         num_epochs = self.config.num_epoch
-        
-        
+
         pass
-    
+
     def fine_tune(self):
         pass
 
@@ -38,7 +37,7 @@ class Yume:
             n_params -= self.transformer.wpe.weight.numel()
         dummy_logger(f"parameter count -> {n_params}")
         return n_params
-    
+
     def save_pretrained(self, name="yume"):
         self.model.save_pretrained(name)
         self.model.push_to_hub(name)
@@ -51,4 +50,4 @@ class Yume:
     def huggingface_login(self, token):
         assert token is not None
         login(token=token)
-        dummy_logger("Logged in successfully")
+        dummy_logger("Logged in successfully")