Fix no white space when using stream_chat with fast tokenizer (#9)

- Fix no white space when using stream_chat with fast tokenizer (065ad5414c08dd04c70abaf3f8f58968cdf84dcb)


Co-authored-by: Shuhao Xing <x54-729@users.noreply.huggingface.co>

tokenization_internlm2_fast.py

@@ -56,14 +56,14 @@ class InternLM2Converter(SpmConverter):
         return unk_id
 
     def decoder(self, replacement, add_prefix_space):
-        return decoders.Sequence(
-            [
-                decoders.Replace("▁", " "),
-                decoders.ByteFallback(),
-                decoders.Fuse(),
-                decoders.Strip(content=" ", left=1),
-            ]
-        )
+        decoders_sequence = [
+            decoders.Replace("▁", " "),
+            decoders.ByteFallback(),
+            decoders.Fuse(),
+        ]
+        if self.proto.normalizer_spec.add_dummy_prefix:
+            decoders_sequence.append(decoders.Strip(content=" ", left=1))
+        return decoders.Sequence(decoders_sequence)
 
     def tokenizer(self, proto):
         model_type = proto.trainer_spec.model_type