Upload processing_mists.py
Browse files- processing_mists.py +10 -3
processing_mists.py
CHANGED
@@ -44,6 +44,7 @@ class MistsProcessor(ProcessorMixin):
|
|
44 |
torch_dtype: Optional[Union[str, torch.dtype]] = torch.float,
|
45 |
time_series_padding: Union[bool, str] = False,
|
46 |
time_series_max_length: Union[int, None] = None,
|
|
|
47 |
) -> BatchFeature:
|
48 |
if time_series is not None:
|
49 |
time_series_values = self.feature_extractor(
|
@@ -55,9 +56,15 @@ class MistsProcessor(ProcessorMixin):
|
|
55 |
)
|
56 |
else:
|
57 |
time_series_values = None
|
58 |
-
|
59 |
-
|
60 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
61 |
|
62 |
return BatchFeature(data={**text_inputs, **time_series_values})
|
63 |
|
|
|
44 |
torch_dtype: Optional[Union[str, torch.dtype]] = torch.float,
|
45 |
time_series_padding: Union[bool, str] = False,
|
46 |
time_series_max_length: Union[int, None] = None,
|
47 |
+
text_tokenize: bool = True,
|
48 |
) -> BatchFeature:
|
49 |
if time_series is not None:
|
50 |
time_series_values = self.feature_extractor(
|
|
|
56 |
)
|
57 |
else:
|
58 |
time_series_values = None
|
59 |
+
if text is not None:
|
60 |
+
if text_tokenize:
|
61 |
+
text_inputs = self.tokenizer(
|
62 |
+
text, return_tensors=return_tensors, padding=padding, truncation=truncation, max_length=max_length
|
63 |
+
)
|
64 |
+
else:
|
65 |
+
text_inputs = {"text": text}
|
66 |
+
else:
|
67 |
+
text_inputs = None
|
68 |
|
69 |
return BatchFeature(data={**text_inputs, **time_series_values})
|
70 |
|