Spaces:
Sleeping
Sleeping
File size: 2,111 Bytes
801501a |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 |
# References: https://github.com/optas/shapeglot
# https://github.com/63days/PartGlot.
from six.moves import cPickle
def unpickle_data(file_name, python2_to_3=False):
"""Restore data previously saved with pickle_data().
:param file_name: file holding the pickled data.
:param python2_to_3: (boolean), if True, pickle happened under python2x, unpickling under python3x.
:return: a generator over the un-pickled items.
Note, about implementing the python2_to_3 see
https://stackoverflow.com/questions/28218466/unpickling-a-python-2-object-with-python-3
"""
in_file = open(file_name, "rb")
if python2_to_3:
size = cPickle.load(in_file, encoding="latin1")
else:
size = cPickle.load(in_file)
for _ in range(size):
if python2_to_3:
yield cPickle.load(in_file, encoding="latin1")
else:
yield cPickle.load(in_file)
in_file.close()
def get_mask_of_game_data(
game_data: DataFrame,
word2int: Dict,
only_correct: bool,
only_easy_context: bool,
max_seq_len: int,
only_one_part_name: bool,
):
"""
only_correct (if True): mask will be 1 in location iff human listener predicted correctly.
only_easy (if True): uses only easy context examples (more dissimilar triplet chairs)
max_seq_len: drops examples with len(utterance) > max_seq_len
only_one_part_name (if True): uses only utterances describing only one part in the give set.
"""
mask = np.array(game_data.correct)
if not only_correct:
mask = np.ones_like(mask, dtype=np.bool)
if only_easy_context:
context_mask = np.array(game_data.context_condition == "easy", dtype=np.bool)
mask = np.logical_and(mask, context_mask)
short_mask = np.array(
game_data.text.apply(lambda x: len(x)) <= max_seq_len, dtype=np.bool
)
mask = np.logical_and(mask, short_mask)
part_indicator, part_mask = get_part_indicator(game_data.text, word2int)
if only_one_part_name:
mask = np.logical_and(mask, part_mask)
return mask, part_indicator
|