Spaces:
Running
Running
| # Copyright 2017 Google Inc. All Rights Reserved. | |
| # | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| # ============================================================================== | |
| """Tests for data_utils.""" | |
| from __future__ import absolute_import | |
| from __future__ import division | |
| from __future__ import print_function | |
| # Dependency imports | |
| import tensorflow as tf | |
| from data import data_utils | |
| data = data_utils | |
| class SequenceWrapperTest(tf.test.TestCase): | |
| def testDefaultTimesteps(self): | |
| seq = data.SequenceWrapper() | |
| t1 = seq.add_timestep() | |
| _ = seq.add_timestep() | |
| self.assertEqual(len(seq), 2) | |
| self.assertEqual(t1.weight, 0.0) | |
| self.assertEqual(t1.label, 0) | |
| self.assertEqual(t1.token, 0) | |
| def testSettersAndGetters(self): | |
| ts = data.SequenceWrapper().add_timestep() | |
| ts.set_token(3) | |
| ts.set_label(4) | |
| ts.set_weight(2.0) | |
| self.assertEqual(ts.token, 3) | |
| self.assertEqual(ts.label, 4) | |
| self.assertEqual(ts.weight, 2.0) | |
| def testTimestepIteration(self): | |
| seq = data.SequenceWrapper() | |
| seq.add_timestep().set_token(0) | |
| seq.add_timestep().set_token(1) | |
| seq.add_timestep().set_token(2) | |
| for i, ts in enumerate(seq): | |
| self.assertEqual(ts.token, i) | |
| def testFillsSequenceExampleCorrectly(self): | |
| seq = data.SequenceWrapper() | |
| seq.add_timestep().set_token(1).set_label(2).set_weight(3.0) | |
| seq.add_timestep().set_token(10).set_label(20).set_weight(30.0) | |
| seq_ex = seq.seq | |
| fl = seq_ex.feature_lists.feature_list | |
| fl_token = fl[data.SequenceWrapper.F_TOKEN_ID].feature | |
| fl_label = fl[data.SequenceWrapper.F_LABEL].feature | |
| fl_weight = fl[data.SequenceWrapper.F_WEIGHT].feature | |
| _ = [self.assertEqual(len(f), 2) for f in [fl_token, fl_label, fl_weight]] | |
| self.assertAllEqual([f.int64_list.value[0] for f in fl_token], [1, 10]) | |
| self.assertAllEqual([f.int64_list.value[0] for f in fl_label], [2, 20]) | |
| self.assertAllEqual([f.float_list.value[0] for f in fl_weight], [3.0, 30.0]) | |
| class DataUtilsTest(tf.test.TestCase): | |
| def testSplitByPunct(self): | |
| output = data.split_by_punct( | |
| 'hello! world, i\'ve been\nwaiting\tfor\ryou for.a long time') | |
| expected = [ | |
| 'hello', 'world', 'i', 've', 'been', 'waiting', 'for', 'you', 'for', | |
| 'a', 'long', 'time' | |
| ] | |
| self.assertListEqual(output, expected) | |
| def _buildDummySequence(self): | |
| seq = data.SequenceWrapper() | |
| for i in range(10): | |
| seq.add_timestep().set_token(i) | |
| return seq | |
| def testBuildLMSeq(self): | |
| seq = self._buildDummySequence() | |
| lm_seq = data.build_lm_sequence(seq) | |
| for i, ts in enumerate(lm_seq): | |
| # For end of sequence, the token and label should be same, and weight | |
| # should be 0.0. | |
| if i == len(lm_seq) - 1: | |
| self.assertEqual(ts.token, i) | |
| self.assertEqual(ts.label, i) | |
| self.assertEqual(ts.weight, 0.0) | |
| else: | |
| self.assertEqual(ts.token, i) | |
| self.assertEqual(ts.label, i + 1) | |
| self.assertEqual(ts.weight, 1.0) | |
| def testBuildSAESeq(self): | |
| seq = self._buildDummySequence() | |
| sa_seq = data.build_seq_ae_sequence(seq) | |
| self.assertEqual(len(sa_seq), len(seq) * 2 - 1) | |
| # Tokens should be sequence twice, minus the EOS token at the end | |
| for i, ts in enumerate(sa_seq): | |
| self.assertEqual(ts.token, seq[i % 10].token) | |
| # Weights should be len-1 0.0's and len 1.0's. | |
| for i in range(len(seq) - 1): | |
| self.assertEqual(sa_seq[i].weight, 0.0) | |
| for i in range(len(seq) - 1, len(sa_seq)): | |
| self.assertEqual(sa_seq[i].weight, 1.0) | |
| # Labels should be len-1 0's, and then the sequence | |
| for i in range(len(seq) - 1): | |
| self.assertEqual(sa_seq[i].label, 0) | |
| for i in range(len(seq) - 1, len(sa_seq)): | |
| self.assertEqual(sa_seq[i].label, seq[i - (len(seq) - 1)].token) | |
| def testBuildLabelSeq(self): | |
| seq = self._buildDummySequence() | |
| eos_id = len(seq) - 1 | |
| label_seq = data.build_labeled_sequence(seq, True) | |
| for i, ts in enumerate(label_seq[:-1]): | |
| self.assertEqual(ts.token, i) | |
| self.assertEqual(ts.label, 0) | |
| self.assertEqual(ts.weight, 0.0) | |
| final_timestep = label_seq[-1] | |
| self.assertEqual(final_timestep.token, eos_id) | |
| self.assertEqual(final_timestep.label, 1) | |
| self.assertEqual(final_timestep.weight, 1.0) | |
| def testBuildBidirLabelSeq(self): | |
| seq = self._buildDummySequence() | |
| reverse_seq = data.build_reverse_sequence(seq) | |
| bidir_seq = data.build_bidirectional_seq(seq, reverse_seq) | |
| label_seq = data.build_labeled_sequence(bidir_seq, True) | |
| for (i, ts), j in zip( | |
| enumerate(label_seq[:-1]), reversed(range(len(seq) - 1))): | |
| self.assertAllEqual(ts.tokens, [i, j]) | |
| self.assertEqual(ts.label, 0) | |
| self.assertEqual(ts.weight, 0.0) | |
| final_timestep = label_seq[-1] | |
| eos_id = len(seq) - 1 | |
| self.assertAllEqual(final_timestep.tokens, [eos_id, eos_id]) | |
| self.assertEqual(final_timestep.label, 1) | |
| self.assertEqual(final_timestep.weight, 1.0) | |
| def testReverseSeq(self): | |
| seq = self._buildDummySequence() | |
| reverse_seq = data.build_reverse_sequence(seq) | |
| for i, ts in enumerate(reversed(reverse_seq[:-1])): | |
| self.assertEqual(ts.token, i) | |
| self.assertEqual(ts.label, 0) | |
| self.assertEqual(ts.weight, 0.0) | |
| final_timestep = reverse_seq[-1] | |
| eos_id = len(seq) - 1 | |
| self.assertEqual(final_timestep.token, eos_id) | |
| self.assertEqual(final_timestep.label, 0) | |
| self.assertEqual(final_timestep.weight, 0.0) | |
| def testBidirSeq(self): | |
| seq = self._buildDummySequence() | |
| reverse_seq = data.build_reverse_sequence(seq) | |
| bidir_seq = data.build_bidirectional_seq(seq, reverse_seq) | |
| for (i, ts), j in zip( | |
| enumerate(bidir_seq[:-1]), reversed(range(len(seq) - 1))): | |
| self.assertAllEqual(ts.tokens, [i, j]) | |
| self.assertEqual(ts.label, 0) | |
| self.assertEqual(ts.weight, 0.0) | |
| final_timestep = bidir_seq[-1] | |
| eos_id = len(seq) - 1 | |
| self.assertAllEqual(final_timestep.tokens, [eos_id, eos_id]) | |
| self.assertEqual(final_timestep.label, 0) | |
| self.assertEqual(final_timestep.weight, 0.0) | |
| def testLabelGain(self): | |
| seq = self._buildDummySequence() | |
| label_seq = data.build_labeled_sequence(seq, True, label_gain=True) | |
| for i, ts in enumerate(label_seq): | |
| self.assertEqual(ts.token, i) | |
| self.assertEqual(ts.label, 1) | |
| self.assertNear(ts.weight, float(i) / (len(seq) - 1), 1e-3) | |
| if __name__ == '__main__': | |
| tf.test.main() | |