Spaces:
Runtime error
Runtime error
File size: 939 Bytes
ee21b96 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 |
#!/usr/bin/env python3 -u
# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
import argparse
import sys
from fairseq.data import Dictionary
def get_parser():
parser = argparse.ArgumentParser(
description="filters a lexicon given a unit dictionary"
)
parser.add_argument("-d", "--unit-dict", help="unit dictionary", required=True)
return parser
def main():
parser = get_parser()
args = parser.parse_args()
d = Dictionary.load(args.unit_dict)
symbols = set(d.symbols)
for line in sys.stdin:
items = line.rstrip().split()
skip = len(items) < 2
for x in items[1:]:
if x not in symbols:
skip = True
break
if not skip:
print(line, end="")
if __name__ == "__main__":
main()
|