Spaces:
Sleeping
Sleeping
| # coding=utf-8 | |
| # Copyright 2020 HuggingFace Datasets Authors. | |
| # | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| # Lint as: python3 | |
| """Introduction to the CoNLL-2003 Shared Task: Language-Independent Named Entity Recognition""" | |
| import datasets | |
| logger = datasets.logging.get_logger(__name__) | |
| _CITATION = """\ | |
| @inproceedings{tjong-kim-sang-de-meulder-2003-introduction, | |
| title = "Introduction to the {C}o{NLL}-2003 Shared Task: Language-Independent Named Entity Recognition", | |
| author = "Tjong Kim Sang, Erik F. and | |
| De Meulder, Fien", | |
| booktitle = "Proceedings of the Seventh Conference on Natural Language Learning at {HLT}-{NAACL} 2003", | |
| year = "2003", | |
| url = "https://www.aclweb.org/anthology/W03-0419", | |
| pages = "142--147", | |
| } | |
| """ | |
| _DESCRIPTION = """\ | |
| The shared task of CoNLL-2003 concerns language-independent named entity recognition. We will concentrate on | |
| four types of named entities: persons, locations, organizations and names of miscellaneous entities that do | |
| not belong to the previous three groups. | |
| The CoNLL-2003 shared task data files contain four columns separated by a single space. Each word has been put on | |
| a separate line and there is an empty line after each sentence. The first item on each line is a word, the second | |
| a part-of-speech (POS) tag, the third a syntactic chunk tag and the fourth the named entity tag. The chunk tags | |
| and the named entity tags have the format I-TYPE which means that the word is inside a phrase of type TYPE. Only | |
| if two phrases of the same type immediately follow each other, the first word of the second phrase will have tag | |
| B-TYPE to show that it starts a new phrase. A word with tag O is not part of a phrase. Note the dataset uses IOB2 | |
| tagging scheme, whereas the original dataset uses IOB1. | |
| For more details see https://www.clips.uantwerpen.be/conll2003/ner/ and https://www.aclweb.org/anthology/W03-0419 | |
| """ | |
| _URL = "../../../data/CoNLL03/" | |
| _TRAINING_FILE = "train.txt" | |
| _DEV_FILE = "valid.txt" | |
| _TEST_FILE = "test.txt" | |
| class Conll2003Config(datasets.BuilderConfig): | |
| """BuilderConfig for Conll2003""" | |
| def __init__(self, **kwargs): | |
| """BuilderConfig forConll2003. | |
| Args: | |
| **kwargs: keyword arguments forwarded to super. | |
| """ | |
| super(Conll2003Config, self).__init__(**kwargs) | |
| class Conll2003(datasets.GeneratorBasedBuilder): | |
| """Conll2003 dataset.""" | |
| BUILDER_CONFIGS = [ | |
| Conll2003Config(name="conll2003", version=datasets.Version("1.0.0"), description="Conll2003 dataset"), | |
| ] | |
| def _info(self): | |
| return datasets.DatasetInfo( | |
| description=_DESCRIPTION, | |
| features=datasets.Features( | |
| { | |
| "id": datasets.Value("string"), | |
| "tokens": datasets.Sequence(datasets.Value("string")), | |
| "pos_tags": datasets.Sequence( | |
| datasets.features.ClassLabel( | |
| names=[ | |
| '"', | |
| "''", | |
| "#", | |
| "$", | |
| "(", | |
| ")", | |
| ",", | |
| ".", | |
| ":", | |
| "``", | |
| "CC", | |
| "CD", | |
| "DT", | |
| "EX", | |
| "FW", | |
| "IN", | |
| "JJ", | |
| "JJR", | |
| "JJS", | |
| "LS", | |
| "MD", | |
| "NN", | |
| "NNP", | |
| "NNPS", | |
| "NNS", | |
| "NN|SYM", | |
| "PDT", | |
| "POS", | |
| "PRP", | |
| "PRP$", | |
| "RB", | |
| "RBR", | |
| "RBS", | |
| "RP", | |
| "SYM", | |
| "TO", | |
| "UH", | |
| "VB", | |
| "VBD", | |
| "VBG", | |
| "VBN", | |
| "VBP", | |
| "VBZ", | |
| "WDT", | |
| "WP", | |
| "WP$", | |
| "WRB", | |
| ] | |
| ) | |
| ), | |
| "chunk_tags": datasets.Sequence( | |
| datasets.features.ClassLabel( | |
| names=[ | |
| "O", | |
| "B-ADJP", | |
| "I-ADJP", | |
| "B-ADVP", | |
| "I-ADVP", | |
| "B-CONJP", | |
| "I-CONJP", | |
| "B-INTJ", | |
| "I-INTJ", | |
| "B-LST", | |
| "I-LST", | |
| "B-NP", | |
| "I-NP", | |
| "B-PP", | |
| "I-PP", | |
| "B-PRT", | |
| "I-PRT", | |
| "B-SBAR", | |
| "I-SBAR", | |
| "B-UCP", | |
| "I-UCP", | |
| "B-VP", | |
| "I-VP", | |
| ] | |
| ) | |
| ), | |
| "ner_tags": datasets.Sequence( | |
| datasets.features.ClassLabel( | |
| names=[ | |
| "O", | |
| "B-PER", | |
| "I-PER", | |
| "B-ORG", | |
| "I-ORG", | |
| "B-LOC", | |
| "I-LOC", | |
| "B-MISC", | |
| "I-MISC", | |
| ] | |
| ) | |
| ), | |
| } | |
| ), | |
| supervised_keys=None, | |
| homepage="https://www.aclweb.org/anthology/W03-0419/", | |
| citation=_CITATION, | |
| ) | |
| def _split_generators(self, dl_manager): | |
| """Returns SplitGenerators.""" | |
| urls_to_download = { | |
| "train": f"{_URL}{_TRAINING_FILE}", | |
| "dev": f"{_URL}{_DEV_FILE}", | |
| "test": f"{_URL}{_TEST_FILE}", | |
| } | |
| downloaded_files = dl_manager.download_and_extract(urls_to_download) | |
| return [ | |
| datasets.SplitGenerator(name=datasets.Split.TRAIN, gen_kwargs={"filepath": downloaded_files["train"]}), | |
| datasets.SplitGenerator(name=datasets.Split.VALIDATION, gen_kwargs={"filepath": downloaded_files["dev"]}), | |
| datasets.SplitGenerator(name=datasets.Split.TEST, gen_kwargs={"filepath": downloaded_files["test"]}), | |
| ] | |
| def _generate_examples(self, filepath): | |
| logger.info("⏳ Generating examples from = %s", filepath) | |
| with open(filepath, encoding="utf-8") as f: | |
| guid = 0 | |
| tokens = [] | |
| pos_tags = [] | |
| chunk_tags = [] | |
| ner_tags = [] | |
| for line in f: | |
| if line.startswith("-DOCSTART-") or line == "" or line == "\n": | |
| if tokens: | |
| yield guid, { | |
| "id": str(guid), | |
| "tokens": tokens, | |
| "pos_tags": pos_tags, | |
| "chunk_tags": chunk_tags, | |
| "ner_tags": ner_tags, | |
| } | |
| guid += 1 | |
| tokens = [] | |
| pos_tags = [] | |
| chunk_tags = [] | |
| ner_tags = [] | |
| else: | |
| # conll2003 tokens are space separated | |
| splits = line.split(" ") | |
| tokens.append(splits[0]) | |
| pos_tags.append(splits[1]) | |
| chunk_tags.append(splits[2]) | |
| ner_tags.append(splits[3].rstrip()) | |
| # last example | |
| yield guid, { | |
| "id": str(guid), | |
| "tokens": tokens, | |
| "pos_tags": pos_tags, | |
| "chunk_tags": chunk_tags, | |
| "ner_tags": ner_tags, | |
| } |