Update spacy pipeline to 3.5.1

README.md

@@ -14,69 +14,69 @@ model-index:
     metrics:
     - name: NER Precision
       type: precision
-      value: 0.8581105169
+      value: 0.8572441922
     - name: NER Recall
       type: recall
-      value: 0.8463431786
+      value: 0.849859353
     - name: NER F Score
       type: f_score
-      value: 0.8521862277
+      value: 0.8535357994
   - task:
       name: TAG
       type: token-classification
     metrics:
     - name: TAG (XPOS) Accuracy
       type: accuracy
-      value: 0.9649265515
+      value: 0.9624844483
   - task:
       name: POS
       type: token-classification
     metrics:
     - name: POS (UPOS) Accuracy
       type: accuracy
-      value: 0.9634910761
+      value: 0.9631543688
   - task:
       name: MORPH
       type: token-classification
     metrics:
     - name: Morph (UFeats) Accuracy
       type: accuracy
-      value: 0.9308067758
+      value: 0.928892717
   - task:
       name: LEMMA
       type: token-classification
     metrics:
     - name: Lemma Accuracy
       type: accuracy
-      value: 0.9738780978
+      value: 0.9728255669
   - task:
       name: UNLABELED_DEPENDENCIES
       type: token-classification
     metrics:
     - name: Unlabeled Attachment Score (UAS)
       type: f_score
-      value: 0.8116901329
+      value: 0.8127597439
   - task:
       name: LABELED_DEPENDENCIES
       type: token-classification
     metrics:
     - name: Labeled Attachment Score (LAS)
       type: f_score
-      value: 0.7417545434
+      value: 0.743681905
   - task:
       name: SENTS
       type: token-classification
     metrics:
     - name: Sentences F-Score
       type: f_score
-      value: 0.9710467706
+      value: 0.9787709497
 ---
 Core Hungarian model for HuSpaCy. Components: tok2vec, senter, tagger, morphologizer, lemmatizer, parser, ner
 
 | Feature | Description |
 | --- | --- |
 | **Name** | `hu_core_news_md` |
-| **Version** | `3.5.0` |
+| **Version** | `3.5.1` |
 | **spaCy** | `>=3.5.0,<3.6.0` |
 | **Default Pipeline** | `tok2vec`, `senter`, `tagger`, `morphologizer`, `lookup_lemmatizer`, `lemmatizer`, `lemma_smoother`, `parser`, `ner` |
 | **Components** | `tok2vec`, `senter`, `tagger`, `morphologizer`, `lookup_lemmatizer`, `lemmatizer`, `lemma_smoother`, `parser`, `ner` |
@@ -108,18 +108,18 @@ Core Hungarian model for HuSpaCy. Components: tok2vec, senter, tagger, morpholog
 | `TOKEN_P` | 99.86 |
 | `TOKEN_R` | 99.93 |
 | `TOKEN_F` | 99.89 |
-| `SENTS_P` | 97.10 |
-| `SENTS_R` | 97.10 |
-| `SENTS_F` | 97.10 |
-| `TAG_ACC` | 96.49 |
-| `POS_ACC` | 96.35 |
-| `MORPH_ACC` | 93.08 |
-| `MORPH_MICRO_P` | 96.72 |
-| `MORPH_MICRO_R` | 95.96 |
-| `MORPH_MICRO_F` | 96.34 |
-| `LEMMA_ACC` | 97.39 |
-| `DEP_UAS` | 81.17 |
-| `DEP_LAS` | 74.18 |
-| `ENTS_P` | 85.81 |
-| `ENTS_R` | 84.63 |
-| `ENTS_F` | 85.22 |
+| `SENTS_P` | 98.21 |
+| `SENTS_R` | 97.55 |
+| `SENTS_F` | 97.88 |
+| `TAG_ACC` | 96.25 |
+| `POS_ACC` | 96.32 |
+| `MORPH_ACC` | 92.89 |
+| `MORPH_MICRO_P` | 96.49 |
+| `MORPH_MICRO_R` | 95.78 |
+| `MORPH_MICRO_F` | 96.14 |
+| `LEMMA_ACC` | 97.28 |
+| `DEP_UAS` | 81.28 |
+| `DEP_LAS` | 74.37 |
+| `ENTS_P` | 85.72 |
+| `ENTS_R` | 84.99 |
+| `ENTS_F` | 85.35 |

config.cfg

@@ -1,8 +1,8 @@
 [paths]
-parser_model = "models/hu_core_news_md-parser-3.5.0/model-best"
-ner_model = "models/hu_core_news_md-ner-3.5.0/model-best"
-lemmatizer_lookups = "models/hu_core_news_md-lookup-lemmatizer-3.5.0"
-tagger_model = "models/hu_core_news_md-tagger-3.5.0/model-best"
+parser_model = "models/hu_core_news_md-parser-3.5.1/model-best"
+ner_model = "models/hu_core_news_md-ner-3.5.1/model-best"
+lemmatizer_lookups = "models/hu_core_news_md-lookup-lemmatizer-3.5.1"
+tagger_model = "models/hu_core_news_md-tagger-3.5.1/model-best"
 train = null
 dev = null
 vectors = null

edit_tree_lemmatizer.py

@@ -0,0 +1,465 @@
+from functools import lru_cache
+
+from typing import cast, Any, Callable, Dict, Iterable, List, Optional
+from typing import Sequence, Tuple, Union
+from collections import Counter
+from copy import deepcopy
+from itertools import islice
+import numpy as np
+
+import srsly
+from thinc.api import Config, Model, SequenceCategoricalCrossentropy, NumpyOps
+from thinc.types import Floats2d, Ints2d
+
+from spacy.pipeline._edit_tree_internals.edit_trees import EditTrees
+from spacy.pipeline._edit_tree_internals.schemas import validate_edit_tree
+from spacy.pipeline.lemmatizer import lemmatizer_score
+from spacy.pipeline.trainable_pipe import TrainablePipe
+from spacy.errors import Errors
+from spacy.language import Language
+from spacy.tokens import Doc, Token
+from spacy.training import Example, validate_examples, validate_get_examples
+from spacy.vocab import Vocab
+from spacy import util
+
+
+TOP_K_GUARDRAIL = 20
+
+
+default_model_config = """
+[model]
+@architectures = "spacy.Tagger.v2"
+
+[model.tok2vec]
+@architectures = "spacy.HashEmbedCNN.v2"
+pretrained_vectors = null
+width = 96
+depth = 4
+embed_size = 2000
+window_size = 1
+maxout_pieces = 3
+subword_features = true
+"""
+DEFAULT_EDIT_TREE_LEMMATIZER_MODEL = Config().from_str(default_model_config)["model"]
+
+
+@Language.factory(
+    "trainable_lemmatizer_v2",
+    assigns=["token.lemma"],
+    requires=[],
+    default_config={
+        "model": DEFAULT_EDIT_TREE_LEMMATIZER_MODEL,
+        "backoff": "orth",
+        "min_tree_freq": 3,
+        "overwrite": False,
+        "top_k": 1,
+        "overwrite_labels": True,
+        "scorer": {"@scorers": "spacy.lemmatizer_scorer.v1"},
+    },
+    default_score_weights={"lemma_acc": 1.0},
+)
+def make_edit_tree_lemmatizer(
+    nlp: Language,
+    name: str,
+    model: Model,
+    backoff: Optional[str],
+    min_tree_freq: int,
+    overwrite: bool,
+    top_k: int,
+    overwrite_labels: bool,
+    scorer: Optional[Callable],
+):
+    """Construct an EditTreeLemmatizer component."""
+    return EditTreeLemmatizer(
+        nlp.vocab,
+        model,
+        name,
+        backoff=backoff,
+        min_tree_freq=min_tree_freq,
+        overwrite=overwrite,
+        top_k=top_k,
+        overwrite_labels=overwrite_labels,
+        scorer=scorer,
+    )
+
+
+# _f = open("lemmatizer.log", "w")
+# def debug(*args):
+#     _f.write(" ".join(args) + "\n")
+def debug(*args):
+    pass
+
+
+class EditTreeLemmatizer(TrainablePipe):
+    """
+    Lemmatizer that lemmatizes each word using a predicted edit tree.
+    """
+
+    def __init__(
+        self,
+        vocab: Vocab,
+        model: Model,
+        name: str = "trainable_lemmatizer",
+        *,
+        backoff: Optional[str] = "orth",
+        min_tree_freq: int = 3,
+        overwrite: bool = False,
+        top_k: int = 1,
+        overwrite_labels,
+        scorer: Optional[Callable] = lemmatizer_score,
+    ):
+        """
+        Construct an edit tree lemmatizer.
+
+        backoff (Optional[str]): backoff to use when the predicted edit trees
+            are not applicable. Must be an attribute of Token or None (leave the
+            lemma unset).
+        min_tree_freq (int): prune trees that are applied less than this
+            frequency in the training data.
+        overwrite (bool): overwrite existing lemma annotations.
+        top_k (int): try to apply at most the k most probable edit trees.
+        """
+        self.vocab = vocab
+        self.model = model
+        self.name = name
+        self.backoff = backoff
+        self.min_tree_freq = min_tree_freq
+        self.overwrite = overwrite
+        self.top_k = top_k
+        self.overwrite_labels = overwrite_labels
+
+        self.trees = EditTrees(self.vocab.strings)
+        self.tree2label: Dict[int, int] = {}
+
+        self.cfg: Dict[str, Any] = {"labels": []}
+        self.scorer = scorer
+        self.numpy_ops = NumpyOps()
+
+    def get_loss(
+        self, examples: Iterable[Example], scores: List[Floats2d]
+    ) -> Tuple[float, List[Floats2d]]:
+        validate_examples(examples, "EditTreeLemmatizer.get_loss")
+        loss_func = SequenceCategoricalCrossentropy(normalize=False, missing_value=-1)
+
+        truths = []
+        for eg in examples:
+            eg_truths = []
+            for (predicted, gold_lemma, gold_pos, gold_sent_start) in zip(
+                eg.predicted,
+                eg.get_aligned("LEMMA", as_string=True),
+                eg.get_aligned("POS", as_string=True),
+                eg.get_aligned_sent_starts(),
+            ):
+                if gold_lemma is None:
+                    label = -1
+                else:
+                    form = self._get_true_cased_form(
+                        predicted.text, gold_sent_start, gold_pos
+                    )
+                    tree_id = self.trees.add(form, gold_lemma)
+                    # debug(f"@get_loss: {predicted}/{gold_pos}[{gold_sent_start}]->{form}|{gold_lemma}[{tree_id}]")
+                    label = self.tree2label.get(tree_id, 0)
+                eg_truths.append(label)
+
+            truths.append(eg_truths)
+
+        d_scores, loss = loss_func(scores, truths)
+        if self.model.ops.xp.isnan(loss):
+            raise ValueError(Errors.E910.format(name=self.name))
+
+        return float(loss), d_scores
+
+    def predict(self, docs: Iterable[Doc]) -> List[Ints2d]:
+        if self.top_k == 1:
+            scores2guesses = self._scores2guesses_top_k_equals_1
+        elif self.top_k <= TOP_K_GUARDRAIL:
+            scores2guesses = self._scores2guesses_top_k_greater_1
+        else:
+            scores2guesses = self._scores2guesses_top_k_guardrail
+        # The behaviour of *_scores2guesses_top_k_greater_1()* is efficient for values
+        # of *top_k>1* that are likely to be useful when the edit tree lemmatizer is used
+        # for its principal purpose of lemmatizing tokens. However, the code could also
+        # be used for other purposes, and with very large values of *top_k* the method
+        # becomes inefficient. In such cases, *_scores2guesses_top_k_guardrail()* is used
+        # instead.
+        n_docs = len(list(docs))
+        if not any(len(doc) for doc in docs):
+            # Handle cases where there are no tokens in any docs.
+            n_labels = len(self.cfg["labels"])
+            guesses: List[Ints2d] = [self.model.ops.alloc2i(0, n_labels) for _ in docs]
+            assert len(guesses) == n_docs
+            return guesses
+        scores = self.model.predict(docs)
+        assert len(scores) == n_docs
+        guesses = scores2guesses(docs, scores)
+        assert len(guesses) == n_docs
+        return guesses
+
+    def _scores2guesses_top_k_equals_1(self, docs, scores):
+        guesses = []
+        for doc, doc_scores in zip(docs, scores):
+            doc_guesses = doc_scores.argmax(axis=1)
+            doc_guesses = self.numpy_ops.asarray(doc_guesses)
+
+            doc_compat_guesses = []
+            for i, token in enumerate(doc):
+                tree_id = self.cfg["labels"][doc_guesses[i]]
+                form: str = self._get_true_cased_form_of_token(token)
+                if self.trees.apply(tree_id, form) is not None:
+                    doc_compat_guesses.append(tree_id)
+                else:
+                    doc_compat_guesses.append(-1)
+            guesses.append(np.array(doc_compat_guesses))
+
+        return guesses
+
+    def _scores2guesses_top_k_greater_1(self, docs, scores):
+        guesses = []
+        top_k = min(self.top_k, len(self.labels))
+        for doc, doc_scores in zip(docs, scores):
+            doc_scores = self.numpy_ops.asarray(doc_scores)
+            doc_compat_guesses = []
+            for i, token in enumerate(doc):
+                for _ in range(top_k):
+                    candidate = int(doc_scores[i].argmax())
+                    candidate_tree_id = self.cfg["labels"][candidate]
+                    form: str = self._get_true_cased_form_of_token(token)
+                    if self.trees.apply(candidate_tree_id, form) is not None:
+                        doc_compat_guesses.append(candidate_tree_id)
+                        break
+                    doc_scores[i, candidate] = np.finfo(np.float32).min
+                else:
+                    doc_compat_guesses.append(-1)
+            guesses.append(np.array(doc_compat_guesses))
+
+        return guesses
+
+    def _scores2guesses_top_k_guardrail(self, docs, scores):
+        guesses = []
+        for doc, doc_scores in zip(docs, scores):
+            doc_guesses = np.argsort(doc_scores)[..., : -self.top_k - 1 : -1]
+            doc_guesses = self.numpy_ops.asarray(doc_guesses)
+
+            doc_compat_guesses = []
+            for token, candidates in zip(doc, doc_guesses):
+                tree_id = -1
+                for candidate in candidates:
+                    candidate_tree_id = self.cfg["labels"][candidate]
+
+                    form: str = self._get_true_cased_form_of_token(token)
+
+                    if self.trees.apply(candidate_tree_id, form) is not None:
+                        tree_id = candidate_tree_id
+                        break
+                doc_compat_guesses.append(tree_id)
+
+            guesses.append(np.array(doc_compat_guesses))
+
+        return guesses
+
+    def set_annotations(self, docs: Iterable[Doc], batch_tree_ids):
+        for i, doc in enumerate(docs):
+            doc_tree_ids = batch_tree_ids[i]
+            if hasattr(doc_tree_ids, "get"):
+                doc_tree_ids = doc_tree_ids.get()
+            for j, tree_id in enumerate(doc_tree_ids):
+                if self.overwrite or doc[j].lemma == 0:
+                    # If no applicable tree could be found during prediction,
+                    # the special identifier -1 is used. Otherwise the tree
+                    # is guaranteed to be applicable.
+                    if tree_id == -1:
+                        if self.backoff is not None:
+                            doc[j].lemma = getattr(doc[j], self.backoff)
+                    else:
+                        form = self._get_true_cased_form_of_token(doc[j])
+                        lemma = self.trees.apply(tree_id, form) or form
+                        # debug(f"@set_annotations: {doc[j]}/{doc[j].pos_}[{doc[j].is_sent_start}]->{form}|{lemma}[{tree_id}]")
+                        doc[j].lemma_ = lemma
+
+    @property
+    def labels(self) -> Tuple[int, ...]:
+        """Returns the labels currently added to the component."""
+        return tuple(self.cfg["labels"])
+
+    @property
+    def hide_labels(self) -> bool:
+        return True
+
+    @property
+    def label_data(self) -> Dict:
+        trees = []
+        for tree_id in range(len(self.trees)):
+            tree = self.trees[tree_id]
+            if "orig" in tree:
+                tree["orig"] = self.vocab.strings[tree["orig"]]
+            if "subst" in tree:
+                tree["subst"] = self.vocab.strings[tree["subst"]]
+            trees.append(tree)
+        return dict(trees=trees, labels=tuple(self.cfg["labels"]))
+
+    def initialize(
+        self,
+        get_examples: Callable[[], Iterable[Example]],
+        *,
+        nlp: Optional[Language] = None,
+        labels: Optional[Dict] = None,
+    ):
+        validate_get_examples(get_examples, "EditTreeLemmatizer.initialize")
+
+        if self.overwrite_labels:
+            if labels is None:
+                self._labels_from_data(get_examples)
+            else:
+                self._add_labels(labels)
+
+        # Sample for the model.
+        doc_sample = []
+        label_sample = []
+        for example in islice(get_examples(), 10):
+            doc_sample.append(example.x)
+            gold_labels: List[List[float]] = []
+            for token in example.reference:
+                if token.lemma == 0:
+                    gold_label = None
+                else:
+                    gold_label = self._pair2label(token.text, token.lemma_)
+
+                gold_labels.append(
+                    [
+                        1.0 if label == gold_label else 0.0
+                        for label in self.cfg["labels"]
+                    ]
+                )
+
+            gold_labels = cast(Floats2d, gold_labels)
+            label_sample.append(self.model.ops.asarray(gold_labels, dtype="float32"))
+
+        self._require_labels()
+        assert len(doc_sample) > 0, Errors.E923.format(name=self.name)
+        assert len(label_sample) > 0, Errors.E923.format(name=self.name)
+
+        self.model.initialize(X=doc_sample, Y=label_sample)
+
+    def from_bytes(self, bytes_data, *, exclude=tuple()):
+        deserializers = {
+            "cfg": lambda b: self.cfg.update(srsly.json_loads(b)),
+            "model": lambda b: self.model.from_bytes(b),
+            "vocab": lambda b: self.vocab.from_bytes(b, exclude=exclude),
+            "trees": lambda b: self.trees.from_bytes(b),
+        }
+
+        util.from_bytes(bytes_data, deserializers, exclude)
+
+        return self
+
+    def to_bytes(self, *, exclude=tuple()):
+        serializers = {
+            "cfg": lambda: srsly.json_dumps(self.cfg),
+            "model": lambda: self.model.to_bytes(),
+            "vocab": lambda: self.vocab.to_bytes(exclude=exclude),
+            "trees": lambda: self.trees.to_bytes(),
+        }
+
+        return util.to_bytes(serializers, exclude)
+
+    def to_disk(self, path, exclude=tuple()):
+        path = util.ensure_path(path)
+        serializers = {
+            "cfg": lambda p: srsly.write_json(p, self.cfg),
+            "model": lambda p: self.model.to_disk(p),
+            "vocab": lambda p: self.vocab.to_disk(p, exclude=exclude),
+            "trees": lambda p: self.trees.to_disk(p),
+        }
+        util.to_disk(path, serializers, exclude)
+
+    def from_disk(self, path, exclude=tuple()):
+        def load_model(p):
+            try:
+                with open(p, "rb") as mfile:
+                    self.model.from_bytes(mfile.read())
+            except AttributeError:
+                raise ValueError(Errors.E149) from None
+
+        deserializers = {
+            "cfg": lambda p: self.cfg.update(srsly.read_json(p)),
+            "model": load_model,
+            "vocab": lambda p: self.vocab.from_disk(p, exclude=exclude),
+            "trees": lambda p: self.trees.from_disk(p),
+        }
+
+        util.from_disk(path, deserializers, exclude)
+        return self
+
+    def _add_labels(self, labels: Dict):
+        if "labels" not in labels:
+            raise ValueError(Errors.E857.format(name="labels"))
+        if "trees" not in labels:
+            raise ValueError(Errors.E857.format(name="trees"))
+
+        self.cfg["labels"] = list(labels["labels"])
+        trees = []
+        for tree in labels["trees"]:
+            errors = validate_edit_tree(tree)
+            if errors:
+                raise ValueError(Errors.E1026.format(errors="\n".join(errors)))
+
+            tree = dict(tree)
+            if "orig" in tree:
+                tree["orig"] = self.vocab.strings[tree["orig"]]
+            if "orig" in tree:
+                tree["subst"] = self.vocab.strings[tree["subst"]]
+
+            trees.append(tree)
+
+        self.trees.from_json(trees)
+
+        for label, tree in enumerate(self.labels):
+            self.tree2label[tree] = label
+
+    def _labels_from_data(self, get_examples: Callable[[], Iterable[Example]]):
+        # Count corpus tree frequencies in ad-hoc storage to avoid cluttering
+        # the final pipe/string store.
+        vocab = Vocab()
+        trees = EditTrees(vocab.strings)
+        tree_freqs: Counter = Counter()
+        repr_pairs: Dict = {}
+        for example in get_examples():
+            for token in example.reference:
+                if token.lemma != 0:
+                    form = self._get_true_cased_form_of_token(token)
+                    # debug("_labels_from_data", str(token) + "->" + form, token.lemma_)
+                    tree_id = trees.add(form, token.lemma_)
+                    tree_freqs[tree_id] += 1
+                    repr_pairs[tree_id] = (form, token.lemma_)
+
+        # Construct trees that make the frequency cut-off using representative
+        # form - token pairs.
+        for tree_id, freq in tree_freqs.items():
+            if freq >= self.min_tree_freq:
+                form, lemma = repr_pairs[tree_id]
+                self._pair2label(form, lemma, add_label=True)
+
+    @lru_cache()
+    def _get_true_cased_form(self, token: str, is_sent_start: bool, pos: str) -> str:
+        if is_sent_start and pos != "PROPN":
+            return token.lower()
+        else:
+            return token
+
+    def _get_true_cased_form_of_token(self, token: Token) -> str:
+        return self._get_true_cased_form(token.text, token.is_sent_start, token.pos_)
+
+    def _pair2label(self, form, lemma, add_label=False):
+        """
+        Look up the edit tree identifier for a form/label pair. If the edit
+        tree is unknown and "add_label" is set, the edit tree will be added to
+        the labels.
+        """
+        tree_id = self.trees.add(form, lemma)
+        if tree_id not in self.tree2label:
+            if not add_label:
+                return None
+
+            self.tree2label[tree_id] = len(self.cfg["labels"])
+            self.cfg["labels"].append(tree_id)
+        return self.tree2label[tree_id]

hu_core_news_md-any-py3-none-any.whl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b5b32840f8af0edb059f49e7042fc2c3675ae777dbabfcddbf3e294b6d515a75
-size 126883509
+oid sha256:49cf69019ae9ecd344dfa914af018ed5f02263beb6194967a21a85fa66460896
+size 126875360

lemma_postprocessing.py

@@ -0,0 +1,113 @@
+"""
+This module contains various rule-based components aiming to improve on baseline lemmatization tools.
+"""
+
+import re
+from typing import List, Callable
+
+from spacy.lang.hu import Hungarian
+from spacy.pipeline import Pipe
+from spacy.tokens import Token
+from spacy.tokens.doc import Doc
+
+
+@Hungarian.component(
+    "lemma_case_smoother",
+    assigns=["token.lemma"],
+    requires=["token.lemma", "token.pos"],
+)
+def lemma_case_smoother(doc: Doc) -> Doc:
+    """Smooth lemma casing by POS.
+
+    DEPRECATED: This is not needed anymore, as the lemmatizer is now case-insensitive.
+
+    Args:
+        doc (Doc): Input document.
+
+    Returns:
+        Doc: Output document.
+    """
+    for token in doc:
+        if token.is_sent_start and token.tag_ != "PROPN":
+            token.lemma_ = token.lemma_.lower()
+
+    return doc
+
+
+class LemmaSmoother(Pipe):
+    """Smooths lemma by fixing common errors of the edit-tree lemmatizer."""
+
+    _DATE_PATTERN = re.compile(r"(\d+)-j?[éá]?n?a?(t[őó]l)?")
+    _NUMBER_PATTERN = re.compile(r"(\d+([-,/_.:]?(._)?\d+)*%?)")
+
+    # noinspection PyUnusedLocal
+    @staticmethod
+    @Hungarian.factory("lemma_smoother", assigns=["token.lemma"], requires=["token.lemma", "token.pos"])
+    def create_lemma_smoother(nlp: Hungarian, name: str) -> "LemmaSmoother":
+        return LemmaSmoother()
+
+    def __call__(self, doc: Doc) -> Doc:
+        rules: List[Callable] = [
+            self._remove_exclamation_marks,
+            self._remove_question_marks,
+            self._remove_date_suffixes,
+            self._remove_suffix_after_numbers,
+        ]
+
+        for token in doc:
+            for rule in rules:
+                rule(token)
+
+        return doc
+
+    @classmethod
+    def _remove_exclamation_marks(cls, token: Token) -> None:
+        """Removes exclamation marks from the lemma.
+
+        Args:
+            token (Token): The original token.
+        """
+
+        if "!" != token.lemma_:
+            exclamation_mark_index = token.lemma_.find("!")
+            if exclamation_mark_index != -1:
+                token.lemma_ = token.lemma_[:exclamation_mark_index]
+
+    @classmethod
+    def _remove_question_marks(cls, token: Token) -> None:
+        """Removes question marks from the lemma.
+
+        Args:
+            token (Token): The original token.
+        """
+
+        if "?" != token.lemma_:
+            question_mark_index = token.lemma_.find("?")
+            if question_mark_index != -1:
+                token.lemma_ = token.lemma_[:question_mark_index]
+
+    @classmethod
+    def _remove_date_suffixes(cls, token: Token) -> None:
+        """Fixes the suffixes of dates.
+
+        Args:
+            token (Token): The original token.
+        """
+
+        if token.pos_ == "NOUN":
+            match = cls._DATE_PATTERN.match(token.lemma_)
+            if match is not None:
+                token.lemma_ = match.group(1) + "."
+
+    @classmethod
+    def _remove_suffix_after_numbers(cls, token: Token) -> None:
+        """Removes suffixes after numbers.
+
+        Args:
+            token (str): The original token.
+        """
+
+        if token.pos_ == "NUM":
+            match = cls._NUMBER_PATTERN.match(token.text)
+            if match is not None:
+                token.lemma_ = match.group(0)

lemmatizer/model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3541d553cb13886db87e71979c900dbb609ac823aa1c8961e218d9b104eff70c
+oid sha256:e929a0bc8b59054f5ae0fc3ef376ac59a2bc5182a064769ae6a3af5242231489
 size 11282980

lookup_lemmatizer.py

@@ -0,0 +1,132 @@
+import re
+from collections import defaultdict
+from operator import itemgetter
+from pathlib import Path
+from re import Pattern
+from typing import Optional, Callable, Iterable, Dict, Tuple
+
+from spacy.lang.hu import Hungarian
+from spacy.language import Language
+from spacy.lookups import Lookups, Table
+from spacy.pipeline import Pipe
+from spacy.pipeline.lemmatizer import lemmatizer_score
+from spacy.tokens import Token
+from spacy.tokens.doc import Doc
+
+# noinspection PyUnresolvedReferences
+from spacy.training.example import Example
+from spacy.util import ensure_path
+
+
+class LookupLemmatizer(Pipe):
+    """
+    LookupLemmatizer learn `(token, pos, morph. feat) -> lemma` mappings during training, and applies them at prediction
+    time.
+    """
+
+    _number_pattern: Pattern = re.compile(r"\d")
+
+    # noinspection PyUnusedLocal
+    @staticmethod
+    @Hungarian.factory(
+        "lookup_lemmatizer",
+        assigns=["token.lemma"],
+        requires=["token.pos"],
+        default_config={"scorer": {"@scorers": "spacy.lemmatizer_scorer.v1"}, "source": ""},
+    )
+    def create(nlp: Language, name: str, scorer: Optional[Callable], source: str) -> "LookupLemmatizer":
+        return LookupLemmatizer(None, source, scorer)
+
+    def train(self, sentences: Iterable[Iterable[Tuple[str, str, str, str]]], min_occurrences: int = 1) -> None:
+        """
+
+        Args:
+            sentences (Iterable[Iterable[Tuple[str, str, str, str]]]): Sentences to learn the mappings from
+            min_occurrences (int): mapping occurring less than this threshold are not learned
+
+        """
+
+        # Lookup table which maps (upos, form) to (lemma -> frequency),
+        # e.g. `{ ("NOUN", "alma"): { "alma" : 99, "alom": 1} }`
+        lemma_lookup_table: Dict[Tuple[str, str], Dict[str, int]] = defaultdict(lambda: defaultdict(int))
+
+        for sentence in sentences:
+            for token, pos, feats, lemma in sentence:
+                token = self.__mask_numbers(token)
+                lemma = self.__mask_numbers(lemma)
+                feats_str = ("|" + feats) if feats else ""
+                key = (token, pos + feats_str)
+                lemma_lookup_table[key][lemma] += 1
+        lemma_lookup_table = dict(lemma_lookup_table)
+
+        self._lookups = Lookups()
+        table = Table(name="lemma_lookups")
+
+        lemma_freq: Dict[str, int]
+        for (form, pos), lemma_freq in dict(lemma_lookup_table).items():
+            most_freq_lemma, freq = sorted(lemma_freq.items(), key=itemgetter(1), reverse=True)[0]
+            if freq >= min_occurrences:
+                if form not in table:
+                    # lemma by pos
+                    table[form]: Dict[str, str] = dict()
+                table[form][pos] = most_freq_lemma
+
+        self._lookups.set_table(name=f"lemma_lookups", table=table)
+
+    def __init__(
+        self,
+        lookups: Optional[Lookups] = None,
+        source: Optional[str] = None,
+        scorer: Optional[Callable] = lemmatizer_score,
+    ):
+        self._lookups: Optional[Lookups] = lookups
+        self.scorer = scorer
+        self.source = source
+
+    def __call__(self, doc: Doc) -> Doc:
+        assert self._lookups is not None, "Lookup table should be initialized first"
+
+        token: Token
+        for token in doc:
+            lemma_lookup_table = self._lookups.get_table(f"lemma_lookups")
+            masked_token = self.__mask_numbers(token.text)
+
+            if masked_token in lemma_lookup_table:
+                lemma_by_pos: Dict[str, str] = lemma_lookup_table[masked_token]
+                feats_str = ("|" + str(token.morph)) if str(token.morph) else ""
+                key = token.pos_ + feats_str
+                if key in lemma_by_pos:
+                    if masked_token != token.text:
+                        # If the token contains numbers, we need to replace the numbers in the lemma as well
+                        token.lemma_ = self.__replace_numbers(lemma_by_pos[key], token.text)
+                        pass
+                    else:
+                        token.lemma_ = lemma_by_pos[key]
+        return doc
+
+    # noinspection PyUnusedLocal
+    def to_disk(self, path, exclude=tuple()):
+        assert self._lookups is not None, "Lookup table should be initialized first"
+
+        path: Path = ensure_path(path)
+        path.mkdir(exist_ok=True)
+        self._lookups.to_disk(path)
+
+    # noinspection PyUnusedLocal
+    def from_disk(self, path, exclude=tuple()) -> "LookupLemmatizer":
+        path: Path = ensure_path(path)
+        lookups = Lookups()
+        self._lookups = lookups.from_disk(path=path)
+        return self
+
+    def initialize(self, get_examples: Callable[[], Iterable[Example]], *, nlp: Language = None) -> None:
+        lookups = Lookups()
+        self._lookups = lookups.from_disk(path=self.source)
+
+    @classmethod
+    def __mask_numbers(cls, token: str) -> str:
+        return cls._number_pattern.sub("0", token)
+
+    @classmethod
+    def __replace_numbers(cls, lemma: str, token: str) -> str:
+        return cls._number_pattern.sub(lambda match: token[match.start()], lemma)

meta.json

@@ -1,7 +1,7 @@
 {
   "lang":"hu",
   "name":"core_news_md",
-  "version":"3.5.0",
+  "version":"3.5.1",
   "description":"Core Hungarian model for HuSpaCy. Components: tok2vec, senter, tagger, morphologizer, lemmatizer, parser, ner",
   "author":"SzegedAI, MILAB",
   "email":"gyorgy@orosz.link",
@@ -1273,80 +1273,85 @@
     "token_p":0.998565417,
     "token_r":0.9993300153,
     "token_f":0.9989475698,
-    "sents_p":0.9710467706,
-    "sents_r":0.9710467706,
-    "sents_f":0.9710467706,
-    "tag_acc":0.9649265515,
-    "pos_acc":0.9634910761,
-    "morph_acc":0.9308067758,
-    "morph_micro_p":0.9672095642,
-    "morph_micro_r":0.9595616674,
-    "morph_micro_f":0.9633704375,
+    "sents_p":0.9820627803,
+    "sents_r":0.9755011136,
+    "sents_f":0.9787709497,
+    "tag_acc":0.9624844483,
+    "pos_acc":0.9631543688,
+    "morph_acc":0.928892717,
+    "morph_micro_p":0.9648917749,
+    "morph_micro_r":0.9578427159,
+    "morph_micro_f":0.9613543239,
     "morph_per_feat":{
       "Definite":{
-        "p":0.9633363886,
+        "p":0.9589416058,
         "r":0.9808679421,
-        "f":0.9720231214
+        "f":0.9697808535
       },
       "PronType":{
-        "p":0.9740331492,
-        "r":0.9729580574,
-        "f":0.9734953065
+        "p":0.9741331866,
+        "r":0.9768211921,
+        "f":0.9754753376
       },
       "Case":{
-        "p":0.973421263,
-        "r":0.9624580123,
-        "f":0.9679085941
+        "p":0.9733840304,
+        "r":0.9610748864,
+        "f":0.9671902963
       },
       "Degree":{
-        "p":0.9190391459,
-        "r":0.8594009983,
-        "f":0.8882201204
+        "p":0.9179170344,
+        "r":0.8652246256,
+        "f":0.8907922912
       },
       "Number":{
-        "p":0.9858179976,
-        "r":0.9785486844,
-        "f":0.9821698907
+        "p":0.9834515366,
+        "r":0.9760348584,
+        "f":0.9797291614
       },
       "Mood":{
-        "p":0.9290393013,
-        "r":0.94345898,
-        "f":0.9361936194
+        "p":0.9142236699,
+        "r":0.933481153,
+        "f":0.923752057
       },
       "Person":{
-        "p":0.9577114428,
-        "r":0.9498355263,
-        "f":0.9537572254
+        "p":0.9505766063,
+        "r":0.9490131579,
+        "f":0.9497942387
       },
       "Tense":{
-        "p":0.9650655022,
+        "p":0.9598262758,
         "r":0.9767955801,
-        "f":0.9708951126
+        "f":0.9682365827
       },
       "VerbForm":{
-        "p":0.952853598,
-        "r":0.9238171612,
-        "f":0.9381107492
+        "p":0.9554822754,
+        "r":0.9294306335,
+        "f":0.9422764228
       },
       "Voice":{
-        "p":0.9596774194,
-        "r":0.9734151329,
-        "f":0.9664974619
+        "p":0.9519038076,
+        "r":0.9713701431,
+        "f":0.9615384615
       },
       "Number[psor]":{
-        "p":0.9696969697,
-        "r":0.9572649573,
-        "f":0.9634408602
+        "p":0.9719764012,
+        "r":0.9387464387,
+        "f":0.9550724638
       },
       "Person[psor]":{
-        "p":0.9668109668,
-        "r":0.9557774608,
-        "f":0.9612625538
+        "p":0.9705014749,
+        "r":0.9386590585,
+        "f":0.9543147208
       },
       "NumType":{
-        "p":0.9191176471,
-        "r":0.9146341463,
-        "f":0.9168704156
+        "p":0.9209876543,
+        "r":0.9097560976,
+        "f":0.9153374233
+      },
+      "Poss":{
+        "p":0.75,
+        "r":1.0,
+        "f":0.8571428571
       },
       "Reflex":{
         "p":1.0,
@@ -1362,121 +1367,116 @@
         "p":0.0,
         "r":0.0,
         "f":0.0
-      },
-      "Poss":{
-        "p":1.0,
-        "r":1.0,
-        "f":1.0
       }
     },
-    "lemma_acc":0.9738780978,
-    "dep_uas":0.8116901329,
-    "dep_las":0.7417545434,
+    "lemma_acc":0.9728255669,
+    "dep_uas":0.8127597439,
+    "dep_las":0.743681905,
     "dep_las_per_type":{
       "det":{
-        "p":0.8646734854,
-        "r":0.875,
-        "f":0.8698060942
+        "p":0.86328125,
+        "r":0.8797770701,
+        "f":0.8714511041
       },
       "amod:att":{
-        "p":0.8457752256,
-        "r":0.8430089943,
-        "f":0.8443898444
+        "p":0.8241758242,
+        "r":0.8585445626,
+        "f":0.8410092111
       },
       "nsubj":{
-        "p":0.6973293769,
-        "r":0.734375,
-        "f":0.7153729072
+        "p":0.7255813953,
+        "r":0.73125,
+        "f":0.7284046693
       },
       "advmod:mode":{
-        "p":0.5777262181,
+        "p":0.5872641509,
         "r":0.6102941176,
-        "f":0.5935637664
+        "f":0.5985576923
       },
       "nmod:att":{
-        "p":0.7375201288,
-        "r":0.7762711864,
-        "f":0.7563996697
+        "p":0.8083941606,
+        "r":0.7508474576,
+        "f":0.7785588752
       },
       "obl":{
-        "p":0.7434554974,
-        "r":0.7668766877,
-        "f":0.7549844927
+        "p":0.7533632287,
+        "r":0.7560756076,
+        "f":0.7547169811
       },
       "obj":{
-        "p":0.8758949881,
-        "r":0.8247191011,
-        "f":0.849537037
+        "p":0.8513513514,
+        "r":0.8494382022,
+        "f":0.8503937008
       },
       "root":{
-        "p":0.8106904232,
-        "r":0.8106904232,
-        "f":0.8106904232
+        "p":0.8049327354,
+        "r":0.7995545657,
+        "f":0.8022346369
       },
       "cc":{
-        "p":0.6863157895,
-        "r":0.6863157895,
-        "f":0.6863157895
+        "p":0.7052401747,
+        "r":0.68,
+        "f":0.6923901393
       },
       "conj":{
-        "p":0.4556213018,
-        "r":0.48125,
-        "f":0.4680851064
+        "p":0.4658634538,
+        "r":0.4833333333,
+        "f":0.4744376278
       },
       "advmod":{
-        "p":0.847826087,
-        "r":0.8210526316,
-        "f":0.8342245989
+        "p":0.8144329897,
+        "r":0.8315789474,
+        "f":0.8229166667
       },
       "flat:name":{
-        "p":0.8591549296,
-        "r":0.8551401869,
-        "f":0.8571428571
+        "p":0.871559633,
+        "r":0.8878504673,
+        "f":0.8796296296
       },
       "appos":{
-        "p":0.4310344828,
-        "r":0.2659574468,
-        "f":0.3289473684
+        "p":0.3714285714,
+        "r":0.2765957447,
+        "f":0.3170731707
       },
       "advcl":{
-        "p":0.2989690722,
-        "r":0.2959183673,
-        "f":0.2974358974
+        "p":0.3571428571,
+        "r":0.2040816327,
+        "f":0.2597402597
       },
       "advmod:tlocy":{
-        "p":0.6905829596,
-        "r":0.6695652174,
-        "f":0.6799116998
+        "p":0.6991869919,
+        "r":0.747826087,
+        "f":0.7226890756
       },
       "ccomp:obj":{
-        "p":0.2619047619,
+        "p":0.2244897959,
         "r":0.3333333333,
-        "f":0.2933333333
+        "f":0.2682926829
       },
       "mark":{
-        "p":0.8066666667,
-        "r":0.7658227848,
-        "f":0.7857142857
+        "p":0.7884615385,
+        "r":0.7784810127,
+        "f":0.7834394904
       },
       "compound:preverb":{
-        "p":0.9339622642,
-        "r":0.9082568807,
-        "f":0.9209302326
+        "p":0.9509803922,
+        "r":0.8899082569,
+        "f":0.9194312796
       },
       "advmod:locy":{
-        "p":0.75,
-        "r":0.46875,
-        "f":0.5769230769
+        "p":0.7222222222,
+        "r":0.40625,
+        "f":0.52
       },
       "cop":{
-        "p":0.8518518519,
-        "r":0.5609756098,
-        "f":0.6764705882
+        "p":0.7777777778,
+        "r":0.512195122,
+        "f":0.6176470588
       },
       "nmod:obl":{
-        "p":0.2368421053,
-        "r":0.225,
-        "f":0.2307692308
+        "p":0.175,
+        "r":0.175,
+        "f":0.175
       },
       "advmod:to":{
         "p":0.0,
@@ -1484,84 +1484,89 @@
         "f":0.0
       },
       "obj:lvc":{
-        "p":0.3333333333,
+        "p":0.2,
         "r":0.0833333333,
-        "f":0.1333333333
+        "f":0.1176470588
       },
       "ccomp:obl":{
-        "p":0.5,
-        "r":0.4375,
-        "f":0.4666666667
+        "p":0.5238095238,
+        "r":0.34375,
+        "f":0.4150943396
       },
       "iobj":{
-        "p":0.4,
-        "r":0.2666666667,
-        "f":0.32
-      },
-      "dep":{
-        "p":0.0,
-        "r":0.0,
-        "f":0.0
+        "p":0.3,
+        "r":0.2,
+        "f":0.24
       },
-      "xcomp":{
-        "p":0.8611111111,
-        "r":0.8378378378,
-        "f":0.8493150685
+      "acl":{
+        "p":0.2772277228,
+        "r":0.3888888889,
+        "f":0.323699422
       },
       "case":{
-        "p":0.9195979899,
-        "r":0.9336734694,
-        "f":0.9265822785
+        "p":0.9487179487,
+        "r":0.943877551,
+        "f":0.9462915601
       },
       "csubj":{
-        "p":0.6666666667,
-        "r":0.3243243243,
-        "f":0.4363636364
+        "p":0.4827586207,
+        "r":0.3783783784,
+        "f":0.4242424242
       },
       "parataxis":{
-        "p":0.375,
+        "p":0.3913043478,
         "r":0.1232876712,
-        "f":0.1855670103
+        "f":0.1875
+      },
+      "xcomp":{
+        "p":0.84,
+        "r":0.8513513514,
+        "f":0.8456375839
       },
       "nummod":{
-        "p":0.5824175824,
-        "r":0.5698924731,
-        "f":0.5760869565
+        "p":0.5647058824,
+        "r":0.5161290323,
+        "f":0.5393258427
       },
-      "acl":{
-        "p":0.4615384615,
-        "r":0.3333333333,
-        "f":0.3870967742
+      "dep":{
+        "p":0.0,
+        "r":0.0,
+        "f":0.0
+      },
+      "aux":{
+        "p":0.7272727273,
+        "r":0.6666666667,
+        "f":0.6956521739
       },
       "advmod:tto":{
-        "p":0.6666666667,
-        "r":0.2,
-        "f":0.3076923077
+        "p":0.75,
+        "r":0.3,
+        "f":0.4285714286
       },
       "nmod":{
-        "p":0.3333333333,
-        "r":0.0909090909,
-        "f":0.1428571429
-      },
-      "aux":{
-        "p":0.9090909091,
-        "r":0.8333333333,
-        "f":0.8695652174
+        "p":0.0,
+        "r":0.0,
+        "f":0.0
       },
       "advmod:tfrom":{
         "p":0.0,
         "r":0.0,
         "f":0.0
       },
+      "ccomp":{
+        "p":0.0,
+        "r":0.0,
+        "f":0.0
+      },
       "goeswith":{
         "p":0.0,
         "r":0.0,
         "f":0.0
       },
       "compound":{
-        "p":0.9285714286,
+        "p":1.0,
         "r":0.975,
-        "f":0.9512195122
+        "f":0.9873417722
       },
       "obl:lvc":{
         "p":0.0,
@@ -1573,6 +1578,11 @@
         "r":0.0,
         "f":0.0
       },
+      "ccomp:pred":{
+        "p":0.0,
+        "r":0.0,
+        "f":0.0
+      },
       "nsubj:lvc":{
         "p":0.0,
         "r":0.0,
@@ -1583,48 +1593,38 @@
         "r":0.1666666667,
         "f":0.2857142857
       },
-      "ccomp":{
-        "p":0.0,
-        "r":0.0,
-        "f":0.0
-      },
       "advmod:que":{
         "p":1.0,
-        "r":0.75,
-        "f":0.8571428571
-      },
-      "ccomp:pred":{
-        "p":0.0,
-        "r":0.0,
-        "f":0.0
+        "r":0.5,
+        "f":0.6666666667
       }
     },
-    "ents_p":0.8581105169,
-    "ents_r":0.8463431786,
-    "ents_f":0.8521862277,
+    "ents_p":0.8572441922,
+    "ents_r":0.849859353,
+    "ents_f":0.8535357994,
     "ents_per_type":{
       "ORG":{
-        "p":0.8835616438,
-        "r":0.8970792768,
-        "f":0.8902691511
+        "p":0.9027777778,
+        "r":0.8738989337,
+        "f":0.8881036514
       },
       "PER":{
-        "p":0.8852163462,
-        "r":0.8799283154,
-        "f":0.8825644098
+        "p":0.8675042833,
+        "r":0.9074074074,
+        "f":0.8870072993
       },
       "LOC":{
-        "p":0.8632326821,
-        "r":0.84375,
-        "f":0.853380158
+        "p":0.888384755,
+        "r":0.8498263889,
+        "f":0.8686779059
       },
       "MISC":{
-        "p":0.6888888889,
-        "r":0.6156028369,
-        "f":0.6501872659
+        "p":0.6461318052,
+        "r":0.6397163121,
+        "f":0.6429080542
       }
     },
-    "speed":1651.4495157666
+    "speed":2535.2452470079
   },
   "sources":[
     {

morphologizer/model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:aebba307a814e36fb5d32de0e25f4315867f4678e73ff2e85aceb5c41d3c0af3
+oid sha256:56ea873c3ffc818958ecd60553379d277ae3b21f74170486ccfaf6f6d60d563f
 size 463022

ner/model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6404b8918a3691cadb17f69cd8fa4bccff7aff4b77ceb8e4dfbe2e3bc9d12a2c
+oid sha256:0f02a2f28c88dfb50af9a8555bbe2929abb3c9a5cf1d29d3b34a91526988b0c2
 size 9791307

parser/model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:075883bd84113bcce6b9a425ee55d44b21fc964b2d2515b6625082872fab2195
+oid sha256:846779dc9fa38c7dae2730aaf1dc8a99bd24f62aa9ec283427e5382343422284
 size 25601129

senter/model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:570ee5f2927cf1603436844338d191ec23ca484b811796c9df37d74dde80e0a6
+oid sha256:dd16a43ec8c789c555386afbb199d4444466c2181c3dc5c4de56c9ca2b57685a
 size 1237

tagger/model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:40bea158134db0033f7559059533e4c2d64792c1cc934a7fb4f414ed0c67ed28
+oid sha256:04d3ab9953b81bf955e264f667dd1eacdf2ff3b319598560680df13b5ac80f75
 size 7297

tok2vec/model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7302c83d08f9da7b5388bb1f505e84ad0fb0125d4ac17c4ff3fc683d697400c9
+oid sha256:eb31d7b818409d19f80994951eabe64fc35391f363623cc461f61f2fffc39b4f
 size 9659749

vocab/strings.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1296dbf1d9d909f4b2521e29174cab614107fe3ab0ed196ba474ee0c59101c5d
-size 6405774
+oid sha256:7a7cffc79e121b8b25771ee3d13f9f35f7f2af63ee8cbb354d0ece1fdf03cf78
+size 6405688