Update spacy pipeline to 3.5.1

README.md

@@ -14,69 +14,69 @@ model-index:
     metrics:
     - name: NER Precision
       type: precision
-      value: 0.8662957645
+      value: 0.861328125
     - name: NER Recall
       type: recall
-      value: 0.848628692
+      value: 0.8528481013
     - name: NER F Score
       type: f_score
-      value: 0.8573712256
+      value: 0.8570671378
   - task:
       name: TAG
       type: token-classification
     metrics:
     - name: TAG (XPOS) Accuracy
       type: accuracy
-      value: 0.9643028041
+      value: 0.9680845973
   - task:
       name: POS
       type: token-classification
     metrics:
     - name: POS (UPOS) Accuracy
       type: accuracy
-      value: 0.9634414777
+      value: 0.9686587875
   - task:
       name: MORPH
       type: token-classification
     metrics:
     - name: Morph (UFeats) Accuracy
       type: accuracy
-      value: 0.9310938846
+      value: 0.9363127422
   - task:
       name: LEMMA
       type: token-classification
     metrics:
     - name: Lemma Accuracy
       type: accuracy
-      value: 0.9722514592
+      value: 0.9747392594
   - task:
       name: UNLABELED_DEPENDENCIES
       type: token-classification
     metrics:
     - name: Unlabeled Attachment Score (UAS)
       type: f_score
-      value: 0.8222334626
+      value: 0.8158633861
   - task:
       name: LABELED_DEPENDENCIES
       type: token-classification
     metrics:
     - name: Labeled Attachment Score (LAS)
       type: f_score
-      value: 0.75479121
+      value: 0.7489046175
   - task:
       name: SENTS
       type: token-classification
     metrics:
     - name: Sentences F-Score
       type: f_score
-      value: 0.9753363229
+      value: 0.983277592
 ---
 Core Hungarian model for HuSpaCy. Components: tok2vec, senter, tagger, morphologizer, lemmatizer, parser, ner
 
 | Feature | Description |
 | --- | --- |
 | **Name** | `hu_core_news_lg` |
-| **Version** | `3.5.0` |
+| **Version** | `3.5.1` |
 | **spaCy** | `>=3.5.0,<3.6.0` |
 | **Default Pipeline** | `tok2vec`, `senter`, `tagger`, `morphologizer`, `lookup_lemmatizer`, `lemmatizer`, `lemma_smoother`, `parser`, `ner` |
 | **Components** | `tok2vec`, `senter`, `tagger`, `morphologizer`, `lookup_lemmatizer`, `lemmatizer`, `lemma_smoother`, `parser`, `ner` |
@@ -108,18 +108,18 @@ Core Hungarian model for HuSpaCy. Components: tok2vec, senter, tagger, morpholog
 | `TOKEN_P` | 99.86 |
 | `TOKEN_R` | 99.93 |
 | `TOKEN_F` | 99.89 |
-| `SENTS_P` | 98.00 |
-| `SENTS_R` | 98.00 |
-| `SENTS_F` | 98.00 |
-| `TAG_ACC` | 96.76 |
-| `POS_ACC` | 96.62 |
-| `MORPH_ACC` | 93.54 |
-| `MORPH_MICRO_P` | 96.68 |
-| `MORPH_MICRO_R` | 96.24 |
-| `MORPH_MICRO_F` | 96.46 |
-| `LEMMA_ACC` | 97.33 |
-| `DEP_UAS` | 81.87 |
-| `DEP_LAS` | 74.99 |
-| `ENTS_P` | 86.26 |
-| `ENTS_R` | 85.76 |
-| `ENTS_F` | 86.01 |
+| `SENTS_P` | 98.44 |
+| `SENTS_R` | 98.22 |
+| `SENTS_F` | 98.33 |
+| `TAG_ACC` | 96.81 |
+| `POS_ACC` | 96.87 |
+| `MORPH_ACC` | 93.63 |
+| `MORPH_MICRO_P` | 96.93 |
+| `MORPH_MICRO_R` | 96.36 |
+| `MORPH_MICRO_F` | 96.65 |
+| `LEMMA_ACC` | 97.47 |
+| `DEP_UAS` | 81.59 |
+| `DEP_LAS` | 74.89 |
+| `ENTS_P` | 86.13 |
+| `ENTS_R` | 85.28 |
+| `ENTS_F` | 85.71 |

config.cfg

@@ -1,8 +1,8 @@
 [paths]
-parser_model = "models/hu_core_news_lg-parser-3.5.0/model-best"
-ner_model = "models/hu_core_news_lg-ner-3.5.0/model-best"
-lemmatizer_lookups = "models/hu_core_news_lg-lookup-lemmatizer-3.5.0"
-tagger_model = "models/hu_core_news_lg-tagger-3.5.0/model-best"
+parser_model = "models/hu_core_news_lg-parser-3.5.1/model-best"
+ner_model = "models/hu_core_news_lg-ner-3.5.1/model-best"
+lemmatizer_lookups = "models/hu_core_news_lg-lookup-lemmatizer-3.5.1"
+tagger_model = "models/hu_core_news_lg-tagger-3.5.1/model-best"
 train = null
 dev = null
 vectors = null

edit_tree_lemmatizer.py

@@ -0,0 +1,465 @@
+from functools import lru_cache
+
+from typing import cast, Any, Callable, Dict, Iterable, List, Optional
+from typing import Sequence, Tuple, Union
+from collections import Counter
+from copy import deepcopy
+from itertools import islice
+import numpy as np
+
+import srsly
+from thinc.api import Config, Model, SequenceCategoricalCrossentropy, NumpyOps
+from thinc.types import Floats2d, Ints2d
+
+from spacy.pipeline._edit_tree_internals.edit_trees import EditTrees
+from spacy.pipeline._edit_tree_internals.schemas import validate_edit_tree
+from spacy.pipeline.lemmatizer import lemmatizer_score
+from spacy.pipeline.trainable_pipe import TrainablePipe
+from spacy.errors import Errors
+from spacy.language import Language
+from spacy.tokens import Doc, Token
+from spacy.training import Example, validate_examples, validate_get_examples
+from spacy.vocab import Vocab
+from spacy import util
+
+
+TOP_K_GUARDRAIL = 20
+
+
+default_model_config = """
+[model]
+@architectures = "spacy.Tagger.v2"
+
+[model.tok2vec]
+@architectures = "spacy.HashEmbedCNN.v2"
+pretrained_vectors = null
+width = 96
+depth = 4
+embed_size = 2000
+window_size = 1
+maxout_pieces = 3
+subword_features = true
+"""
+DEFAULT_EDIT_TREE_LEMMATIZER_MODEL = Config().from_str(default_model_config)["model"]
+
+
+@Language.factory(
+    "trainable_lemmatizer_v2",
+    assigns=["token.lemma"],
+    requires=[],
+    default_config={
+        "model": DEFAULT_EDIT_TREE_LEMMATIZER_MODEL,
+        "backoff": "orth",
+        "min_tree_freq": 3,
+        "overwrite": False,
+        "top_k": 1,
+        "overwrite_labels": True,
+        "scorer": {"@scorers": "spacy.lemmatizer_scorer.v1"},
+    },
+    default_score_weights={"lemma_acc": 1.0},
+)
+def make_edit_tree_lemmatizer(
+    nlp: Language,
+    name: str,
+    model: Model,
+    backoff: Optional[str],
+    min_tree_freq: int,
+    overwrite: bool,
+    top_k: int,
+    overwrite_labels: bool,
+    scorer: Optional[Callable],
+):
+    """Construct an EditTreeLemmatizer component."""
+    return EditTreeLemmatizer(
+        nlp.vocab,
+        model,
+        name,
+        backoff=backoff,
+        min_tree_freq=min_tree_freq,
+        overwrite=overwrite,
+        top_k=top_k,
+        overwrite_labels=overwrite_labels,
+        scorer=scorer,
+    )
+
+
+# _f = open("lemmatizer.log", "w")
+# def debug(*args):
+#     _f.write(" ".join(args) + "\n")
+def debug(*args):
+    pass
+
+
+class EditTreeLemmatizer(TrainablePipe):
+    """
+    Lemmatizer that lemmatizes each word using a predicted edit tree.
+    """
+
+    def __init__(
+        self,
+        vocab: Vocab,
+        model: Model,
+        name: str = "trainable_lemmatizer",
+        *,
+        backoff: Optional[str] = "orth",
+        min_tree_freq: int = 3,
+        overwrite: bool = False,
+        top_k: int = 1,
+        overwrite_labels,
+        scorer: Optional[Callable] = lemmatizer_score,
+    ):
+        """
+        Construct an edit tree lemmatizer.
+
+        backoff (Optional[str]): backoff to use when the predicted edit trees
+            are not applicable. Must be an attribute of Token or None (leave the
+            lemma unset).
+        min_tree_freq (int): prune trees that are applied less than this
+            frequency in the training data.
+        overwrite (bool): overwrite existing lemma annotations.
+        top_k (int): try to apply at most the k most probable edit trees.
+        """
+        self.vocab = vocab
+        self.model = model
+        self.name = name
+        self.backoff = backoff
+        self.min_tree_freq = min_tree_freq
+        self.overwrite = overwrite
+        self.top_k = top_k
+        self.overwrite_labels = overwrite_labels
+
+        self.trees = EditTrees(self.vocab.strings)
+        self.tree2label: Dict[int, int] = {}
+
+        self.cfg: Dict[str, Any] = {"labels": []}
+        self.scorer = scorer
+        self.numpy_ops = NumpyOps()
+
+    def get_loss(
+        self, examples: Iterable[Example], scores: List[Floats2d]
+    ) -> Tuple[float, List[Floats2d]]:
+        validate_examples(examples, "EditTreeLemmatizer.get_loss")
+        loss_func = SequenceCategoricalCrossentropy(normalize=False, missing_value=-1)
+
+        truths = []
+        for eg in examples:
+            eg_truths = []
+            for (predicted, gold_lemma, gold_pos, gold_sent_start) in zip(
+                eg.predicted,
+                eg.get_aligned("LEMMA", as_string=True),
+                eg.get_aligned("POS", as_string=True),
+                eg.get_aligned_sent_starts(),
+            ):
+                if gold_lemma is None:
+                    label = -1
+                else:
+                    form = self._get_true_cased_form(
+                        predicted.text, gold_sent_start, gold_pos
+                    )
+                    tree_id = self.trees.add(form, gold_lemma)
+                    # debug(f"@get_loss: {predicted}/{gold_pos}[{gold_sent_start}]->{form}|{gold_lemma}[{tree_id}]")
+                    label = self.tree2label.get(tree_id, 0)
+                eg_truths.append(label)
+
+            truths.append(eg_truths)
+
+        d_scores, loss = loss_func(scores, truths)
+        if self.model.ops.xp.isnan(loss):
+            raise ValueError(Errors.E910.format(name=self.name))
+
+        return float(loss), d_scores
+
+    def predict(self, docs: Iterable[Doc]) -> List[Ints2d]:
+        if self.top_k == 1:
+            scores2guesses = self._scores2guesses_top_k_equals_1
+        elif self.top_k <= TOP_K_GUARDRAIL:
+            scores2guesses = self._scores2guesses_top_k_greater_1
+        else:
+            scores2guesses = self._scores2guesses_top_k_guardrail
+        # The behaviour of *_scores2guesses_top_k_greater_1()* is efficient for values
+        # of *top_k>1* that are likely to be useful when the edit tree lemmatizer is used
+        # for its principal purpose of lemmatizing tokens. However, the code could also
+        # be used for other purposes, and with very large values of *top_k* the method
+        # becomes inefficient. In such cases, *_scores2guesses_top_k_guardrail()* is used
+        # instead.
+        n_docs = len(list(docs))
+        if not any(len(doc) for doc in docs):
+            # Handle cases where there are no tokens in any docs.
+            n_labels = len(self.cfg["labels"])
+            guesses: List[Ints2d] = [self.model.ops.alloc2i(0, n_labels) for _ in docs]
+            assert len(guesses) == n_docs
+            return guesses
+        scores = self.model.predict(docs)
+        assert len(scores) == n_docs
+        guesses = scores2guesses(docs, scores)
+        assert len(guesses) == n_docs
+        return guesses
+
+    def _scores2guesses_top_k_equals_1(self, docs, scores):
+        guesses = []
+        for doc, doc_scores in zip(docs, scores):
+            doc_guesses = doc_scores.argmax(axis=1)
+            doc_guesses = self.numpy_ops.asarray(doc_guesses)
+
+            doc_compat_guesses = []
+            for i, token in enumerate(doc):
+                tree_id = self.cfg["labels"][doc_guesses[i]]
+                form: str = self._get_true_cased_form_of_token(token)
+                if self.trees.apply(tree_id, form) is not None:
+                    doc_compat_guesses.append(tree_id)
+                else:
+                    doc_compat_guesses.append(-1)
+            guesses.append(np.array(doc_compat_guesses))
+
+        return guesses
+
+    def _scores2guesses_top_k_greater_1(self, docs, scores):
+        guesses = []
+        top_k = min(self.top_k, len(self.labels))
+        for doc, doc_scores in zip(docs, scores):
+            doc_scores = self.numpy_ops.asarray(doc_scores)
+            doc_compat_guesses = []
+            for i, token in enumerate(doc):
+                for _ in range(top_k):
+                    candidate = int(doc_scores[i].argmax())
+                    candidate_tree_id = self.cfg["labels"][candidate]
+                    form: str = self._get_true_cased_form_of_token(token)
+                    if self.trees.apply(candidate_tree_id, form) is not None:
+                        doc_compat_guesses.append(candidate_tree_id)
+                        break
+                    doc_scores[i, candidate] = np.finfo(np.float32).min
+                else:
+                    doc_compat_guesses.append(-1)
+            guesses.append(np.array(doc_compat_guesses))
+
+        return guesses
+
+    def _scores2guesses_top_k_guardrail(self, docs, scores):
+        guesses = []
+        for doc, doc_scores in zip(docs, scores):
+            doc_guesses = np.argsort(doc_scores)[..., : -self.top_k - 1 : -1]
+            doc_guesses = self.numpy_ops.asarray(doc_guesses)
+
+            doc_compat_guesses = []
+            for token, candidates in zip(doc, doc_guesses):
+                tree_id = -1
+                for candidate in candidates:
+                    candidate_tree_id = self.cfg["labels"][candidate]
+
+                    form: str = self._get_true_cased_form_of_token(token)
+
+                    if self.trees.apply(candidate_tree_id, form) is not None:
+                        tree_id = candidate_tree_id
+                        break
+                doc_compat_guesses.append(tree_id)
+
+            guesses.append(np.array(doc_compat_guesses))
+
+        return guesses
+
+    def set_annotations(self, docs: Iterable[Doc], batch_tree_ids):
+        for i, doc in enumerate(docs):
+            doc_tree_ids = batch_tree_ids[i]
+            if hasattr(doc_tree_ids, "get"):
+                doc_tree_ids = doc_tree_ids.get()
+            for j, tree_id in enumerate(doc_tree_ids):
+                if self.overwrite or doc[j].lemma == 0:
+                    # If no applicable tree could be found during prediction,
+                    # the special identifier -1 is used. Otherwise the tree
+                    # is guaranteed to be applicable.
+                    if tree_id == -1:
+                        if self.backoff is not None:
+                            doc[j].lemma = getattr(doc[j], self.backoff)
+                    else:
+                        form = self._get_true_cased_form_of_token(doc[j])
+                        lemma = self.trees.apply(tree_id, form) or form
+                        # debug(f"@set_annotations: {doc[j]}/{doc[j].pos_}[{doc[j].is_sent_start}]->{form}|{lemma}[{tree_id}]")
+                        doc[j].lemma_ = lemma
+
+    @property
+    def labels(self) -> Tuple[int, ...]:
+        """Returns the labels currently added to the component."""
+        return tuple(self.cfg["labels"])
+
+    @property
+    def hide_labels(self) -> bool:
+        return True
+
+    @property
+    def label_data(self) -> Dict:
+        trees = []
+        for tree_id in range(len(self.trees)):
+            tree = self.trees[tree_id]
+            if "orig" in tree:
+                tree["orig"] = self.vocab.strings[tree["orig"]]
+            if "subst" in tree:
+                tree["subst"] = self.vocab.strings[tree["subst"]]
+            trees.append(tree)
+        return dict(trees=trees, labels=tuple(self.cfg["labels"]))
+
+    def initialize(
+        self,
+        get_examples: Callable[[], Iterable[Example]],
+        *,
+        nlp: Optional[Language] = None,
+        labels: Optional[Dict] = None,
+    ):
+        validate_get_examples(get_examples, "EditTreeLemmatizer.initialize")
+
+        if self.overwrite_labels:
+            if labels is None:
+                self._labels_from_data(get_examples)
+            else:
+                self._add_labels(labels)
+
+        # Sample for the model.
+        doc_sample = []
+        label_sample = []
+        for example in islice(get_examples(), 10):
+            doc_sample.append(example.x)
+            gold_labels: List[List[float]] = []
+            for token in example.reference:
+                if token.lemma == 0:
+                    gold_label = None
+                else:
+                    gold_label = self._pair2label(token.text, token.lemma_)
+
+                gold_labels.append(
+                    [
+                        1.0 if label == gold_label else 0.0
+                        for label in self.cfg["labels"]
+                    ]
+                )
+
+            gold_labels = cast(Floats2d, gold_labels)
+            label_sample.append(self.model.ops.asarray(gold_labels, dtype="float32"))
+
+        self._require_labels()
+        assert len(doc_sample) > 0, Errors.E923.format(name=self.name)
+        assert len(label_sample) > 0, Errors.E923.format(name=self.name)
+
+        self.model.initialize(X=doc_sample, Y=label_sample)
+
+    def from_bytes(self, bytes_data, *, exclude=tuple()):
+        deserializers = {
+            "cfg": lambda b: self.cfg.update(srsly.json_loads(b)),
+            "model": lambda b: self.model.from_bytes(b),
+            "vocab": lambda b: self.vocab.from_bytes(b, exclude=exclude),
+            "trees": lambda b: self.trees.from_bytes(b),
+        }
+
+        util.from_bytes(bytes_data, deserializers, exclude)
+
+        return self
+
+    def to_bytes(self, *, exclude=tuple()):
+        serializers = {
+            "cfg": lambda: srsly.json_dumps(self.cfg),
+            "model": lambda: self.model.to_bytes(),
+            "vocab": lambda: self.vocab.to_bytes(exclude=exclude),
+            "trees": lambda: self.trees.to_bytes(),
+        }
+
+        return util.to_bytes(serializers, exclude)
+
+    def to_disk(self, path, exclude=tuple()):
+        path = util.ensure_path(path)
+        serializers = {
+            "cfg": lambda p: srsly.write_json(p, self.cfg),
+            "model": lambda p: self.model.to_disk(p),
+            "vocab": lambda p: self.vocab.to_disk(p, exclude=exclude),
+            "trees": lambda p: self.trees.to_disk(p),
+        }
+        util.to_disk(path, serializers, exclude)
+
+    def from_disk(self, path, exclude=tuple()):
+        def load_model(p):
+            try:
+                with open(p, "rb") as mfile:
+                    self.model.from_bytes(mfile.read())
+            except AttributeError:
+                raise ValueError(Errors.E149) from None
+
+        deserializers = {
+            "cfg": lambda p: self.cfg.update(srsly.read_json(p)),
+            "model": load_model,
+            "vocab": lambda p: self.vocab.from_disk(p, exclude=exclude),
+            "trees": lambda p: self.trees.from_disk(p),
+        }
+
+        util.from_disk(path, deserializers, exclude)
+        return self
+
+    def _add_labels(self, labels: Dict):
+        if "labels" not in labels:
+            raise ValueError(Errors.E857.format(name="labels"))
+        if "trees" not in labels:
+            raise ValueError(Errors.E857.format(name="trees"))
+
+        self.cfg["labels"] = list(labels["labels"])
+        trees = []
+        for tree in labels["trees"]:
+            errors = validate_edit_tree(tree)
+            if errors:
+                raise ValueError(Errors.E1026.format(errors="\n".join(errors)))
+
+            tree = dict(tree)
+            if "orig" in tree:
+                tree["orig"] = self.vocab.strings[tree["orig"]]
+            if "orig" in tree:
+                tree["subst"] = self.vocab.strings[tree["subst"]]
+
+            trees.append(tree)
+
+        self.trees.from_json(trees)
+
+        for label, tree in enumerate(self.labels):
+            self.tree2label[tree] = label
+
+    def _labels_from_data(self, get_examples: Callable[[], Iterable[Example]]):
+        # Count corpus tree frequencies in ad-hoc storage to avoid cluttering
+        # the final pipe/string store.
+        vocab = Vocab()
+        trees = EditTrees(vocab.strings)
+        tree_freqs: Counter = Counter()
+        repr_pairs: Dict = {}
+        for example in get_examples():
+            for token in example.reference:
+                if token.lemma != 0:
+                    form = self._get_true_cased_form_of_token(token)
+                    # debug("_labels_from_data", str(token) + "->" + form, token.lemma_)
+                    tree_id = trees.add(form, token.lemma_)
+                    tree_freqs[tree_id] += 1
+                    repr_pairs[tree_id] = (form, token.lemma_)
+
+        # Construct trees that make the frequency cut-off using representative
+        # form - token pairs.
+        for tree_id, freq in tree_freqs.items():
+            if freq >= self.min_tree_freq:
+                form, lemma = repr_pairs[tree_id]
+                self._pair2label(form, lemma, add_label=True)
+
+    @lru_cache()
+    def _get_true_cased_form(self, token: str, is_sent_start: bool, pos: str) -> str:
+        if is_sent_start and pos != "PROPN":
+            return token.lower()
+        else:
+            return token
+
+    def _get_true_cased_form_of_token(self, token: Token) -> str:
+        return self._get_true_cased_form(token.text, token.is_sent_start, token.pos_)
+
+    def _pair2label(self, form, lemma, add_label=False):
+        """
+        Look up the edit tree identifier for a form/label pair. If the edit
+        tree is unknown and "add_label" is set, the edit tree will be added to
+        the labels.
+        """
+        tree_id = self.trees.add(form, lemma)
+        if tree_id not in self.tree2label:
+            if not add_label:
+                return None
+
+            self.tree2label[tree_id] = len(self.cfg["labels"])
+            self.cfg["labels"].append(tree_id)
+        return self.tree2label[tree_id]

hu_core_news_lg-any-py3-none-any.whl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:68356766408dd914bc61b88be6ef02c4c237fb979b9e107835aa1928261d0bd6
-size 401362147
+oid sha256:266ecaf2bc079609a5f8d8438e98d690b8a63e560f0a8d5af1bfb8ce24a9ff02
+size 401249360

lemma_postprocessing.py

@@ -0,0 +1,113 @@
+"""
+This module contains various rule-based components aiming to improve on baseline lemmatization tools.
+"""
+
+import re
+from typing import List, Callable
+
+from spacy.lang.hu import Hungarian
+from spacy.pipeline import Pipe
+from spacy.tokens import Token
+from spacy.tokens.doc import Doc
+
+
+@Hungarian.component(
+    "lemma_case_smoother",
+    assigns=["token.lemma"],
+    requires=["token.lemma", "token.pos"],
+)
+def lemma_case_smoother(doc: Doc) -> Doc:
+    """Smooth lemma casing by POS.
+
+    DEPRECATED: This is not needed anymore, as the lemmatizer is now case-insensitive.
+
+    Args:
+        doc (Doc): Input document.
+
+    Returns:
+        Doc: Output document.
+    """
+    for token in doc:
+        if token.is_sent_start and token.tag_ != "PROPN":
+            token.lemma_ = token.lemma_.lower()
+
+    return doc
+
+
+class LemmaSmoother(Pipe):
+    """Smooths lemma by fixing common errors of the edit-tree lemmatizer."""
+
+    _DATE_PATTERN = re.compile(r"(\d+)-j?[éá]?n?a?(t[őó]l)?")
+    _NUMBER_PATTERN = re.compile(r"(\d+([-,/_.:]?(._)?\d+)*%?)")
+
+    # noinspection PyUnusedLocal
+    @staticmethod
+    @Hungarian.factory("lemma_smoother", assigns=["token.lemma"], requires=["token.lemma", "token.pos"])
+    def create_lemma_smoother(nlp: Hungarian, name: str) -> "LemmaSmoother":
+        return LemmaSmoother()
+
+    def __call__(self, doc: Doc) -> Doc:
+        rules: List[Callable] = [
+            self._remove_exclamation_marks,
+            self._remove_question_marks,
+            self._remove_date_suffixes,
+            self._remove_suffix_after_numbers,
+        ]
+
+        for token in doc:
+            for rule in rules:
+                rule(token)
+
+        return doc
+
+    @classmethod
+    def _remove_exclamation_marks(cls, token: Token) -> None:
+        """Removes exclamation marks from the lemma.
+
+        Args:
+            token (Token): The original token.
+        """
+
+        if "!" != token.lemma_:
+            exclamation_mark_index = token.lemma_.find("!")
+            if exclamation_mark_index != -1:
+                token.lemma_ = token.lemma_[:exclamation_mark_index]
+
+    @classmethod
+    def _remove_question_marks(cls, token: Token) -> None:
+        """Removes question marks from the lemma.
+
+        Args:
+            token (Token): The original token.
+        """
+
+        if "?" != token.lemma_:
+            question_mark_index = token.lemma_.find("?")
+            if question_mark_index != -1:
+                token.lemma_ = token.lemma_[:question_mark_index]
+
+    @classmethod
+    def _remove_date_suffixes(cls, token: Token) -> None:
+        """Fixes the suffixes of dates.
+
+        Args:
+            token (Token): The original token.
+        """
+
+        if token.pos_ == "NOUN":
+            match = cls._DATE_PATTERN.match(token.lemma_)
+            if match is not None:
+                token.lemma_ = match.group(1) + "."
+
+    @classmethod
+    def _remove_suffix_after_numbers(cls, token: Token) -> None:
+        """Removes suffixes after numbers.
+
+        Args:
+            token (str): The original token.
+        """
+
+        if token.pos_ == "NUM":
+            match = cls._NUMBER_PATTERN.match(token.text)
+            if match is not None:
+                token.lemma_ = match.group(0)

lemmatizer/model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bc78b274c26afb6cdc046ef08e700eadd5ac67afccfb637f3c9fcdeda2d2f8d3
+oid sha256:e0e69dbbfcec14a02ecead9533398c131c27b19821afddf82abfe54f014452c6
 size 61643136

lookup_lemmatizer.py

@@ -0,0 +1,132 @@
+import re
+from collections import defaultdict
+from operator import itemgetter
+from pathlib import Path
+from re import Pattern
+from typing import Optional, Callable, Iterable, Dict, Tuple
+
+from spacy.lang.hu import Hungarian
+from spacy.language import Language
+from spacy.lookups import Lookups, Table
+from spacy.pipeline import Pipe
+from spacy.pipeline.lemmatizer import lemmatizer_score
+from spacy.tokens import Token
+from spacy.tokens.doc import Doc
+
+# noinspection PyUnresolvedReferences
+from spacy.training.example import Example
+from spacy.util import ensure_path
+
+
+class LookupLemmatizer(Pipe):
+    """
+    LookupLemmatizer learn `(token, pos, morph. feat) -> lemma` mappings during training, and applies them at prediction
+    time.
+    """
+
+    _number_pattern: Pattern = re.compile(r"\d")
+
+    # noinspection PyUnusedLocal
+    @staticmethod
+    @Hungarian.factory(
+        "lookup_lemmatizer",
+        assigns=["token.lemma"],
+        requires=["token.pos"],
+        default_config={"scorer": {"@scorers": "spacy.lemmatizer_scorer.v1"}, "source": ""},
+    )
+    def create(nlp: Language, name: str, scorer: Optional[Callable], source: str) -> "LookupLemmatizer":
+        return LookupLemmatizer(None, source, scorer)
+
+    def train(self, sentences: Iterable[Iterable[Tuple[str, str, str, str]]], min_occurrences: int = 1) -> None:
+        """
+
+        Args:
+            sentences (Iterable[Iterable[Tuple[str, str, str, str]]]): Sentences to learn the mappings from
+            min_occurrences (int): mapping occurring less than this threshold are not learned
+
+        """
+
+        # Lookup table which maps (upos, form) to (lemma -> frequency),
+        # e.g. `{ ("NOUN", "alma"): { "alma" : 99, "alom": 1} }`
+        lemma_lookup_table: Dict[Tuple[str, str], Dict[str, int]] = defaultdict(lambda: defaultdict(int))
+
+        for sentence in sentences:
+            for token, pos, feats, lemma in sentence:
+                token = self.__mask_numbers(token)
+                lemma = self.__mask_numbers(lemma)
+                feats_str = ("|" + feats) if feats else ""
+                key = (token, pos + feats_str)
+                lemma_lookup_table[key][lemma] += 1
+        lemma_lookup_table = dict(lemma_lookup_table)
+
+        self._lookups = Lookups()
+        table = Table(name="lemma_lookups")
+
+        lemma_freq: Dict[str, int]
+        for (form, pos), lemma_freq in dict(lemma_lookup_table).items():
+            most_freq_lemma, freq = sorted(lemma_freq.items(), key=itemgetter(1), reverse=True)[0]
+            if freq >= min_occurrences:
+                if form not in table:
+                    # lemma by pos
+                    table[form]: Dict[str, str] = dict()
+                table[form][pos] = most_freq_lemma
+
+        self._lookups.set_table(name=f"lemma_lookups", table=table)
+
+    def __init__(
+        self,
+        lookups: Optional[Lookups] = None,
+        source: Optional[str] = None,
+        scorer: Optional[Callable] = lemmatizer_score,
+    ):
+        self._lookups: Optional[Lookups] = lookups
+        self.scorer = scorer
+        self.source = source
+
+    def __call__(self, doc: Doc) -> Doc:
+        assert self._lookups is not None, "Lookup table should be initialized first"
+
+        token: Token
+        for token in doc:
+            lemma_lookup_table = self._lookups.get_table(f"lemma_lookups")
+            masked_token = self.__mask_numbers(token.text)
+
+            if masked_token in lemma_lookup_table:
+                lemma_by_pos: Dict[str, str] = lemma_lookup_table[masked_token]
+                feats_str = ("|" + str(token.morph)) if str(token.morph) else ""
+                key = token.pos_ + feats_str
+                if key in lemma_by_pos:
+                    if masked_token != token.text:
+                        # If the token contains numbers, we need to replace the numbers in the lemma as well
+                        token.lemma_ = self.__replace_numbers(lemma_by_pos[key], token.text)
+                        pass
+                    else:
+                        token.lemma_ = lemma_by_pos[key]
+        return doc
+
+    # noinspection PyUnusedLocal
+    def to_disk(self, path, exclude=tuple()):
+        assert self._lookups is not None, "Lookup table should be initialized first"
+
+        path: Path = ensure_path(path)
+        path.mkdir(exist_ok=True)
+        self._lookups.to_disk(path)
+
+    # noinspection PyUnusedLocal
+    def from_disk(self, path, exclude=tuple()) -> "LookupLemmatizer":
+        path: Path = ensure_path(path)
+        lookups = Lookups()
+        self._lookups = lookups.from_disk(path=path)
+        return self
+
+    def initialize(self, get_examples: Callable[[], Iterable[Example]], *, nlp: Language = None) -> None:
+        lookups = Lookups()
+        self._lookups = lookups.from_disk(path=self.source)
+
+    @classmethod
+    def __mask_numbers(cls, token: str) -> str:
+        return cls._number_pattern.sub("0", token)
+
+    @classmethod
+    def __replace_numbers(cls, lemma: str, token: str) -> str:
+        return cls._number_pattern.sub(lambda match: token[match.start()], lemma)

meta.json

@@ -1,7 +1,7 @@
 {
   "lang":"hu",
   "name":"core_news_lg",
-  "version":"3.5.0",
+  "version":"3.5.1",
   "description":"Core Hungarian model for HuSpaCy. Components: tok2vec, senter, tagger, morphologizer, lemmatizer, parser, ner",
   "author":"SzegedAI, MILAB",
   "email":"gyorgy@orosz.link",
@@ -1273,85 +1273,85 @@
     "token_p":0.998565417,
     "token_r":0.9993300153,
     "token_f":0.9989475698,
-    "sents_p":0.9819413093,
-    "sents_r":0.9688195991,
-    "sents_f":0.9753363229,
-    "tag_acc":0.9643028041,
-    "pos_acc":0.9634414777,
-    "morph_acc":0.9310938846,
-    "morph_micro_p":0.9679604064,
-    "morph_micro_r":0.9581435324,
-    "morph_micro_f":0.9630269523,
+    "sents_p":0.984375,
+    "sents_r":0.9821826281,
+    "sents_f":0.983277592,
+    "tag_acc":0.9680845973,
+    "pos_acc":0.9686587875,
+    "morph_acc":0.9363127422,
+    "morph_micro_p":0.9693092418,
+    "morph_micro_r":0.9636441771,
+    "morph_micro_f":0.9664684079,
     "morph_per_feat":{
       "Definite":{
-        "p":0.9642201835,
-        "r":0.9808679421,
-        "f":0.9724728198
+        "p":0.9693877551,
+        "r":0.9752683154,
+        "f":0.972319144
       },
       "PronType":{
-        "p":0.971869829,
-        "r":0.972406181,
-        "f":0.972137931
+        "p":0.9778516058,
+        "r":0.9746136865,
+        "f":0.9762299613
       },
       "Case":{
-        "p":0.9724220624,
-        "r":0.9614700652,
-        "f":0.9669150522
+        "p":0.9743895176,
+        "r":0.9697688204,
+        "f":0.972073678
       },
       "Degree":{
-        "p":0.9230072464,
-        "r":0.8477537438,
-        "f":0.8837814397
+        "p":0.914507772,
+        "r":0.881031614,
+        "f":0.8974576271
       },
       "Number":{
-        "p":0.9844515802,
-        "r":0.9762024468,
-        "f":0.98030966
+        "p":0.9877475663,
+        "r":0.986257751,
+        "f":0.9870020964
       },
       "Mood":{
-        "p":0.9429198683,
-        "r":0.9523281596,
-        "f":0.9476006619
+        "p":0.9290393013,
+        "r":0.94345898,
+        "f":0.9361936194
       },
       "Person":{
-        "p":0.9542429285,
-        "r":0.9432565789,
-        "f":0.9487179487
+        "p":0.9529220779,
+        "r":0.9654605263,
+        "f":0.9591503268
       },
       "Tense":{
-        "p":0.9660087719,
-        "r":0.973480663,
-        "f":0.9697303247
+        "p":0.9628820961,
+        "r":0.9745856354,
+        "f":0.9686985173
       },
       "VerbForm":{
-        "p":0.9516393443,
-        "r":0.9310344828,
-        "f":0.9412241589
+        "p":0.9615713066,
+        "r":0.9029671211,
+        "f":0.9313482217
       },
       "Voice":{
-        "p":0.9615773509,
-        "r":0.972392638,
-        "f":0.9669547534
+        "p":0.9576612903,
+        "r":0.9713701431,
+        "f":0.9644670051
       },
       "Number[psor]":{
-        "p":0.9737609329,
-        "r":0.9515669516,
-        "f":0.9625360231
+        "p":0.9852724595,
+        "r":0.952991453,
+        "f":0.9688631427
       },
       "Person[psor]":{
-        "p":0.9752186589,
-        "r":0.9543509272,
-        "f":0.9646719539
+        "p":0.9867452135,
+        "r":0.9557774608,
+        "f":0.9710144928
       },
       "NumType":{
-        "p":0.9423558897,
-        "r":0.9170731707,
-        "f":0.9295426452
+        "p":0.9097387173,
+        "r":0.9341463415,
+        "f":0.9217809868
       },
       "Poss":{
-        "p":0.75,
+        "p":0.6,
         "r":1.0,
-        "f":0.8571428571
+        "f":0.75
       },
       "Reflex":{
         "p":1.0,
@@ -1359,9 +1359,9 @@
         "f":0.9333333333
       },
       "Aspect":{
-        "p":0.0,
-        "r":0.0,
-        "f":0.0
+        "p":1.0,
+        "r":0.25,
+        "f":0.4
       },
       "Number[psed]":{
         "p":0.0,
@@ -1369,114 +1369,114 @@
         "f":0.0
       }
     },
-    "lemma_acc":0.9722514592,
-    "dep_uas":0.8222334626,
-    "dep_las":0.75479121,
+    "lemma_acc":0.9747392594,
+    "dep_uas":0.8158633861,
+    "dep_las":0.7489046175,
     "dep_las_per_type":{
       "det":{
-        "p":0.8744149766,
-        "r":0.8925159236,
-        "f":0.8833727344
+        "p":0.8498452012,
+        "r":0.8742038217,
+        "f":0.8618524333
       },
       "amod:att":{
-        "p":0.8396150762,
+        "p":0.8512195122,
         "r":0.8560915781,
-        "f":0.8477732794
+        "f":0.8536485936
       },
       "nsubj":{
-        "p":0.7182890855,
-        "r":0.7609375,
-        "f":0.7389984825
+        "p":0.7018813314,
+        "r":0.7578125,
+        "f":0.7287753569
       },
       "advmod:mode":{
-        "p":0.6243523316,
-        "r":0.5906862745,
-        "f":0.6070528967
+        "p":0.5764705882,
+        "r":0.6004901961,
+        "f":0.5882352941
       },
       "nmod:att":{
-        "p":0.7721943049,
-        "r":0.7813559322,
-        "f":0.7767481045
+        "p":0.7673267327,
+        "r":0.7881355932,
+        "f":0.7775919732
       },
       "obl":{
-        "p":0.8051575931,
-        "r":0.7587758776,
-        "f":0.781278962
+        "p":0.7942583732,
+        "r":0.7470747075,
+        "f":0.7699443414
       },
       "obj":{
-        "p":0.8633257403,
-        "r":0.8516853933,
-        "f":0.8574660633
+        "p":0.8322295806,
+        "r":0.8471910112,
+        "f":0.8396436526
       },
       "root":{
-        "p":0.7968397291,
-        "r":0.7861915367,
-        "f":0.7914798206
+        "p":0.7991071429,
+        "r":0.7973273942,
+        "f":0.7982162765
       },
       "cc":{
-        "p":0.7083333333,
-        "r":0.68,
-        "f":0.693877551
+        "p":0.7133479212,
+        "r":0.6863157895,
+        "f":0.6995708155
       },
       "conj":{
-        "p":0.5010799136,
-        "r":0.4833333333,
-        "f":0.4920466596
+        "p":0.4870775348,
+        "r":0.5104166667,
+        "f":0.498474059
       },
       "advmod":{
-        "p":0.7884615385,
-        "r":0.8631578947,
-        "f":0.824120603
+        "p":0.8235294118,
+        "r":0.8842105263,
+        "f":0.8527918782
       },
       "flat:name":{
-        "p":0.850678733,
-        "r":0.8785046729,
-        "f":0.8643678161
+        "p":0.9103773585,
+        "r":0.9018691589,
+        "f":0.9061032864
       },
       "appos":{
-        "p":0.3428571429,
+        "p":0.45,
         "r":0.3829787234,
-        "f":0.3618090452
+        "f":0.4137931034
       },
       "advcl":{
-        "p":0.2909090909,
-        "r":0.3265306122,
-        "f":0.3076923077
+        "p":0.297029703,
+        "r":0.306122449,
+        "f":0.3015075377
       },
       "advmod:tlocy":{
-        "p":0.7136929461,
-        "r":0.747826087,
-        "f":0.7303609342
+        "p":0.7222222222,
+        "r":0.6782608696,
+        "f":0.6995515695
       },
       "ccomp:obj":{
-        "p":0.34375,
-        "r":0.3333333333,
-        "f":0.3384615385
+        "p":0.3111111111,
+        "r":0.4242424242,
+        "f":0.358974359
       },
       "mark":{
-        "p":0.8481012658,
-        "r":0.8481012658,
-        "f":0.8481012658
+        "p":0.8246753247,
+        "r":0.8037974684,
+        "f":0.8141025641
       },
       "compound:preverb":{
-        "p":0.8859649123,
+        "p":0.9439252336,
         "r":0.9266055046,
-        "f":0.9058295964
+        "f":0.9351851852
       },
       "advmod:locy":{
-        "p":0.8333333333,
-        "r":0.46875,
-        "f":0.6
+        "p":0.72,
+        "r":0.5625,
+        "f":0.6315789474
       },
       "cop":{
-        "p":0.7567567568,
-        "r":0.6829268293,
-        "f":0.7179487179
+        "p":0.8636363636,
+        "r":0.4634146341,
+        "f":0.6031746032
       },
       "nmod:obl":{
-        "p":0.1739130435,
-        "r":0.1,
-        "f":0.126984127
+        "p":0.3125,
+        "r":0.25,
+        "f":0.2777777778
       },
       "advmod:to":{
         "p":0.0,
@@ -1484,69 +1484,69 @@
         "f":0.0
       },
       "obj:lvc":{
-        "p":0.5,
-        "r":0.0833333333,
-        "f":0.1428571429
+        "p":0.0,
+        "r":0.0,
+        "f":0.0
       },
       "ccomp:obl":{
-        "p":0.6086956522,
-        "r":0.4375,
-        "f":0.5090909091
+        "p":0.6470588235,
+        "r":0.34375,
+        "f":0.4489795918
       },
       "iobj":{
-        "p":0.2941176471,
-        "r":0.3333333333,
-        "f":0.3125
-      },
-      "case":{
-        "p":0.942408377,
-        "r":0.9183673469,
-        "f":0.9302325581
+        "p":0.1818181818,
+        "r":0.2666666667,
+        "f":0.2162162162
       },
       "csubj":{
-        "p":0.6666666667,
-        "r":0.3783783784,
-        "f":0.4827586207
+        "p":0.6428571429,
+        "r":0.2432432432,
+        "f":0.3529411765
+      },
+      "case":{
+        "p":0.9059405941,
+        "r":0.9336734694,
+        "f":0.9195979899
       },
       "parataxis":{
-        "p":0.0454545455,
-        "r":0.0136986301,
-        "f":0.0210526316
+        "p":0.1666666667,
+        "r":0.0410958904,
+        "f":0.0659340659
       },
       "xcomp":{
-        "p":0.9,
-        "r":0.8513513514,
-        "f":0.875
+        "p":0.8378378378,
+        "r":0.8378378378,
+        "f":0.8378378378
       },
       "nummod":{
-        "p":0.5943396226,
-        "r":0.6774193548,
-        "f":0.6331658291
-      },
-      "acl":{
-        "p":0.4057971014,
-        "r":0.3888888889,
-        "f":0.3971631206
+        "p":0.6071428571,
+        "r":0.5483870968,
+        "f":0.5762711864
       },
       "dep":{
         "p":0.0,
         "r":0.0,
         "f":0.0
       },
+      "acl":{
+        "p":0.3783783784,
+        "r":0.3888888889,
+        "f":0.3835616438
+      },
       "advmod:tto":{
-        "p":0.4545454545,
-        "r":0.5,
-        "f":0.4761904762
+        "p":0.2,
+        "r":0.1,
+        "f":0.1333333333
       },
       "nmod":{
-        "p":0.6,
-        "r":0.2727272727,
-        "f":0.375
+        "p":0.2,
+        "r":0.0909090909,
+        "f":0.125
       },
       "aux":{
-        "p":0.8571428571,
-        "r":0.5,
-        "f":0.6315789474
+        "p":0.875,
+        "r":0.5833333333,
+        "f":0.7
       },
       "advmod:tfrom":{
         "p":0.0,
@@ -1559,9 +1559,9 @@
         "f":0.0
       },
       "compound":{
-        "p":0.95,
-        "r":0.95,
-        "f":0.95
+        "p":0.9285714286,
+        "r":0.975,
+        "f":0.9512195122
       },
       "obl:lvc":{
         "p":0.0,
@@ -1579,9 +1579,9 @@
         "f":0.0
       },
       "list":{
-        "p":0.2222222222,
-        "r":0.3333333333,
-        "f":0.2666666667
+        "p":1.0,
+        "r":0.1666666667,
+        "f":0.2857142857
       },
       "ccomp":{
         "p":0.0,
@@ -1599,32 +1599,32 @@
         "f":0.0
       }
     },
-    "ents_p":0.8662957645,
-    "ents_r":0.848628692,
-    "ents_f":0.8573712256,
+    "ents_p":0.861328125,
+    "ents_r":0.8528481013,
+    "ents_f":0.8570671378,
     "ents_per_type":{
       "ORG":{
-        "p":0.8850889193,
-        "r":0.8998609179,
-        "f":0.8924137931
+        "p":0.8911439114,
+        "r":0.8956884562,
+        "f":0.8934104046
       },
       "PER":{
-        "p":0.8915009042,
-        "r":0.8835125448,
-        "f":0.8874887489
+        "p":0.8787346221,
+        "r":0.8960573477,
+        "f":0.8873114463
       },
       "LOC":{
-        "p":0.9098922625,
-        "r":0.8064236111,
-        "f":0.8550391164
+        "p":0.8728888889,
+        "r":0.8524305556,
+        "f":0.8625384278
       },
       "MISC":{
-        "p":0.6838340486,
-        "r":0.6780141844,
-        "f":0.6809116809
+        "p":0.6914556962,
+        "r":0.619858156,
+        "f":0.6537023186
       }
     },
-    "speed":757.2485282534
+    "speed":877.9572815434
   },
   "sources":[
     {

morphologizer/model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a57eb9dae96e9bf8e54a9752ba55e6e3912d979a7dcedd0688262c08e9e29fc4
+oid sha256:d5f93099945740c800708e5c5ed5f7b9acefa3122363f4a4e9d09f89ea3e7bc9
 size 1379030

ner/model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:349a93f95cb97ee8646a8f145ae82ee106815ea50e02b19ce909155e05f3ac81
+oid sha256:94bb30ee14bb5ccf6ba9239a594775221b296dc02f803b7828ff46721ecfa749
 size 56989063

parser/model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:93ef0b7782b1abcd56ae9bbcfe04056d69c40592f38b586e4a00930411b475b4
+oid sha256:76fa906db716c185d6ff1b495b845ff804bf1b924407117e2a856e0e6df00a51
 size 26010735

senter/model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:09c4e947afb6e7ce0585e255fdb58e3aa725acefdd375ae2886539ee44578908
+oid sha256:b53fc92e5cb9031751bbba9c32bbabe686e72181cdbaaeec17a3235b3923e315
 size 2845

tagger/model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:49dcfc46103a35c67051f7e925655f3cac46e67ae9d49b3965d8667c28f51911
+oid sha256:bfeeaf5292f94e27f75c696e8ded3a8dcd5f4b5747e0cf85eabfb6b88c5b8ce9
 size 20905

tok2vec/model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ac424f6dac3b36b4b913cc44e0f43004ec61fbe122ac27d5971915e155c71816
+oid sha256:78afb26fb2e04038f881d9ed816644ac1fda5dac73dbc3dfeac9ec37be3869e3
 size 56806299

vocab/strings.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fddb9576a688eb5303f9d3eec78385396083f8bd525fa342302245ccea6da82d
-size 6402729
+oid sha256:61f851567cea49829a0db0015d50da8cfab49a4c614fc210d339334ca3a99f34
+size 6404011