radiobee/detect.py

"""Detect language via polyglot and fastlid."""
# pylint: disable=

from typing import Any, Callable, List, Optional

from polyglot.text import Detector
import polyglot.detect.base
from polyglot.detect.base import UnknownLanguage
from fastlid import fastlid

from logzero import logger

polyglot.detect.base.logger.setLevel("ERROR")


def with_func_attrs(**attrs: Any) -> Callable:
    """Define func_attrs."""

    def with_attrs(fct: Callable) -> Callable:
        for key, val in attrs.items():
            setattr(fct, key, val)
        return fct

    return with_attrs


# @with_func_attrs(set_languages=None)
# def detect(text: str) -> str:
def detect(text: str, set_languages: Optional[List[str]] = None) -> str:
    """Detect language via polyglot and fastlid.

    check first with fastlid, if conf < 0.3, check with

    Alternative in detec_alt.py
    """
    # if not text.strip(): return "en"
    fastlid.set_languages = set_languages
    lang, conf = fastlid(text)
    detect.lang_conf = lang, conf
    if conf >= 0.3 or lang in ["zh"]:
        return lang

    try:
        langs = [(elm.code[:2], elm.confidence) for elm in Detector(text).languages]
        detect.lang_conf = langs
        # lang, conf = _[0]
    except UnknownLanguage:
        if set_languages is None:
            def_lang = "en"
        else:
            # def_lang = set_languages[-1]
            def_lang = set_languages[0]
        logger.warning(" UnknownLanguage exception: probably snippet too short, setting to %s", def_lang)
        langs = [(def_lang, 0)]
    except Exception as exc:
        logger.error(exc)
        langs = [("en", 0)]

    del conf

    # return first enrty's lang
    if set_languages is None:
        def_lang = langs[0][0]
    else:
        def_lang = "en"

        # pick the first in Detector(text).languages

        # just to silence pyright
        # set_languages_: List[str] = [""] if set_languages is None else set_languages

        for elm in langs:
            if elm[0] in set_languages:  # type: ignore
                def_lang = elm[0]
                break

    # set_languages is set
    if not isinstance(set_languages, (list, tuple)):
        logger.warning("set_languages (%s) ought to be a list/tuple")

    return def_lang