import json import os import re from copy import deepcopy from distutils.util import strtobool from pathlib import Path import yaml def build_from_file(file_path: str): path = Path(file_path) workers_path = path / "workers" if not workers_path.exists(): return [] # init config dict worker_configs = {} other_configs = {} file_names = {} for file in path.rglob("*"): if file.suffix not in [".json", ".yaml", ".yml"]: continue key = file.name.split(".", 1)[0] if key in file_names: raise Exception( f"Duplicate file name [{key}] found:\n" f"File 1: {file_names[key]}\n" f"File 2: {file}" ) file_names[key] = file try: with open(file, "r") as f: if file.suffix == ".json": content = json.load(f) else: # .yaml or .yml content = yaml.load(f, Loader=yaml.FullLoader) except Exception as e: raise Exception(f"Error loading file {file}: {str(e)}") if workers_path in file.parents: worker_configs[key] = content else: other_configs[key] = content for conf in worker_configs.values(): prep_config(conf, other_configs, []) worker_configs_list = [] for worker_config in worker_configs.values(): if isinstance(worker_config, list): worker_configs_list.extend(worker_config) else: worker_configs_list.append(worker_config) return worker_configs_list def prep_config(sub_config: dict | list, config: dict, forbid_keys: list): if isinstance(sub_config, dict): for key, conf in sub_config.items(): if isinstance(conf, str): if match := re.search(r"\$\{sub\|([^}]+)\}", conf): module_key = match.group(1).strip() if module_key not in config: raise Exception( "Incomplete configuration, lack module [{}]".format( module_key ) ) elif module_key in forbid_keys: raise Exception( "Can't reference submodule recursively. [{}]".format( module_key ) ) sub_config[key] = deepcopy(config[module_key]) prep_config(sub_config[key], config, forbid_keys + [module_key]) elif match := re.search(r"\$\{env\|([^,}]+)(?:,([^}]+))?\}", conf): env_key = match.group(1).strip() default_value = match.group(2) env_value = os.getenv(env_key) if env_value: sub_config[key] = env_value elif not env_value and default_value: sub_config[key] = default_value.strip() if sub_config[key] == "null" or sub_config[key] == "~": sub_config[key] = None else: raise ValueError( f"Environmental variable {env_key} need to be set." ) elif isinstance(conf, dict): prep_config(sub_config[key], config, forbid_keys) elif isinstance(conf, list): for i, item in enumerate(conf): if isinstance(item, dict): prep_config(sub_config[key][i], config, forbid_keys) elif isinstance(sub_config, list): for item in sub_config: prep_config(item, config, forbid_keys)