def md_update_links(this_lang_code, content):
    """ Update footer links in markdown to be language relative """
    result = content
    try:
        common_lang_codes = ["en/", "es/", "de/", "fr/", "es/", "ja/", "resources/"]
        exclude_common_langs = "|".join(list(map(lambda code: f"{code}",common_lang_codes)))
        relative_regex = re.compile("^(\\[[0-9]+]\:\\s*)(\/(?!" + exclude_common_langs + ").*)$", re.MULTILINE | re.IGNORECASE)
        substitute = "\g<1>/" + this_lang_code.lower() + "\g<2>"
        result = relative_regex.sub(substitute, content)
    except Exception as e:
        result = content
        logger.exception("fail to update md links")
    finally:
        return result

def
md_update_links
(
this_lang_code
,
content
)
:
result
=
content
try
:
common_lang_codes
=
[
"en/"
,
"es/"
,
"de/"
,
"fr/"
,
"es/"
,
"ja/"
,
"resources/"
]
exclude_common_langs
=
"|"
.
join
(
list
(
map
(
lambda
code
:
f"{code}"
,
common_lang_codes
)
)
)
relative_regex
=
re
.
compile
(
"^(\\[[0-9]+]\:\\s*)(\/(?!"
+
exclude_common_langs
+
").*)$"
,
re
.
MULTILINE
|
re
.
IGNORECASE
)
substitute
=
"\g<1>/"
+
this_lang_code
.
lower
(
)
+
"\g<2>"
result
=
relative_regex
.
sub
(
substitute
,
content
)
except
Exception
as
e
:
result
=
content
logger
.
exception
(
"fail to update md links"
)
finally
:
return
result


Look at all files within a given folder tree ( starting from path ) except For the exclusions folders : param path : Root folder into which to look for private URL : param exclusions : Array of folder names to exclude for looking for private files . : return private_urls : A list of HTML doc private file paths .

def find_private_url(path, exclusions):
    """
    Look at all files within a given folder tree (starting from path) except
    For the `exclusions` folders
    :param path: Root folder into which to look for private URL
    :param exclusions: Array of folder names to exclude for looking for private files.
    :return private_urls: A list of HTML doc private file paths.
    """
    private_urls=[]

    if not os.path.exists(path):
        raise ValueError('Content folder path incorrect')
    else:
        for (dirpath, dirnames, filenames) in os.walk(path):
            dirnames[:] = [d for d in dirnames if d not in exclusions]
            filenames[:] = [f for f in filenames if f.endswith('.html')]
            for filename in filenames:
              with open(os.path.join(dirpath, filename), 'rt', encoding='utf-8') as current_file:
                html = BeautifulSoup(current_file, "html.parser")
                if html.find_all('meta', {'name': 'robots', 'content': 'noindex, nofollow'}):
                  print('\x1b[32mINFO\x1b[0m: Skipping private page: %s' % dirpath)
                  private_urls.append(dirpath)

    return private_urls


Transforms URL returned by removing the public / ( name of the local folder with all hugo html files ) into real documentation links for Algolia : param private_urls : Array of file links in public / to transform into doc links . : return new_private_urls : A list of documentation URL links that correspond to private doc files .

def transform_url(private_urls):
    """
    Transforms URL returned by removing the public/ (name of the local folder with all hugo html files)
    into "real" documentation links for Algolia
    :param private_urls: Array of file links in public/ to transform into doc links.
    :return new_private_urls: A list of documentation URL links that correspond to private doc files.
    """
    new_private_urls = []
    for url in private_urls:

        ## We check if the url is not a localised FAQ url, or an API page if so we don't include it in list of stopped url
        ## Since Localised FAQs are not indexed anyway and the API page is indexed as a whole
        if not (re.match(r"public/fr/.*/faq/.*", url) or re.match(r"public/ja/.*/faq/.*", url) or re.match(r"public/ja/api/.*", url) or re.match(r"public/fr/api/.*", url)):

          ## We add /$ to all links in order to make them all "final", in fact
          ## Algolia stop_url parameter uses regex and not "perfect matching" link logic
          new_private_urls.append(url.replace('public/','docs.datadoghq.com/') + '/$')

    return new_private_urls


Updates the Algolia Docsearch configuration file with the list of private links to exclude from Algolia indexes . : param docs_index_config : Original configuration file for the Algolia Doc search : param private_urls : A list of documentation URL links that correspond to private doc files .

def update_algolia_private_url(docs_index_config,private_urls):
    """
    Updates the Algolia Docsearch configuration file with the list of private links to exclude from
    Algolia indexes.
    :param docs_index_config: Original configuration file for the Algolia Doc search
    :param private_urls: A list of documentation URL links that correspond to private doc files.
    """
    with open(docs_index_config, 'rt', encoding='utf-8') as json_file:
        print("\x1b[32mINFO\x1b[0m: Configuration file {} correctly loaded.".format(docs_index_config))
        config = json.load(json_file)

        print("\x1b[32mINFO\x1b[0m: Adding list of private URLs.")

        ## adding list or private urls while removing duplicates + sorting the global list
        config["stop_urls"] = sorted(list(dict.fromkeys(config["stop_urls"] + private_urls)))

    print("\x1b[32mINFO\x1b[0m: Addition complete, updating Algolia main configuration file with the new one.")

    with open(docs_index_config, 'w+', encoding='utf-8') as json_file:
        json.dump(config, json_file)


only downloads fresh file if we don t have one or we do and the sha has changed

def cache_by_sha(func):
    """ only downloads fresh file, if we don't have one or we do and the sha has changed """

    @wraps(func)
    def cached_func(*args, **kwargs):
        cache = {}
        list_item = args[1]
        dest_dir = kwargs.get("dest_dir")
        path_to_file = list_item.get("path", "")
        file_out = "{}{}".format(dest_dir, path_to_file)
        p_file_out = "{}{}.pickle".format(
            dest_dir, path_to_file
        )
        makedirs(dirname(file_out), exist_ok=True)
        if exists(p_file_out) and exists(file_out):
            with open(p_file_out, "rb") as pf:
                cache = pickle.load(pf)
        cache_sha = cache.get("sha", False)
        input_sha = list_item.get("sha", False)
        if (cache_sha and input_sha and cache_sha == input_sha):
            # do nothing as we have the up to date file already
            return None
        else:
            with open(p_file_out, mode="wb+") as pf:
                pickle.dump(
                    list_item, pf, pickle.HIGHEST_PROTOCOL
                )
            return func(*args, **kwargs)

    return cached_func


Takes a string from the agent config template and formats it for output in agent_config shortcode . - If string contains exactly one # and no alphabetic characters remove the # and any new line characters . - If the string contains exactly one # and has alphabetic characters remove the # ( this represents a config key / value )

def format_agent_config_string(string):
    """
    Takes a string from the agent config template and formats it for output in
    agent_config shortcode.
      - If string contains exactly one '#' and no alphabetic characters,
        remove the '#' and any new line characters.
      - If the string contains exactly one '#' and has alphabetic characters,
        remove the '#' (this represents a config key/value)
    """
    # Match any lines containing strictly one '#' character that can have any
    # number of leading or trailing whitespaces, and no words.
    # Matching lines in the config template file are for spacing only.
    regex_string = r"[^a-zA-Z0-9#]*#[^a-zA-Z0-9#]*$"

    if re.match(regex_string, string):
        return string.replace('#', '').replace('\n', '').strip()
    elif '#' in string and '##' not in string:
        return string.replace('# ', '') + '\n'
    else:
        return string + '\n'


Returns a dictionary where the keys represent each config type ( i . e . Basic Configuration Log Collection Configuration etc ) and the values are a string containing the config options comments .

def create_agent_config_dict(dd_agent_config_string):
    """
    Returns a dictionary where the keys represent each config type
    (i.e. "Basic Configuration", "Log Collection Configuration", etc),
    and the values are a string containing the config options comments.
    """
    config_type_header_delimiter = '#######'
    agent_config_array = dd_agent_config_string.splitlines()
    agent_config_dict = {}
    current_config_type = ''

    for index, line in enumerate(agent_config_array):
        if config_type_header_delimiter in line:
            config_type = agent_config_array[index + 1].replace('#', '') \
                .strip().lower()

            if config_type:
                agent_config_dict.setdefault(config_type, '')
                current_config_type = config_type
        else:
            # Skip any Go template strings or comment boxes used to delineate
            # config types.
            if '{{' not in line and not re.match(r"^##.+##$", line):
                formatted_string = format_agent_config_string(line)
                agent_config_dict[current_config_type] += formatted_string

    return agent_config_dict


Returns a single string containing all available config types for Docs team to reference .

def document_config_types(agent_config_dict):
    """
    Returns a single string containing all available config types,
    for Docs team to reference.
    """
    return 'Available config types: \n\n' + '\n'.join(agent_config_dict.keys())

def
document_config_types
(
agent_config_dict
)
:
return
'Available config types: \n\n'
+
'\n'
.
join
(
agent_config_dict
.
keys
(
)
)


Takes the Datadog Agent Config template as a string separates it by type formats the strings and outputs the results as json to be consumed by the agent_config . html shortcode for display .

def process_agent_config(dd_agent_config_string):
    """
    Takes the Datadog Agent Config template as a string, separates it by type,
    formats the strings, and outputs the results as json to be consumed by the
    agent_config.html shortcode for display.
    """
    try:
        agent_config_dict = create_agent_config_dict(dd_agent_config_string)
        formatted_agent_config_json = json.dumps(agent_config_dict)
        config_types_string = document_config_types(agent_config_dict)

        with open('data/agent_config.json', 'w+') as agent_json_config_outfile:
            agent_json_config_outfile.write(formatted_agent_config_json)

        # Documenting what config types are available for Docs team to use in
        # agent config shortcode.
        with open('agent_config_types_list.txt', 'w+') as config_types_outfile:
            config_types_outfile.write(config_types_string)
    except Exception as err:
        print('An error occurred building agent config data:')
        print(err)

        if getenv("LOCAL") != 'True':
            sys.exit(1)


Takes github info and file globs and downloads files from github using multiple processes : param github_token : A valide Github token to download content with the Github Class : param org : github organization or person : param repo : github repo name : param branch : the branch name : param globs : list of strings in glob format of what to extract : param extract_dir : Directory in which to put all downloaded content . : param commit_sha : sha if we want to provide one : return :

def download_from_repo(github_token, org, repo, branch, globs, extract_dir, commit_sha=None):
    """
    Takes github info and file globs and downloads files from github using multiple processes
    :param github_token: A valide Github token to download content with the Github Class
    :param org: github organization or person
    :param repo: github repo name
    :param branch: the branch name
    :param globs: list of strings in glob format of what to extract
    :param extract_dir: Directory in which to put all downloaded content.
    :param commit_sha: sha if we want to provide one
    :return:
    """
    pool_size = cpu_count()

    with GitHub(github_token) as gh:
        listing = gh.list(org, repo, branch, commit_sha, globs)
        dest = "{0}{1}{2}".format(
            extract_dir, repo, sep
        )
        with Pool(processes=pool_size) as pool:
            with requests.Session() as s:
                r = [
                    x
                    for x in pool.imap_unordered(
                        partial(
                            gh.raw,
                            request_session=s,
                            org=org,
                            repo=repo,
                            branch=branch if not commit_sha else commit_sha,
                            dest_dir=dest,
                        ),
                        listing,
                    )
                ]

def download_from_repo ( github_token , org , repo , branch , globs , extract_dir , commit_sha = None ) : pool_size = cpu_count ( ) with GitHub ( github_token ) as gh : listing = gh . list ( org , repo , branch , commit_sha , globs ) dest = "{0}{1}{2}" . format ( extract_dir , repo , sep ) with Pool ( processes = pool_size ) as pool : with requests . Session ( ) as s : r = [ x for x in pool . imap_unordered ( partial ( gh . raw , request_session = s , org = org , repo = repo , branch = branch if not commit_sha else commit_sha , dest_dir = dest , ) , listing , ) ]


Depending if the repo is local or we downloaded it we need to update the globs to match the final version of the repo to use : param new_path : new_path to update the globs with : param globs : list of globs to update

def update_globs(new_path, globs):
    """
    Depending if the repo is local or we downloaded it we need to update the globs to match
    the final version of the repo to use
    :param new_path: new_path to update the globs with
    :param globs: list of globs to update
    """
    new_globs = []
    for item in globs:
        new_globs.append("{}{}".format(new_path, item))
    return new_globs

def update_globs ( new_path , globs ) : new_globs = [ ] for item in globs : new_globs . append ( "{}{}" . format ( new_path , item ) ) return new_globs


This goes through the list_of_contents and check for each repo specified If a local version exists otherwise we download it from the upstream repo on Github Local version of the repo should be in the same folder as the documentation / folder . : param github_token : A valide Github token to download content with the Github Class : param extract_dir : Directory into which to put all content downloaded . : param list_of_content : List of content to check if available locally or if it needs to be downloaded from Github

def local_or_upstream(github_token, extract_dir, list_of_contents):
    """
    This goes through the list_of_contents and check for each repo specified
    If a local version exists otherwise we download it from the upstream repo on Github
    Local version of the repo should be in the same folder as the documentation/ folder.
    :param github_token: A valide Github token to download content with the Github Class
    :param extract_dir: Directory into which to put all content downloaded.
    :param list_of_content: List of content to check if available locally or if it needs to be downloaded from Github
    """
    for content in list_of_contents:
        repo_name = "../" + content["repo_name"] + sep
        if isdir(repo_name):
            print("\x1b[32mINFO\x1b[0m: Local version of {} found".format(
                content["repo_name"]))
            content["globs"] = update_globs(
                repo_name,
                content["globs"],
            )
        elif github_token != "false":
            print(
                "\x1b[32mINFO\x1b[0m: No local version of {} found, downloading content from upstream version".format(
                    content["repo_name"]
                )
            )
            download_from_repo(github_token,
                               content["org_name"],
                               content["repo_name"],
                               content["branch"],
                               content["globs"],
                               extract_dir,
                               content.get("sha", None)
                               )
            content[
                "globs"
            ] = update_globs(
                "{0}{1}{2}".format(
                    extract_dir,
                    content["repo_name"],
                    sep,
                ),
                content["globs"],
            )
        elif getenv("LOCAL") == 'True':
            print(
                "\x1b[33mWARNING\x1b[0m: No local version of {} found, no GITHUB_TOKEN available. Documentation is now in degraded mode".format(content["repo_name"]))
            content["action"] = "Not Available"
        else:
            print(
                "\x1b[31mERROR\x1b[0m: No local version of {} found, no GITHUB_TOKEN available.".format(
                    content["repo_name"]
                )
            )
            raise ValueError
    return list_of_contents


This pulls the content from the configuration file at configuration location then parses it to populate the list_of_content variable that contains all contents that needs to be pulled and processed . : param configuration : Documentation build configuration file path .

def extract_config(configuration):
    """
    This pulls the content from the configuration file at `configuration` location
    then parses it to populate the list_of_content variable that contains all contents
    that needs to be pulled and processed.
    :param configuration: Documentation build configuration file path.
    """
    list_of_contents = []

    for org in configuration:
        for repo in org["repos"]:
            for content in repo["contents"]:
                content_temp = {}
                content_temp["org_name"] = org[
                    "org_name"
                ]
                content_temp["repo_name"] = repo[
                    "repo_name"
                ]
                content_temp["branch"] = content[
                    "branch"
                ]
                content_temp["sha"] = content.get("sha", None)
                content_temp["action"] = content[
                    "action"
                ]
                content_temp["globs"] = content["globs"]

                if content["action"] in ("pull-and-push-folder", "pull-and-push-file", "security-rules", "compliance-rules"):
                    content_temp["options"] = content["options"]

                list_of_contents.append(
                    content_temp
                )

    return list_of_contents

def extract_config ( configuration ) : list_of_contents = [ ] for org in configuration : for repo in org [ "repos" ] : for content in repo [ "contents" ] : content_temp = { } content_temp [ "org_name" ] = org [ "org_name" ] content_temp [ "repo_name" ] = repo [ "repo_name" ] content_temp [ "branch" ] = content [ "branch" ] content_temp [ "sha" ] = content . get ( "sha" , None ) content_temp [ "action" ] = content [ "action" ] content_temp [ "globs" ] = content [ "globs" ] if content [ "action" ] in ( "pull-and-push-folder" , "pull-and-push-file" , "security-rules" , "compliance-rules" ) : content_temp [ "options" ] = content [ "options" ] list_of_contents . append ( content_temp ) return list_of_contents


Prepares the content for the documentation build . It checks for all content whether or not it s available locally or if it should be downloaded . : param configuration : Documentation build configuration file path . : param github_token : A valide Github token to download content with the Github Class : param extract_dir : Directory into which to put all content downloaded .

def prepare_content(configuration, github_token, extract_dir):
    """
    Prepares the content for the documentation build. It checks for all content whether or
    not it's available locally or if it should be downloaded.
    :param configuration: Documentation build configuration file path.
    :param github_token: A valide Github token to download content with the Github Class
    :param extract_dir: Directory into which to put all content downloaded.
    """
    try:
        list_of_contents = local_or_upstream(
            github_token, extract_dir, extract_config(configuration))
    except:
        if getenv("LOCAL") == 'True':
            print(
                "\x1b[33mWARNING\x1b[0m: Downloading files failed, documentation is now in degraded mode.")
        else:
            print(
                "\x1b[31mERROR\x1b[0m: Downloading files failed, stopping build.")
            sys.exit(1)
    return list_of_contents


Take the content from a folder following github logic and transform it to be displayed in the doc in dest_dir folder See https : // github . com / DataDog / documentation / wiki / Documentation - Build#pull - and - push - folder to learn more : param content : content to process : param content_dir : The directory where content should be put

def pull_and_push_folder(content, content_dir):
    """
    Take the content from a folder following github logic
    and transform it to be displayed in the doc in dest_dir folder
    See https://github.com/DataDog/documentation/wiki/Documentation-Build#pull-and-push-folder to learn more
    :param content: content to process
    :param content_dir: The directory where content should be put
    """
    for file_name in chain.from_iterable(glob.iglob(pattern, recursive=True) for pattern in content["globs"]):
        with open(file_name, mode="r+", encoding="utf-8", errors="ignore") as f:
            file_content = f.read()
            boundary = re.compile(r'^-{3,}$', re.MULTILINE)
            split = boundary.split(file_content, 2)
            new_yml = {}
            txt = file_content
            if len(split) == 3:
                _, fm, txt = split
                new_yml = yaml.safe_load(fm)
            elif len(split) == 1:
                txt = split[0]
            # if front matter update existing
            if "front_matters" in content["options"]:
                new_yml.update(content["options"]["front_matters"])
                # if the dependency ends with a `/` e.g folder then lets try replace with the actual filename
                new_deps = []
                for dep in new_yml.get("dependencies", []):
                    if dep.endswith('/'):
                        new_deps.append('{}{}'.format(
                            dep,
                            basename(file_name)
                        ))
                new_yml['dependencies'] = new_deps
            front_matter = yaml.dump(new_yml, default_flow_style=False).strip()
            # Replacing links that point to the Github folder by link that point to the doc.
            new_link = (
                content["options"]["dest_dir"] + "\\2"
            )
            regex_github_link = re.compile(
                r"(https:\/\/github\.com\/{}\/{}\/blob\/{}\/{})(\S+)\.md".format(
                    content["org_name"],
                    content["repo_name"],
                    content["branch"],
                    content["options"][
                        "path_to_remove"
                    ],
                )
            )
            txt = re.sub(
                regex_github_link,
                new_link,
                txt,
                count=0,
            )
            file_content = TEMPLATE.format(front_matter=front_matter, content=txt.strip())
            # Replacing the master README.md by _index.md to follow Hugo logic
            if file_name.endswith("README.md"):
                file_name = "_index.md"
        # Writing the new content to the documentation file
        dirp = "{}{}".format(
            content_dir,
            content["options"]["dest_dir"][1:],
        )
        makedirs(dirp, exist_ok=True)
        with open(
            "{}{}".format(dirp, basename(file_name)),
            mode="w+",
            encoding="utf-8",
        ) as f:
            f.write(file_content)


Takes the content from a file from a github repo and pushed it to the doc See https : // github . com / DataDog / documentation / wiki / Documentation - Build#pull - and - push - files to learn more : param content : object with a file_name a file_path and options to apply : param content_dir : The directory where content should be put

def pull_and_push_file(content, content_dir):
    """
    Takes the content from a file from a github repo and
    pushed it to the doc
    See https://github.com/DataDog/documentation/wiki/Documentation-Build#pull-and-push-files to learn more
    :param content: object with a file_name, a file_path, and options to apply
    :param content_dir: The directory where content should be put
    """
    with open("".join(content["globs"]), mode="r+") as f:
        file_content = f.read()
        # If options include front params, then the H1 title of the source file is striped
        # and the options front params are inlined
        if "front_matters" in content["options"]:
            front_matter = yaml.dump(content["options"]["front_matters"], default_flow_style=False).strip()
            # remove h1 if exists
            file_content = re.sub(re.compile(r"^#{1}(?!#)(.*)", re.MULTILINE), "", file_content, count=1)
            file_content = TEMPLATE.format(front_matter=front_matter, content=file_content.strip())
        elif "datadog-agent" in content["repo_name"] and "config_template.yaml" in "".join(content["globs"]):
            process_agent_config(file_content)

    output_content = content["options"].get("output_content", True)

    if output_content:
        destination_path = content["options"]["dest_path"].lstrip('/')

        with open(
            "{}{}{}".format(
                content_dir,
                destination_path,
                basename(content["options"]["file_name"]),
            ),
            mode="w+",
            encoding="utf-8",
        ) as f:
            f.write(file_content)


def security_rules(content, content_dir):
    """
    Takes the content from a file from a github repo and
    pushed it to the doc
    See https://github.com/DataDog/documentation/wiki/Documentation-Build#pull-and-push-files to learn more
    :param content: object with a file_name, a file_path, and options to apply
    :param content_dir: The directory where content should be put
    """
    logger.info("Starting security rules action...")
    global_aliases = []
    for file_name in chain.from_iterable(glob.glob(pattern, recursive=True) for pattern in content["globs"]):

        data = None
        if file_name.endswith(".json"):
            with open(file_name, mode="r+") as f:
                try:
                    data = json.loads(f.read())
                except:
                    logger.warn(f"Error parsing {file_name}")
        elif file_name.endswith(".yaml"):
            with open(file_name, mode="r+") as f:
                try:
                    file_text_content = f.read()
                    if 'jinja2' in file_text_content:
                        data = load_templated_file(f)
                    else:
                        data = yaml.safe_load(file_text_content)
                except:
                    logger.warn(f"Error parsing {file_name}")

        p = Path(f.name)
        message_file_name = p.with_suffix('.md')

        if data and message_file_name.exists():
            # delete file or skip if staged
            # any() will return True when at least one of the elements is Truthy
            if len(data.get('restrictedToOrgs', [])) > 0 or data.get('isStaged', False) or data.get('isDeleted', False) or not data.get('isEnabled', True):
                if p.exists():
                    logger.info(f"removing file {p.name}")
                    global_aliases.append(f"/security_monitoring/default_rules/{p.stem}")
                    global_aliases.append(f"/security_platform/default_rules/{p.stem}")
                    p.unlink()
                else:
                    logger.info(f"skipping file {p.name}")
            else:
                # The message of a detection rule is located in a Markdown file next to the rule definition
                with open(str(message_file_name), mode="r+") as message_file:
                    message = message_file.read()

                    # strip out [text] e.g "[CIS Docker] Ensure that.." becomes "Ensure that..."
                    parsed_title = re.sub(r"\[.+\]\s?(.*)", "\\1", data.get('name', ''), 0, re.MULTILINE)
                    page_data = {
                        "title": parsed_title,
                        "kind": "documentation",
                        "type": "security_rules",
                        "disable_edit": True,
                        "aliases": [
                            f"{data.get('defaultRuleId', '').strip()}",
                            f"/security_monitoring/default_rules/{data.get('defaultRuleId', '').strip()}",
                            f"/security_monitoring/default_rules/{p.stem}"
                        ],
                        "rule_category": [],
                        "integration_id": ""
                    }

                    # we need to get the path relative to the repo root for comparisons
                    extract_dir, relative_path = str(p.parent).split(f"/{content['repo_name']}/")
                    # lets build up this categorization for filtering purposes

                    # previous categorization
                    if relative_path.startswith('configuration'):
                        page_data['rule_category'].append('Posture Management (Cloud)')
                    elif relative_path.startswith('runtime'):
                        if 'compliance' in relative_path:
                            page_data['rule_category'].append('Posture Management (Infra)')
                        else:
                            page_data['rule_category'].append('Workload Security')

                    # new categorization
                    if 'security-monitoring' in relative_path:
                        page_data['rule_category'].append('Cloud SIEM')

                    if 'posture-management' in relative_path:
                        if 'cloud-configuration' in relative_path:
                            page_data['rule_category'].append('Posture Management (Cloud)')
                        if 'infrastructure-configuration' in relative_path:
                            page_data['rule_category'].append('Posture Management (Infra)')

                    if 'workload-security' in relative_path:
                        page_data['rule_category'].append('Workload Security')

                    if 'application-security' in relative_path:
                        page_data['rule_category'].append('Application Security')

                    tags = data.get('tags', [])
                    if tags:
                        for tag in tags:
                            if ':' in tag:
                                key, value = tag.split(':')
                                page_data[key] = value
                        if data.get('source', ''):
                            page_data["source"] = data.get('source', '')
                    else:
                        # try build up manually
                        source = data.get('source', None)
                        tech = data.get('framework', {}).get('name', '').replace('cis-', '')
                        page_data["source"] = source or tech
                        page_data["security"] = "compliance"
                        page_data["framework"] = data.get('framework', {}).get('name', '')
                        page_data["control"] = data.get('control', '')
                        page_data["scope"] = tech

                    # lowercase them
                    if page_data.get("source", None):
                        page_data["source"] = page_data["source"].lower()
                    if page_data.get("scope", None):
                        page_data["scope"] = page_data["scope"].lower()

                    # integration id
                    page_data["integration_id"] = page_data.get("scope", None) or page_data.get("source", "")
                    cloud = page_data.get("cloud", None)
                    if cloud and cloud == 'aws':
                        page_data["integration_id"] = "amazon-{}".format(page_data["integration_id"])

                    front_matter = yaml.dump(page_data, default_flow_style=False).strip()
                    output_content = TEMPLATE.format(front_matter=front_matter, content=message.strip())

                    dest_dir = Path(f"{content_dir}{content['options']['dest_path']}")
                    dest_dir.mkdir(exist_ok=True)
                    dest_file = dest_dir.joinpath(p.name).with_suffix('.md')
                    logger.info(dest_file)
                    with open(dest_file, mode='w', encoding='utf-8') as out_file:
                        out_file.write(output_content)

    # add global aliases from deleted files to _index.md
    if os.environ.get('CI_ENVIRONMENT_NAME', '') in ('live', 'preview'):
        index_path = Path(f"{content_dir}{content['options']['dest_path']}_index.md")
        update_global_aliases(index_path, global_aliases)


def metric_csv_to_yaml(key_name, csv_filename, yml_filename):
        """
        Given a file path to a single csv file convert it to a yaml file

        :param key_name: integration key name for root object
        :param csv_filename: path to input csv file
        :param yml_filename: path to output yml file
        """
        yaml_data = {key_name: []}
        with open(csv_filename) as csv_file:
            reader = csv.DictReader(csv_file, delimiter=",")
            yaml_data[key_name] = [
                dict(line) for line in reader
            ]
        if yaml_data[key_name]:
            # Transforming the metric description to html in order to interpret markdown in
            # integrations metrics table.
            # the char strip is to compensate for the lib adding <p> </p><br> tags
            for metric in yaml_data[key_name]:
                metric['description'] = str(
                    markdown2.markdown(metric['description']))[3:-5]
            with open(
                file=yml_filename,
                mode="w",
                encoding="utf-8",
            ) as f:
                f.write(
                    yaml.dump(
                        yaml_data, default_flow_style=False
                    )
                )


Goes through a section and remove all reference links it can found .

def inline_references(self, integration_readme, regex_skip_sections_start, regex_skip_sections_end):
        """
        Goes through a section and remove all reference links it can found.

        :param section: An array of lines representing a section.
        """

        skip = False
        all_references = []
        section_without_references = []
        section_with_all_links = []
        regex_bottom_reference_link = r"^\s*\[(\d*?)\]: (\S*)"

        # Collecting all references and removing them from section
        # looking at each line, if a line is a reference then we remove it and store the reference.

        #section = integration_file.readlines()
        for line in integration_readme.splitlines(True):
            if skip:
                section_without_references.append(line)
                if re.search(regex_skip_sections_end, line):
                    skip = False
            elif not skip:

                if re.search(regex_skip_sections_start, line):
                    section_without_references.append(line)
                    skip = True

                elif re.search(regex_bottom_reference_link, line):

                    reference = re.search(regex_bottom_reference_link, line)
                    all_references.append([reference.group(1),
                                           reference.group(2)])
                else:
                    section_without_references.append(line)

        # By the end of the for loop skip should always be false otherwise it means that a codeblock is not closed.
        if skip:
            raise ValueError

        for line in section_without_references:
            if skip:
                if re.search(regex_skip_sections_end, line):
                    skip = False
            elif not skip:
                if re.search(regex_skip_sections_start, line):
                    skip = True
                else:
                    for reference in all_references:
                        reference_index, reference_val = reference

                        curent_link = '][' + reference_index + ']'

                        line = line.replace(
                            curent_link, '](' + reference_val + ')')

            section_with_all_links.append(line)

        # By the end of the for loop skip should always be false otherwise it means that a codeblock is not closed.
        if skip:
            raise ValueError

        integration_content_with_link_inlined = ''.join(section_with_all_links)

        return integration_content_with_link_inlined


Goes through all files needed for integrations build and triggers the right function for the right type of file . #integrations to learn more . See https : // github . com / DataDog / documentation / wiki / Documentation - Build : param content : integrations content to process

def process_integrations(self, content, marketplace=False):
        """
        Goes through all files needed for integrations build
        and triggers the right function for the right type of file.
        #integrations to learn more.
        See https://github.com/DataDog/documentation/wiki/Documentation-Build
        :param content: integrations content to process
        """
        for file_name in chain.from_iterable(
            glob.iglob(pattern, recursive=True)
            for pattern in content["globs"]
        ):
            if file_name.endswith(".csv"):
                self.process_integration_metric(file_name)

            elif file_name.endswith("manifest.json"):
                self.process_integration_manifest(file_name)

            elif file_name.endswith("service_checks.json"):
                self.process_service_checks(file_name)

            elif file_name.endswith(".md"):
                self.process_integration_readme(file_name, marketplace)

            elif file_name.endswith((".png", ".svg", ".jpg", ".jpeg", ".gif")) and marketplace:
                self.process_images(file_name)

            elif file_name.endswith(".go"):
                self.process_npm_integrations(file_name)

def process_integrations ( self , content , marketplace = False ) : for file_name in chain . from_iterable ( glob . iglob ( pattern , recursive = True ) for pattern in content [ "globs" ] ) : if file_name . endswith ( ".csv" ) : self . process_integration_metric ( file_name ) elif file_name . endswith ( "manifest.json" ) : self . process_integration_manifest ( file_name ) elif file_name . endswith ( "service_checks.json" ) : self . process_service_checks ( file_name ) elif file_name . endswith ( ".md" ) : self . process_integration_readme ( file_name , marketplace ) elif file_name . endswith ( ( ".png" , ".svg" , ".jpg" , ".jpeg" , ".gif" ) ) and marketplace : self . process_images ( file_name ) elif file_name . endswith ( ".go" ) : self . process_npm_integrations ( file_name )


def merge_integrations(self):
        """ Merges integrations that come under one """
        for (
            name,
            action_obj,
        ) in self.integration_mutations.items():
            if name not in self.initial_integration_files:
                action = action_obj.get("action")
                target = action_obj.get("target")
                input_file = "{}{}.md".format(
                    self.content_integrations_dir, name
                )
                output_file = "{}{}.md".format(
                    self.content_integrations_dir, target
                )
                if action == "merge":
                    with open(
                        input_file, "r"
                    ) as content_file, open(
                        output_file, "a"
                    ) as target_file:
                        content = content_file.read()
                        content = re.sub(
                            self.regex_fm,
                            r"\2",
                            content,
                            count=0,
                        )
                        if action_obj.get(
                            "remove_header", False
                        ):
                            content = re.sub(
                                self.regex_h1,
                                "",
                                content,
                                count=0,
                            )
                        else:
                            content = re.sub(
                                self.regex_h1_replace,
                                r"##\2",
                                content,
                                count=0,
                            )
                        regex_skip_sections_end = r"(```|\{\{< \/code-block >\}\})"
                        regex_skip_sections_start = r"(```|\{\{< code-block)"

                        ## Inlining all link from the file to merge
                        ## to avoid link ref colision with the existing references.
                        content = self.inline_references(content,regex_skip_sections_start,regex_skip_sections_end)

                        target_file.write(content)

                    ## Formating all link as reference in the new merged integration file
                    try:
                        final_text = format_link_file(output_file,regex_skip_sections_start,regex_skip_sections_end)
                        with open(output_file, 'w') as final_file:
                            final_file.write(final_text)
                    except Exception as e:
                        print(e)

                    try:
                        remove(input_file)
                    except OSError:
                        print(
                            "\x1b[31mERROR\x1b[0m: The file {} was not found and could not be removed during merge action".format(
                                input_file
                            )
                        )
                elif action == "truncate":
                    if exists(output_file):
                        with open(
                            output_file, "r+"
                        ) as target_file:
                            content = target_file.read()
                            content = re.sub(
                                self.regex_fm,
                                r"---\n\1\n---\n",
                                content,
                                count=0,
                            )
                            target_file.truncate(0)
                            target_file.seek(0)
                            target_file.write(content)
                    else:
                        open(output_file, "w").close()
                elif action == "discard":
                    try:
                        remove(input_file)
                    except OSError:
                        print(
                            "\x1b[31mERROR\x1b[0m: The file {} was not found and could not be removed during discard action".format(
                                input_file
                            )
                        )
                elif action == "create":
                    with open(output_file, "w+") as f:
                        fm = yaml.dump(
                            action_obj.get("fm"),
                            default_flow_style=False,
                        ).rstrip()
                        data = "---\n{0}\n---\n".format(fm)
                        f.write(data)


Take a single metadata csv file and convert it to yaml : param file_name : path to a metadata csv file

def process_integration_metric(self, file_name):
        """
        Take a single metadata csv file and convert it to yaml
        :param file_name: path to a metadata csv file
        """
        if file_name.endswith("/metadata.csv"):
            key_name = basename(
                dirname(normpath(file_name))
            )
        else:
            key_name = basename(
                file_name.replace("_metadata.csv", "")
            )
        new_file_name = "{}{}.yaml".format(
            self.data_integrations_dir, key_name
        )
        self.metric_csv_to_yaml(key_name, file_name, new_file_name)

def process_integration_metric ( self , file_name ) : if file_name . endswith ( "/metadata.csv" ) : key_name = basename ( dirname ( normpath ( file_name ) ) ) else : key_name = basename ( file_name . replace ( "_metadata.csv" , "" ) ) new_file_name = "{}{}.yaml" . format ( self . data_integrations_dir , key_name ) self . metric_csv_to_yaml ( key_name , file_name , new_file_name )


Take a single manifest json file and upsert to integrations . json data set is_public to false to hide integrations we merge later : param file_name : path to a manifest json file

def process_integration_manifest(self, file_name):
        """
        Take a single manifest json file and upsert to integrations.json data
        set is_public to false to hide integrations we merge later
        :param file_name: path to a manifest json file
        """

        names = [
            d.get("name", "").lower()
            for d in self.datafile_json
            if "name" in d
        ]
        with open(file_name) as f:
            try:
                data = json.load(f)
                data = self.process_manifest(data, basename(dirname(file_name)))
                data_name = data.get("name", "").lower()
                if data_name in [
                    k
                    for k, v in self.integration_mutations.items()
                    if v.get("action") == "merge"
                ]:
                    data["is_public"] = False
                if data_name in names:
                    item = [
                        d
                        for d in self.datafile_json
                        if d.get("name", "").lower() == data_name
                    ]
                    if len(item) > 0:
                        item[0].update(data)
                else:
                    self.datafile_json.append(data)
            except JSONDecodeError:
                if getenv("LOCAL") == 'True':
                    print(
                        "\x1b[33mWARNING\x1b[0m: manifest could not be parsed {}".format(file_name))
                else:
                    print(
                        "\x1b[31mERROR\x1b[0m: manifest could not be parsed {}".format(file_name))
                    raise JSONDecodeError


Take a single service_checks . json file and copies it to the data folder as the integration name it came from e . g / data / service_checks / docker . json : param file_name : path to a service_checks json file

def process_service_checks(self, file_name):
        """
        Take a single service_checks.json file and copies it to the data folder
        as the integration name it came from e.g /data/service_checks/docker.json
        :param file_name: path to a service_checks json file
        """

        if file_name.endswith("/assets/service_checks.json"):
            file_list = file_name.split(sep)
            key_name = file_list[len(file_list)-3]
        else:
            key_name = basename(
                dirname(normpath(file_name))
            )

        new_file_name = "{}{}.json".format(
            self.data_service_checks_dir, key_name
        )

        shutil.copy(
            file_name,
            new_file_name,
        )

def process_service_checks ( self , file_name ) : if file_name . endswith ( "/assets/service_checks.json" ) : file_list = file_name . split ( sep ) key_name = file_list [ len ( file_list ) - 3 ] else : key_name = basename ( dirname ( normpath ( file_name ) ) ) new_file_name = "{}{}.json" . format ( self . data_service_checks_dir , key_name ) shutil . copy ( file_name , new_file_name , )


Save the defaults . go file from AWS as a json file / data / npm / aws . json

def process_npm_integrations(self, file_name):
        """
        Save the defaults.go file from AWS as a json file
        /data/npm/aws.json
        """

        dict_npm = {}
        new_file_name = ""

        if file_name.endswith("defaults.go"):

            with open(file_name) as fh:

                line_list = filter(None, fh.read().splitlines())

                for line in line_list:
                    if line.endswith("service{"):
                        integration = line.split('"')[1]
                        dict_npm[integration] = {"name": integration}

            new_file_name = "{}aws.json".format(self.data_npm_dir)

        elif file_name.endswith("gcp_services.go"):

            with open(file_name) as fh:

                line_list = filter(None, fh.read().splitlines())

                for line in line_list:
                    if line.endswith(","):
                        integration = line.split('"')[3]
                        dict_npm[integration] = {"name": integration}

            new_file_name = "{}gcp.json".format(self.data_npm_dir)

        if new_file_name != "":
            with open(
                    file=new_file_name,
                    mode="w",
                    encoding="utf-8",
                ) as f:
                    json.dump(
                        dict_npm, f, indent = 2, sort_keys = True
                    )

def process_npm_integrations ( self , file_name ) : dict_npm = { } new_file_name = "" if file_name . endswith ( "defaults.go" ) : with open ( file_name ) as fh : line_list = filter ( None , fh . read ( ) . splitlines ( ) ) for line in line_list : if line . endswith ( "service{" ) : integration = line . split ( '"' ) [ 1 ] dict_npm [ integration ] = { "name" : integration } new_file_name = "{}aws.json" . format ( self . data_npm_dir ) elif file_name . endswith ( "gcp_services.go" ) : with open ( file_name ) as fh : line_list = filter ( None , fh . read ( ) . splitlines ( ) ) for line in line_list : if line . endswith ( "," ) : integration = line . split ( '"' ) [ 3 ] dict_npm [ integration ] = { "name" : integration } new_file_name = "{}gcp.json" . format ( self . data_npm_dir ) if new_file_name != "" : with open ( file = new_file_name , mode = "w" , encoding = "utf-8" , ) as f : json . dump ( dict_npm , f , indent = 2 , sort_keys = True )


Copies a single image file to the static / images / folder creating a new directory if needed .

def process_images(self, file_name):
        """
        Copies a single image file to the static/images/ folder, creating a new directory if needed.
        """
        image_filename = basename(file_name) # img.png
        integration_image_path = file_name.replace('../', '') # if it found local marketplace repo
        integration_image_path = integration_image_path.replace('./integrations_data/extracted/', '') # marketplace/nerdvision/images/img.png
        integration_image_directory = dirname(integration_image_path) # marketplace/nerdvision/images/
        destination_directory = './static/images/{}'.format(integration_image_directory) # static/images/marketplace/nerdvision/images/
        full_destination_path = '{}/{}'.format(destination_directory, image_filename) # static/images/marketplace/nerdvision/images/img.png

        makedirs(destination_directory, exist_ok=True)
        copyfile(file_name, full_destination_path)


Takes a markdown string and replaces any image markdown with our img shortcode pointing to the static / images folder . This is needed when dealing with Marketplace Integrations to properly display images pulled from a private repo .

def replace_image_src(markdown_string, integration_name):
        """
        Takes a markdown string and replaces any image markdown with our img shortcode, pointing to the static/images folder.
        This is needed when dealing with Marketplace Integrations to properly display images pulled from a private repo.
        """
        markdown_img_search_regex = r"!\[(.*?)\]\((.*?)\)"
        img_shortcode = "{{< img src=\"marketplace/" + integration_name + "/\\2\" alt=\"\\1\" >}}"
        integration_img_prefix = 'https://raw.githubusercontent.com/DataDog/marketplace/master/{}/'.format(integration_name)

        replaced_markdown_string = markdown_string.replace(integration_img_prefix, '')
        regex_result = re.sub(markdown_img_search_regex, img_shortcode, replaced_markdown_string, 0, re.MULTILINE)

        if regex_result:
            return regex_result
        else:
            return markdown_string


Removes a section from markdown by deleting all content starting from provided h2_header_string argument and ending one index before the next h2 header . h2_header_string argument is expected in markdown format ; e . g . ## Steps

def remove_markdown_section(markdown_string, h2_header_string):
        """
        Removes a section from markdown by deleting all content starting from provided h2_header_string argument and ending one index before the next h2 header.
        h2_header_string argument is expected in markdown format; e.g. '## Steps'
        """

        if not h2_header_string.startswith('##'):
            return markdown_string

        h2_markdown_regex = r"(^|\n)(#{2}) (\w+)"
        h2_list = re.finditer(h2_markdown_regex, markdown_string)
        replaced_result = ''

        for match in h2_list:
            group = match.group(0)
            start = match.start()
            end = match.end() - 1

            if h2_header_string in group:
                start_index = start
                end_index = next(h2_list).start()
                content_to_remove = markdown_string[start_index:end_index]
                replaced_result = markdown_string.replace(content_to_remove, '')

        if replaced_result:
            return replaced_result
        else:
            return markdown_string

def
remove_markdown_section
(
markdown_string
,
h2_header_string
)
:
if
not
h2_header_string
.
startswith
(
'##'
)
:
return
markdown_string
h2_markdown_regex
=
r"(^|\n)(#{2}) (\w+)"
h2_list
=
re
.
finditer
(
h2_markdown_regex
,
markdown_string
)
replaced_result
=
''
for
match
in
h2_list
:
group
=
match
.
group
(
0
)
start
=
match
.
start
(
)
end
=
match
.
end
(
)
1
if
h2_header_string
in
group
:
start_index
=
start
end_index
=
next
(
h2_list
)
.
start
(
)
content_to_remove
=
markdown_string
[
start_index
:
end_index
]
replaced_result
=
markdown_string
.
replace
(
content_to_remove
,
''
)
if
replaced_result
:
return
replaced_result
else
:
return
markdown_string


Validates marketplace integration markdown string does not contain sensitive content . The build should fail if we found any sections that should not be displayed in Docs . Current exclude list : [ Setup Pricing Tiered Pricing ]

def validate_marketplace_integration_markdown(markdown_string):
        """
        Validates marketplace integration markdown string does not contain sensitive content.
        The build should fail if we found any sections that should not be displayed in Docs.
        Current exclude list: ["Setup", "Pricing", "Tiered Pricing"]
        """
        setup_header_markdown_regex = r"(#{1,6})(\s*)(Setup|Pricing|Tiered Pricing)"
        matches = re.search(setup_header_markdown_regex, markdown_string, re.MULTILINE | re.IGNORECASE)
        return matches == None


Take a single README . md file and 1 . extract the first h1 if this isn t a merge item 2 . add tabs if they exist 3 . inject metrics after ### Metrics header if metrics exists for file 4 . inject service checks after ### Service Checks if file exists 5 . inject hugo front matter params at top of file 6 . write out file to content / integrations with filename changed to integrationname . md : param file_name : path to a readme md file

def process_integration_readme(self, file_name, marketplace=False):
        """
        Take a single README.md file and
        1. extract the first h1, if this isn't a merge item
        2. add tabs if they exist
        3. inject metrics after ### Metrics header if metrics exists for file
        4. inject service checks after ### Service Checks if file exists
        5. inject hugo front matter params at top of file
        6. write out file to content/integrations with filename changed to integrationname.md
        :param file_name: path to a readme md file
        """
        no_integration_issue = True
        tab_logic = False
        metrics = glob.glob(
            "{path}{sep}*metadata.csv".format(
                path=dirname(file_name), sep=sep
            )
        )
        metrics = metrics[0] if len(metrics) > 0 else None
        metrics_exist = (metrics and exists(metrics)
                         and linecache.getline(metrics, 2))
        service_check = glob.glob("{file}.json".format(
            file=self.data_service_checks_dir + basename(dirname(file_name))))
        service_check = (
            service_check[0]
            if len(service_check) > 0
            else None
        )
        service_check_exist = service_check and exists(
            service_check
        )
        manifest = "{0}{1}{2}".format(
            dirname(file_name), sep, "manifest.json"
        )

        if exists(manifest):
            try:
                manifest_json = json.load(open(manifest))
                manifest_json = self.process_manifest(manifest_json, basename(dirname(file_name)))
            except JSONDecodeError:
                no_integration_issue = False
                manifest_json = {}
                if getenv("LOCAL") == 'True':
                    print(
                        "\x1b[33mWARNING\x1b[0m: manifest could not be parsed {}".format(manifest))
                else:
                    print(
                        "\x1b[31mERROR\x1b[0m: manifest could not be parsed {}".format(manifest))
                    raise JSONDecodeError
        else:
            no_integration_issue = False
            manifest_json = {}
            print(
                "\x1b[33mWARNING\x1b[0m: No manifest found for {}".format(file_name))

        dependencies = self.add_dependencies(file_name)
        new_file_name = "{}.md".format(
            basename(dirname(file_name))
        )
        # is this the same as a committed hardcoded integration
        exist_already = (self.content_integrations_dir + new_file_name in self.initial_integration_files)
        # is this overwriting another generated integration
        exist_collision = exists(
            self.content_integrations_dir + new_file_name
        )

        regex_skip_sections_end = r"(```|\{\{< \/code-block |\{\{< \/site-region >\}\})"
        regex_skip_sections_start = r"(```|\{\{< code-block |\{\{< site-region)"

        ## Formating all link as reference to avoid any corner cases
        ## Replace image filenames in markdown for marketplace interations
        result = ''
        if not marketplace:
            try:
                result = format_link_file(file_name,regex_skip_sections_start,regex_skip_sections_end)
            except Exception as e:
                print(e)
        else:
            with open(file_name, 'r+') as f:
                markdown_string = f.read()
                markdown_with_replaced_images = self.replace_image_src(markdown_string, basename(dirname(file_name)))
                updated_markdown = self.remove_markdown_section(markdown_with_replaced_images, '## Setup')
                is_marketplace_integration_markdown_valid = self.validate_marketplace_integration_markdown(updated_markdown)

                if not is_marketplace_integration_markdown_valid:
                    raise Exception('Potential setup or pricing information included in Marketplace Integration markdown.  Check {} for Setup or Pricing sections.'.format(file_name))
                else:
                    result = updated_markdown

        ## Check if there is a integration tab logic in the integration file:
        if "<!-- xxx tabs xxx -->" in result:
            tab_logic = True
            ## Inlining all links
            result = self.inline_references(result,regex_skip_sections_start,regex_skip_sections_end)
        else:
            tab_logic= False

        title = manifest_json.get("name", "").lower()
        if title not in [
            k
            for k, v in self.integration_mutations.items()
            if v.get("action") == "merge"
        ]:
            result = re.sub(
                self.regex_h1, "", result, 1
            )
            result = re.sub(
                self.regex_tabs_open, "{{< tabs >}}", result, 0
            )
            result = re.sub(
                self.regex_tabs_close, "{{< /tabs >}}", result, 0
            )
            result = re.sub(
                self.regex_tab_open, "{{% tab", result, 0
            )
            result = re.sub(
                self.regex_tab_close, "{{% /tab %}}", result, 0
            )
            result = re.sub(
                self.regex_tab_end, " %}}", result, 0
            )
            result = re.sub(
                self.regex_partial_open, "", result, 0
            )
            result = re.sub(
                self.regex_partial_close, "", result, 0
            )

        if metrics_exist:
            result = re.sub(
                self.regex_metrics,
                r'\1{{< get-metrics-from-git "%s" >}}\n\3\4'
                % format(title),
                result,
                0,
            )
        if service_check_exist:
            result = re.sub(
                self.regex_service_check,
                r'\1{{< get-service-checks-from-git "%s" >}}\n\3\4'
                % format(title),
                result,
                0,
            )

        # if __init__.py exists lets grab the integration id
        integration_id = manifest_json.get("integration_id", "") or ""
        initpy = "{0}{1}{2}".format(dirname(file_name), sep, "__init__.py")
        if exists(initpy):
            with open(initpy) as f:
                # look for ID = "integration-name" and extract
                matches = re.search("^ID\s*=\s*(?:\'|\")([A-Z-a-z-_0-9]+)(?:\'|\")$", f.read(), re.MULTILINE)
                if matches:
                    integration_id = matches.group(1)

        # if __about__.py exists lets grab the integration version
        integration_version = manifest_json.get("integration_version", "") or ""
        integration_name = basename(dirname(file_name))
        aboutpy = "{0}{1}{2}{3}{4}{5}{6}".format(dirname(file_name), sep, "datadog_checks", sep, integration_name, sep, "__about__.py")

        if exists(aboutpy):
            with open(aboutpy) as f:
                # look for version = "integration-version" and extract
                matches = re.search("^__version__\s*=\s*(?:\'|\")([0-9.]+)(?:\'|\")$", f.read(), re.MULTILINE)
                if matches:
                    integration_version = matches.group(1)

        if not exist_already and no_integration_issue:
            # lets only write out file.md if its going to be public
            if manifest_json.get("is_public", False):
                out_name = self.content_integrations_dir + new_file_name

                # lets make relative app links to integrations tile absolute
                regex = r"(?<!https://app.datadoghq.com)(/account/settings#integrations[^.)\s]*)"
                regex_result = re.sub(regex, "https://app.datadoghq.com\\1", result, 0, re.MULTILINE)
                if regex_result:
                    result = regex_result

                # if the same integration exists in multiple locations try name md file differently
                # integration_id.md -> name.md -> original_collision_name.md
                if exist_collision:
                    f_name = integration_id.replace('-', '_') or manifest_json.get("name", "") or new_file_name
                    manifest_json["name"] = f_name
                    f_name = f_name if f_name.endswith('.md') else f_name + ".md"
                    out_name = self.content_integrations_dir + f_name
                    print("\x1b[33mWARNING\x1b[0m: Collision, duplicate integration {} trying as {}".format(
                        new_file_name, f_name))
                    result = self.add_integration_frontmatter(
                        f_name, result, dependencies, integration_id, integration_version, manifest_json
                    )
                else:
                    result = self.add_integration_frontmatter(
                        new_file_name, result, dependencies, integration_id, integration_version
                    )

                with open(out_name, "w", ) as out:
                    out.write(result)

                ## Reformating all links now that all processing is done
                if tab_logic:
                    final_text = format_link_file(out_name, regex_skip_sections_start, regex_skip_sections_end)
                    with open(out_name, 'w') as final_file:
                        final_file.write(final_text)


Takes an integration README . md and injects front matter yaml based on manifest . json data of the same integration : param file_name : new integration markdown filename e . g airbrake . md : param content : string of markdown content : return : formatted string

def add_integration_frontmatter(
        self, file_name, content, dependencies=[], integration_id="", integration_version="", manifest_json=None
    ):
        """
        Takes an integration README.md and injects front matter yaml based on manifest.json data of the same integration
        :param file_name: new integration markdown filename e.g airbrake.md
        :param content: string of markdown content
        :return: formatted string
        """
        fm = {}
        template = "---\n{front_matter}\n---\n\n{content}\n"
        if file_name not in self.initial_integration_files:
            if manifest_json:
                item = manifest_json
            else:
                matches = [
                    d
                    for d in self.datafile_json
                    if d.get("name", "").lower() == basename(file_name).replace(".md", "")
                ]
                item = matches[0] if len(matches) > 0 else []
            if item:
                item["kind"] = "integration"
                item["integration_title"] = (
                    item
                    .get("public_title", "")
                    .replace("Datadog-", "")
                    .replace("Integration", "")
                    .strip()
                )
                item["git_integration_title"] = (
                    item.get("name", "").lower()
                )
                if item.get("type", None):
                    item["ddtype"] = item.get("type")
                    del item["type"]
                item["dependencies"] = dependencies
                item["draft"] = not item.get("is_public", False)
                item["integration_id"] = item.get("integration_id", integration_id)
                item["integration_version"] = item.get("integration_version", integration_version)
                fm = yaml.safe_dump(
                    item, width=float("inf"), default_style='"', default_flow_style=False, allow_unicode=True
                ).rstrip()
                # simple bool cleanups with replace
                fm = fm.replace('!!bool "false"', 'false')
                fm = fm.replace('!!bool "true"', 'true')
            else:
                fm = yaml.safe_dump({"kind": "integration"}, width=float("inf"), default_style='"', default_flow_style=False,
                                    allow_unicode=True).rstrip()
        return template.format(
            front_matter=fm, content=content
        )


def add_dependencies(self, file_name):
        """ Adds dependencies to the integration file in order to be able to find the source file in Github """
        dependencies = []
        if file_name.startswith(
            "{0}{1}{2}".format(
                self.extract_dir, "integrations-core", sep
            )
        ):
            dependencies.append(
                file_name.replace(
                    "{0}{1}{2}".format(
                        self.extract_dir,
                        "integrations-core",
                        sep,
                    ),
                    "https://github.com/DataDog/integrations-core/blob/master/",
                )
            )

        elif file_name.startswith(
            "{0}{1}{2}".format(
                self.extract_dir, "integrations-extras", sep
            )
        ):
            dependencies.append(
                file_name.replace(
                    "{0}{1}{2}".format(
                        self.extract_dir,
                        "integrations-extras",
                        sep,
                    ),
                    "https://github.com/DataDog/integrations-extras/blob/master/",
                )
            )

        return dependencies


def process_manifest(self, manifest_json, name):
        """ Takes manifest and converts v2 and above to v1 expected formats for now """
        manifest_version = (manifest_json.get("manifest_version", '1.0.0') or '1.0.0').split('.')
        split_version = manifest_version[0] if len(manifest_version) > 1 else '1'
        if split_version != '1':
            # v2 or above
            manifest_json["integration_id"] = manifest_json.get("app_id", "")
            categories = []
            supported_os = []
            for tag in manifest_json.get("classifier_tags", []):
                # in some cases tag was null/None
                if tag:
                    key, value = tag.split("::")
                    if key.lower() == "category":
                        categories.append(value.lower())
                    if key.lower() == "supported os":
                        supported_os.append(value.lower())
            manifest_json["categories"] = categories
            manifest_json["supported_os"] = supported_os
            manifest_json["public_title"] = manifest_json.get("tile", {}).get("title", '')
            manifest_json["is_public"] = manifest_json.get("display_on_public_website", False)
            manifest_json["short_description"] = manifest_json.get("tile", {}).get("description", '')
            manifest_json["name"] = manifest_json.get("name", name)
        return manifest_json

def
process_manifest
(
self
,
manifest_json
,
name
)
:
manifest_version
=
(
manifest_json
.
get
(
"manifest_version"
,
'1.0.0'
)
or
'1.0.0'
)
.
split
(
'.'
)
split_version
=
manifest_version
[
0
]
if
len
(
manifest_version
)
>
1
else
'1'
if
split_version
!=
'1'
:
# v2 or above
manifest_json
[
"integration_id"
]
=
manifest_json
.
get
(
"app_id"
,
""
)
categories
=
[
]
supported_os
=
[
]
for
tag
in
manifest_json
.
get
(
"classifier_tags"
,
[
]
)
:
# in some cases tag was null/None
if
tag
:
key
,
value
=
tag
.
split
(
"::"
)
if
key
.
lower
(
)
==
"category"
:
categories
.
append
(
value
.
lower
(
)
)
if
key
.
lower
(
)
==
"supported os"
:
supported_os
.
append
(
value
.
lower
(
)
)
manifest_json
[
"categories"
]
=
categories
manifest_json
[
"supported_os"
]
=
supported_os
manifest_json
[
"public_title"
]
=
manifest_json
.
get
(
"tile"
,
{
}
)
.
get
(
"title"
,
''
)
manifest_json
[
"is_public"
]
=
manifest_json
.
get
(
"display_on_public_website"
,
False
)
manifest_json
[
"short_description"
]
=
manifest_json
.
get
(
"tile"
,
{
}
)
.
get
(
"description"
,
''
)
manifest_json
[
"name"
]
=
manifest_json
.
get
(
"name"
,
name
)
return
manifest_json


Goes through a file and parses it into different sections . Those sections are a list of lines and are put within an Array . The first item of the Array is the main section all other item if any are sub sections a . k . a tabs within the page .

def prepare_file(file):
    """
    Goes through a file and parses it into different sections. Those sections are a list of lines and are put within an Array.
    The first item of the Array is the main section, all other item if any are sub sections, a.k.a tabs within the page.

    :param file: file to break down into sections.
    :return array_of_sections: Each section is a list of lines within this section.
    """

    # A state machine is used here, the function takes line within the file one by one, and depending of the states, either
    # 1. Consider it is in the main section
    # 2. Consider it is within a tabs group
    # 3. Consider within a tab
    #
    # We keep the {{< tabs >}} lines and co. in the main section since it will be used to inline the proper content afterwards.
    state = 'main'

    main_section = []
    sub_sections = []
    temp_section = []

    with open(file, 'r', encoding='utf-8') as f:
        for line in f:
            if state == 'main':
                main_section.append(line)
                if (re.search(r"{{< tabs >}}", line.strip()) or re.search(r"{{< programming-lang-wrapper", line.strip())):
                    state = 'tabs'
            elif state == 'tabs':
                main_section.append(line)
                if (re.search(r"{{% tab ", line.strip()) or re.search(r"{{< programming-lang ", line.strip())):
                    state = 'tab'
                if (re.search(r"{{< /tabs >}}", line.strip()) or re.search(r"{{< /programming-lang-wrapper >}}", line.strip())):
                    state = 'main'
            elif state == 'tab':
                if (re.search(r"{{% /tab %}}", line.strip()) or re.search(r"{{< /programming-lang >}}", line.strip())):
                    state = 'tabs'
                    main_section.append(line)
                    sub_sections.append(temp_section)
                    temp_section = []
                else:
                    temp_section.append(line)

    if state != 'main':
        raise ValueError

    return [main_section] + sub_sections


Goes through a list of reference link and dedupe it and if two references share the same index throw an error .

def check_references(all_references):
    """
    Goes through a list of reference link and dedupe it, and if two references share the same index, throw an error.

    :param all_references: An array of references
    :return all_references_deduped: An array of references deduped.
    """
    all_references_deduped = []
    reference_indexes_used = []
    duplicated_references = []
    is_duplicated = False

    for reference in all_references:
        if reference not in all_references_deduped:

            reference_index, reference_val = reference

            if reference_index not in reference_indexes_used:
                reference_indexes_used.append(reference_index)
                all_references_deduped.append(reference)
            else:
                duplicated_references.append(reference)
                is_duplicated = True

    if is_duplicated:
        for duplicated_reference in duplicated_references:
            duplicated_reference_index, duplicated_reference_val = duplicated_reference
            print('Duplicated reference: [{}]: {}'.format(
                duplicated_reference_index, duplicated_reference_val))
        raise AssertionError

    return all_references_deduped

def
check_references
(
all_references
)
:
all_references_deduped
=
[
]
reference_indexes_used
=
[
]
duplicated_references
=
[
]
is_duplicated
=
False
for
reference
in
all_references
:
if
reference
not
in
all_references_deduped
:
reference_index
,
reference_val
=
reference
if
reference_index
not
in
reference_indexes_used
:
reference_indexes_used
.
append
(
reference_index
)
all_references_deduped
.
append
(
reference
)
else
:
duplicated_references
.
append
(
reference
)
is_duplicated
=
True
if
is_duplicated
:
for
duplicated_reference
in
duplicated_references
:
duplicated_reference_index
,
duplicated_reference_val
=
duplicated_reference
print
(
'Duplicated reference: [{}]: {}'
.
format
(
duplicated_reference_index
,
duplicated_reference_val
)
)
raise
AssertionError
return
all_references_deduped


def remove_reference(section, regex_skip_sections_start,
                     regex_skip_sections_end):
    """
    Goes through a section and remove all reference links it can found.

    :param section: An array of lines representing a section.
    :param regex_skip_sections_start: regex defining the start line that indicates a block of line that shouldn't be processed.
    :param regex_skip_sections_end: regex defining the end line that indicates a block of line that shouldn't be processed.
    :return [all_references, section_without_references]: Returns an array of array, the first item contains all references found in the section
                                                          The second item contains the section without any reference in it.
    """
    skip = False
    all_references = []
    section_without_references = []
    regex_bottom_reference_link = r"^\s*\[(\d*?)\]: (\S*)"

    # Collecting all references and removing them from section
    # looking at each line, if a line is a reference then we remove it and store the reference.

    for line in section:
        if skip:
            section_without_references.append(line)
            if re.search(regex_skip_sections_end, line):
                skip = False
        elif not skip:

            if re.search(regex_skip_sections_start, line):
                section_without_references.append(line)
                skip = True

            elif re.search(regex_bottom_reference_link, line):

                reference = re.search(regex_bottom_reference_link, line)
                all_references.append([reference.group(1),
                                       reference.group(2)])
            else:
                section_without_references.append(line)
    # By the end of the for loop skip should always be false otherwise it means that a codeblock is not closed.
    if skip:
        raise ValueError

    try:
        all_references_checked = check_references(all_references)
    except AssertionError:
        raise AssertionError

    return [all_references_checked, section_without_references]


Goes through a section with a list of references and inline all references .

def inlining_all_links(
        section_without_references,
        all_references,
        regex_skip_sections_start,
        regex_skip_sections_end):
    """
    Goes through a section with a list of references and inline all references.

    :param section_without_references: An array of lines representing a section, for this function to work, all reference should be removed previously from the section.
    :param regex_skip_sections_start: regex defining the start line that indicates a block of line that shouldn't be processed.
    :param regex_skip_sections_end: regex defining the end line that indicates a block of line that shouldn't be processed.
    :return section_with_all_links: A section (an array of lines) is returned without any references in it, just pure inlined links.
    """

    section_with_all_links = []
    skip = False

    for line in section_without_references:
        if skip:
            if re.search(regex_skip_sections_end, line):
                skip = False
        elif not skip:
            if re.search(regex_skip_sections_start, line):
                skip = True
            else:
                for reference in all_references:
                    reference_index, reference_val = reference

                    curent_link = '][' + reference_index + ']'

                    line = line.replace(
                        curent_link, '](' + reference_val + ')')

        section_with_all_links.append(line)

    # By the end of the for loop skip should always be false otherwise it means that a codeblock is not closed.
    if skip:
        raise ValueError

    return section_with_all_links


Goes through a section and extract all inlined links it can found .

def collect_all_links(section_with_all_links,
                      regex_skip_sections_start,
                      regex_skip_sections_end):
    """
    Goes through a section and extract all inlined links it can found.

    :param section_with_all_links: An array of lines representing a section. For this function to work, all links must be inlined first.
    :param regex_skip_sections_start: regex defining the start line that indicates a block of line that shouldn't be processed.
    :param regex_skip_sections_end: regex defining the end line that indicates a block of line that shouldn't be processed.
    :return all_links: An array of all unique links that where found within a section.
    """
    regex_link_inlined = r"\[.*?\]\((?![#?])(\S*?)\)"
    all_links = []
    skip = False
    for line in section_with_all_links:
        if skip:
            if re.search(regex_skip_sections_end, line):
                skip = False
        elif not skip:
            if re.search(regex_skip_sections_start, line):
                skip = True
            else:
                line_links = re.findall(regex_link_inlined, line,
                                        re.MULTILINE)
                if not line_links == []:
                    for link in line_links:

                        # If the link is already in the array, then it doesn't add it to avoid duplicated link

                        if link not in all_links:
                            all_links.append(link)

    # By the end of the for loop skip should always be false otherwise it means that a codeblock is not closed.
    if skip:
        raise ValueError

    return all_links


def transform_link_to_references(
    section_with_all_links,
    all_links,
    regex_skip_sections_start,
    regex_skip_sections_end,
):
    """
    Goes through a section where all link are inlined and transform them in references

    :param section_with_all_links: An array of lines representing a section where all link are inlined.
    :param all_links: An array of links representing all unique list associated to section_with_all_links.
    :param regex_skip_sections_start: regex defining the start line that indicates a block of line that shouldn't be processed.
    :param regex_skip_sections_end: regex defining the end line that indicates a block of line that shouldn't be processed.
    :return section_with_references: A section (an array of lines), with all inlined links transformed into a reference link.
    """

    section_with_references = []

    skip = False

    for line in section_with_all_links:
        if skip:
            if re.search(regex_skip_sections_end, line):
                skip = False
        elif not skip:
            if re.search(regex_skip_sections_start, line):
                skip = True
            else:
                for i, link in enumerate(all_links):
                    link_to_reference = '](' + str(link) + ')'
                    # i is incremented by one in order to start references indexes at 1
                    line = line.replace(link_to_reference,
                                        '][' + str(i + 1) + ']')

        section_with_references.append(line)

    # By the end of the for loop skip should always be false otherwise it means that a codeblock is not closed.
    if skip:
        raise ValueError

    return section_with_references


Goes through a section . A section is an array of lines from a file and format all links to transform them into reference link .

def process_section(section, regex_skip_sections_start,
                    regex_skip_sections_end):
    """
    Goes through a section. A section is an array of lines from a file and format all links to transform them into reference link.

    :param section: Array of lines to analyse.
    :param regex_skip_sections_start: regex defining the start line that indicates a block of line that shouldn't be processed.
    :param regex_skip_sections_end: regex defining the end line that indicates a block of line that shouldn't be processed.
    :return section_with_references: Returns the section but with all link set as reference.
    """
    try:
        all_references, section_without_references = remove_reference(
            section, regex_skip_sections_start, regex_skip_sections_end)
    except AssertionError:
        print('\x1b[31mERROR\x1b[0m: Some references are duplicated.')
        raise AssertionError
    except ValueError:
        print('\x1b[31mERROR\x1b[0m: A skip section is not closed')
        raise ValueError

    # Inlining refrences
    # Looking at each line, it replaces reference link [.*][\d] by the full inlined link

    section_with_all_links = \
        inlining_all_links(section_without_references, all_references,
                           regex_skip_sections_start,
                           regex_skip_sections_end)

    # Collecting all links from file
    # Looking at each line, it extracts all links it can found and add it to all_links array

    all_links = collect_all_links(section_with_all_links,
                                  regex_skip_sections_start,
                                  regex_skip_sections_end)

    # Now that all link are extracted, it creates a new section with all inlined referenced link

    section_with_references = \
        transform_link_to_references(section_with_all_links, all_links,
                                     regex_skip_sections_start, regex_skip_sections_end)

    # Finally it adds all refrerences at the end of the section

    for i, link in enumerate(all_links):

        # 1 is added to i in order to start references link index at 1 instead of 0

        section_with_references.append('[' + str(i + 1) + ']: ' + link + '\n')

    return section_with_references


Takes a prepared file ( and Array of sections prepared ) and transform it into the final file that can be written back .

def inline_section(file_prepared):
    """
    Takes a prepared file (and Array of sections prepared) and transform it into the final file that can be written back.

    :param file_prepared: Array of sections, where the first item is the main section, an all the other one are sub sections in order of appearance.
    :return final_text: Returns text with all sections with reference link inlined in the main section.
    """
    # inlining sections

    final_text = []

    end_section_pattern = r"\s*{{% /tab %}}.*"
    end_lang_section_pattern = r"\s*{{< /programming-lang >}}.*"

    i = 1

    try:
        for line in file_prepared[0]:
            if (re.match(end_section_pattern, line) or re.match(end_lang_section_pattern, line)):
                final_text += file_prepared[i]
                i += 1
            final_text.append(line)
    except:
        raise ValueError

    return ''.join(final_text)

def
inline_section
(
file_prepared
)
:
# inlining sections
final_text
=
[
]
end_section_pattern
=
r"\s*{{% /tab %}}.*"
end_lang_section_pattern
=
r"\s*{{< /programming-lang >}}.*"
i
=
1
try
:
for
line
in
file_prepared
[
0
]
:
if
(
re
.
match
(
end_section_pattern
,
line
)
or
re
.
match
(
end_lang_section_pattern
,
line
)
)
:
final_text
+=
file_prepared
[
i
]
i
+=
1
final_text
.
append
(
line
)
except
:
raise
ValueError
return
''
.
join
(
final_text
)


Take a file and transform all links into reference link within this file .

def format_link_file(file, regex_skip_sections_start,
                     regex_skip_sections_end):
    """
    Take a file and transform all links into reference link within this file.

    :param file: Array of lines to analyse.
    :param regex_skip_sections_start: regex defining the start line that indicates a block of line that shouldn't be processed.
    :param regex_skip_sections_end: regex defining the end line that indicates a block of line that shouldn't be processed.
    :return final_text: Returns the file but with all links set as reference.
    """

    try:
        prepared_file = prepare_file(file)
    except ValueError:
        print("\x1b[31mERROR\x1b[0m: Couldn't split the file into multiple section correctly for file: {}".format(file))
        raise ValueError

    final_text = []

    for section in prepared_file:
        try:
            final_text.append(process_section(section,
                                              regex_skip_sections_start,
                                              regex_skip_sections_end))
        except:
            print(
                '\x1b[31mERROR\x1b[0m: There was an issue processing a section for file: {}'.format(file))
            raise ValueError

    try:
        file_with_references = inline_section(final_text)
    except ValueError:
        print(
            '\x1b[31mERROR\x1b[0m: Could not inline sections properly for file: {}'.format(file))
        raise ValueError

    return file_with_references