src/file_readers.py

"""
Notes
-----
This module contains the functions for audiobook_gen that read in the
file formats that require for parsing than plain text (pdf, html, epub),
as well as the preprocessing function for all input files.
"""
import re

from bs4 import BeautifulSoup
from nltk import tokenize, download
from textwrap import TextWrapper
from stqdm import stqdm

from src import config

download('punkt', quiet=True)
wrapper = TextWrapper(config.MAX_CHAR_LEN, fix_sentence_endings=True)


def preprocess_text(file):
    """
    Preprocesses and tokenizes a section of text from the corpus:
    1. Removes residual HTML tags
    2. Handles un-supported characters
    3. Tokenizes text and confirms max token size

    Parameters
    ----------
    file : file_like
        list of strings,
        section of corpus to be pre-processed and tokenized

    Returns
    -------
    text_list :  : array_like
        list of strings,
        body of tokenized text from which audio is generated

    """
    input_text = BeautifulSoup(file, "html.parser").text
    text_list = []
    for paragraph in input_text.split('\n'):
        paragraph = paragraph.replace('—', '-')
        paragraph = paragraph.replace(' .', '')
        paragraph = re.sub(r'[^\x00-\x7f]', "", paragraph)
        paragraph = re.sub(r'x0f', " ", paragraph)
        sentences = tokenize.sent_tokenize(paragraph)

        sentence_list = []
        for sentence in sentences:
            if any(chr.isdigit() for chr in sentence):
                sentence = extract_replace(sentence)
            sentence = replace_symbols(sentence)
            if not re.search('[a-zA-Z]', sentence):
                sentence = ''
            wrapped_sentences = wrapper.wrap(sentence)
            sentence_list.append(wrapped_sentences)
        trunc_sentences = [phrase for sublist in sentence_list for phrase in sublist]
        text_list.append(trunc_sentences)
    text_list = [text for sentences in text_list for text in sentences]
    return text_list


def extract_replace(entry_string):
    import inflect
    
    result = (entry_string + '.')[:-1]
    p = inflect.engine()
    i = 0 

    #initialize array with three random numbers to enter the loop, then find if there are numbers or not.
    array = [3 , 2 , 3]

    #take every number from the entry string, locate and store the number in digits in a sentence (using find_num_index), apply number_to_words
    #to that number specifically then replace it back in the sentence.
    while(len(array) > 2):
        #update array with first and last indexes of every number in digits in a sentence
        array = find_num_index(result)
        number = result[array[i] : array[i+1] + 1]
        k = p.number_to_words(number)
        position = array[i]
        number_of_characters = array[i+1] - array[i] + 1

        #update sentence with the new word to numbers until there are no numbers in digits left
        result = result[:position] + k + result[position + number_of_characters:]

    return result


def find_num_index(entry_string): 
    result0 = []

    #fill result0 array with all the indexes of digit characters in a sentence
    for i in range(len(entry_string)):
        if (entry_string[i].isdigit() == True):
            result0.append(i)

    result1 = []

    try:
        result1.append(result0[0])
    except IndexError:
        result0 = 'null'
    if(result0 != 'null'):

    # append only indexes of first and last characters of numbers to result1 array 
        for k in range(len(result0) - 1):
            if ((result0[k+1] - result0[k]) > 2):
                result1.append(result0[k])
                result1.append(result0[k+1])
        try:
            result1.append(result0[len(result0) - 1])
        except IndexError:
            result1 = 'null'

    # return array of even length that contains first and last index of every number in a sentence
    return result1


def replace_symbols(text):
    import re
    
    symbol_map = {
        '+': ' plus ',
        '-': ' minus ',
        '—': ' dash ',
        '=': ' equals ',
        '≈': ' approximately equal to ',
        '*': ' times ',
        '%': ' percent ',
        '/': ' divided by ',
        '#': ' number ',
        '@': ' at ',
        '&': ' ampersand ',
        '°': ' degrees '
    }
    
    symbol_regex = re.compile('|'.join(re.escape(key) for key in symbol_map.keys()))
    text = symbol_regex.sub(lambda x: symbol_map[x.group()], text)
                              
    return text


def read_pdf(file):
    """
    Invokes PyPDF2 PdfReader to extract main body text from PDF file_like input,
    and preprocesses text section by section.

    Parameters
    ----------
    file : file_like
        PDF file input to be parsed and preprocessed

    Returns
    -------
    corpus : array_like
        list of list of strings,
        body of tokenized text from which audio is generated

    """
    from PyPDF2 import PdfReader

    reader = PdfReader(file)
    corpus = []
    for item in stqdm(list(reader.pages), desc="Pages in pdf:"):
        text_list = preprocess_text(item.extract_text())
        corpus.append(text_list)
    return corpus


def read_epub(file):
    """
    Invokes ebooklib read_epub to extract main body text from epub file_like input,
    and preprocesses text section by section.

    Parameters
    ----------
    file : file_like
        EPUB file input to be parsed and preprocessed

    Returns
    -------
    corpus : array_like
        list of list of strings,
        body of tokenized text from which audio is generated

    file_title : str
        title of document, used to name output files

    """
    from pathlib import Path
    
    import ebooklib
    from ebooklib import epub
    from tempfile import NamedTemporaryFile

    with NamedTemporaryFile(dir='.', suffix='.csv') as f:
        f.write(file.getbuffer())
        file = Path(f.name)
        book = epub.read_epub(file)
    file_title = book.get_metadata('DC', 'title')[0][0]
    file_title = file_title.lower().replace(' ', '_')
    corpus = []
    for item in stqdm(list(book.get_items()), desc="Chapters in ebook:"):
        if item.get_type() == ebooklib.ITEM_DOCUMENT:
            text_list = preprocess_text(item.get_content())
            corpus.append(text_list)
    return corpus, file_title