Spaces:
Build error
Build error
File size: 6,336 Bytes
cbbc229 ab137d9 985fb81 cbbc229 ab137d9 985fb81 cbbc229 9f51aca cbbc229 9f51aca cbbc229 9f51aca cbbc229 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 |
"""
Notes
-----
This module contains the functions for audiobook_gen that read in the
file formats that require for parsing than plain text (pdf, html, epub),
as well as the preprocessing function for all input files.
"""
import re
from bs4 import BeautifulSoup
from nltk import tokenize, download
from textwrap import TextWrapper
from stqdm import stqdm
from src import config
download('punkt', quiet=True)
wrapper = TextWrapper(config.MAX_CHAR_LEN, fix_sentence_endings=True)
def preprocess_text(file):
"""
Preprocesses and tokenizes a section of text from the corpus:
1. Removes residual HTML tags
2. Handles un-supported characters
3. Tokenizes text and confirms max token size
Parameters
----------
file : file_like
list of strings,
section of corpus to be pre-processed and tokenized
Returns
-------
text_list : : array_like
list of strings,
body of tokenized text from which audio is generated
"""
input_text = BeautifulSoup(file, "html.parser").text
text_list = []
for paragraph in input_text.split('\n'):
paragraph = paragraph.replace('—', '-')
paragraph = paragraph.replace(' .', '')
paragraph = re.sub(r'[^\x00-\x7f]', "", paragraph)
paragraph = re.sub(r'x0f', " ", paragraph)
sentences = tokenize.sent_tokenize(paragraph)
sentence_list = []
for sentence in sentences:
if any(chr.isdigit() for chr in sentence):
sentence = extract_replace(sentence)
sentence = replace_symbols(sentence)
if not re.search('[a-zA-Z]', sentence):
sentence = ''
wrapped_sentences = wrapper.wrap(sentence)
sentence_list.append(wrapped_sentences)
trunc_sentences = [phrase for sublist in sentence_list for phrase in sublist]
text_list.append(trunc_sentences)
text_list = [text for sentences in text_list for text in sentences]
return text_list
def extract_replace(entry_string):
import inflect
result = (entry_string + '.')[:-1]
p = inflect.engine()
i = 0
#initialize array with three random numbers to enter the loop, then find if there are numbers or not.
array = [3 , 2 , 3]
#take every number from the entry string, locate and store the number in digits in a sentence (using find_num_index), apply number_to_words
#to that number specifically then replace it back in the sentence.
while(len(array) > 2):
#update array with first and last indexes of every number in digits in a sentence
array = find_num_index(result)
number = result[array[i] : array[i+1] + 1]
k = p.number_to_words(number)
position = array[i]
number_of_characters = array[i+1] - array[i] + 1
#update sentence with the new word to numbers until there are no numbers in digits left
result = result[:position] + k + result[position + number_of_characters:]
return result
def find_num_index(entry_string):
result0 = []
#fill result0 array with all the indexes of digit characters in a sentence
for i in range(len(entry_string)):
if (entry_string[i].isdigit() == True):
result0.append(i)
result1 = []
try:
result1.append(result0[0])
except IndexError:
result0 = 'null'
if(result0 != 'null'):
# append only indexes of first and last characters of numbers to result1 array
for k in range(len(result0) - 1):
if ((result0[k+1] - result0[k]) > 2):
result1.append(result0[k])
result1.append(result0[k+1])
try:
result1.append(result0[len(result0) - 1])
except IndexError:
result1 = 'null'
# return array of even length that contains first and last index of every number in a sentence
return result1
def replace_symbols(text):
import re
symbol_map = {
'+': ' plus ',
'-': ' minus ',
'—': ' dash ',
'=': ' equals ',
'≈': ' approximately equal to ',
'*': ' times ',
'%': ' percent ',
'/': ' divided by ',
'#': ' number ',
'@': ' at ',
'&': ' ampersand ',
'°': ' degrees '
}
symbol_regex = re.compile('|'.join(re.escape(key) for key in symbol_map.keys()))
text = symbol_regex.sub(lambda x: symbol_map[x.group()], text)
return text
def read_pdf(file):
"""
Invokes PyPDF2 PdfReader to extract main body text from PDF file_like input,
and preprocesses text section by section.
Parameters
----------
file : file_like
PDF file input to be parsed and preprocessed
Returns
-------
corpus : array_like
list of list of strings,
body of tokenized text from which audio is generated
"""
from PyPDF2 import PdfReader
reader = PdfReader(file)
corpus = []
for item in stqdm(list(reader.pages), desc="Pages in pdf:"):
text_list = preprocess_text(item.extract_text())
corpus.append(text_list)
return corpus
def read_epub(file):
"""
Invokes ebooklib read_epub to extract main body text from epub file_like input,
and preprocesses text section by section.
Parameters
----------
file : file_like
EPUB file input to be parsed and preprocessed
Returns
-------
corpus : array_like
list of list of strings,
body of tokenized text from which audio is generated
file_title : str
title of document, used to name output files
"""
from pathlib import Path
import ebooklib
from ebooklib import epub
from tempfile import NamedTemporaryFile
with NamedTemporaryFile(dir='.', suffix='.csv') as f:
f.write(file.getbuffer())
file = Path(f.name)
book = epub.read_epub(file)
file_title = book.get_metadata('DC', 'title')[0][0]
file_title = file_title.lower().replace(' ', '_')
corpus = []
for item in stqdm(list(book.get_items()), desc="Chapters in ebook:"):
if item.get_type() == ebooklib.ITEM_DOCUMENT:
text_list = preprocess_text(item.get_content())
corpus.append(text_list)
return corpus, file_title
|