coverylanguagedetection1 / text_extraction.py
Felixogunwale's picture
Upload 18 files
a63752b
raw
history blame
13.3 kB
#importing of modules to be used to run the program
import extract
import csv
import os
import csv
import cv2
import logging
import pytesseract
import pandas as pd
import numpy as np
from scipy.stats import mode
from PIL import Image
import argparse
import os
import random
#from google.colab.patches import cv2_imshow
#import detectron2
#from detectron2.utils.logger import setup_logger
#setup_logger()
#from detectron2 import model_zoo
#from detectron2.engine import DefaultPredictor
#from detectron2.config import get_cfg
#from detectron2.utils.visualizer import Visualizer
import logging
import cv2
import numpy as np
from scipy.stats import mode
#import and unzip the dataset
#!ls
#!unzip "Text_Detection_Dataset_COCO_Format.zip"
#preparing the imported and extracted dataset with json
#import json
#from detectron2.structures import BoxMode
#def get_board_dicts(imgdir):
# json_file = imgdir+"/dataset.json"
# with open(json_file) as f:
# dataset_dicts = json.load(f)
# for i in dataset_dicts:
# filename = i["file_name"]
# i["file_name"] = imgdir+"/"+filename
# for j in i["annotations"]:
# j["bbox_mode"] = BoxMode.XYWH_ABS
# j["category_id"] = int(j["category_id"])
# return dataset_dicts
#preprocessing the image pre-processing and pattern matching.
#This python module can perform the following functions:
#Binarization - method binary_img(img) performs this function
#Skew correction - method skew_correction(img) performs this function
#Need to introduce machine learning of some sort to make the skew correction method run faster :( Or... A simple fix would be to resize the #image first, and then apply the skew correction method! That'll probably take lesser time...
logging.basicConfig(
level=logging.DEBUG,
format="%(levelname)s: %(asctime)s {%(filename)s:%(lineno)d}: %(message)s "
)
kernel = np.ones((5, 5), np.uint8)
clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8, 8))
img = cv2.imread('image_resize.png') # read image file to be processed
"""
Method to binarize an image
Input: Grayscale image
Output: Binary image
The nature of the output is such that the text(foreground) has a colour
value of (255,255,255), and the background has a value of (0,0,0).
"""
def binary_img(img):
# img_erode = cv2.dilate(img,kernel,iterations = 2)
blur = cv2.medianBlur(img, 5)
# mask1 = np.ones(img.shape[:2],np.uint8)
"""Applying histogram equalization"""
cl1 = clahe.apply(blur)
circles_mask = cv2.dilate(cl1, kernel, iterations=1)
circles_mask = (255 - circles_mask)
thresh = 1
circles_mask = cv2.threshold(circles_mask, thresh, 255, cv2.THRESH_BINARY)[1]
edges = cv2.Canny(cl1, 100, 200)
edges = cv2.bitwise_and(edges, edges, mask=circles_mask)
dilation = cv2.dilate(edges, kernel, iterations=1)
display = cv2.bitwise_and(img, img, mask=dilation)
cl2 = clahe.apply(display)
cl2 = clahe.apply(cl2)
ret, th = cv2.threshold(cl2, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
th = 255 - th
thg = cv2.adaptiveThreshold(display, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, \
cv2.THRESH_BINARY, 11, 2)
# final = cv2.bitwise_and(dilation,dilation,mask=th)
finalg = cv2.bitwise_and(dilation, dilation, mask=thg)
finalg = 255 - finalg
abso = cv2.bitwise_and(dilation, dilation, mask=finalg)
return abso
"""
Method to resize the image. This is going to help in reducing the number
of computations, as the size of data will reduce.
"""
def resize(img):
r = 1000.0 / img.shape[1]
dim = (1000, int(img.shape[0] * r))
resized = cv2.resize(img, dim, interpolation=cv2.INTER_AREA)
# cv2.imshow('resized', resized)
return resized
"""
Method to correct the skew of an image
Input: Binary image
Output: Skew corrected binary image
The nature of the output is such that the binary image is rotated appropriately
to remove any angular skew.
Find out the right place to insert the resizing method call.
Try to find one bounding rectangle around all the contours
"""
def skew_correction(img):
areas = [] # stores all the areas of corresponding contours
dev_areas = [] # stores all the areas of the contours within 1st std deviation in terms of area#stores all the white pixels of the largest contour within 1st std deviation
all_angles = []
k = 0
binary = binary_img(img)
# binary = resize(binary)
im2, contours, hierarchy = cv2.findContours(binary, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
# cnt = contours[0]
# upper_bound=len(contours)
height_orig, width_orig = img.shape[:2]
words = np.zeros(img.shape[:2], np.uint8)
for c in contours:
areas.append(cv2.contourArea(c))
std_dev = np.std(areas)
for i in areas:
dev_areas.append(i - std_dev)
dev_contours = np.zeros(img.shape[:2], np.uint8)
for i in dev_areas:
if ((i > (-std_dev)) and (i <= (std_dev))):
cv2.drawContours(dev_contours, contours, k, (255, 255, 255), -1)
k += 1
sobely = cv2.Sobel(dev_contours, cv2.CV_64F, 0, 1, ksize=5)
abs_sobel64f = np.absolute(sobely)
sobel_8u = np.uint8(abs_sobel64f)
cv2.imshow('Output2',sobel_8u)
minLineLength = 100
maxLineGap = 10
lines = cv2.HoughLinesP(sobel_8u, 1, np.pi / 180, 100, minLineLength, maxLineGap)
for x1, y1, x2, y2 in lines[0]:
cv2.line(words, (x1, y1), (x2, y2), (255, 255, 255), 2)
# cv2.imshow('hough',words)
height_orig, width_orig = img.shape[:2]
all_angles = []
im2, contours, hierarchy = cv2.findContours(words, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
logging.debug(len(contours))
contour_count = 0
for c in contours:
# max_index = np.argmax(areas)
# current_contour = np.zeros(img.shape[:2],np.uint8)
current_contour = np.zeros(img.shape[:2], np.uint8)
cv2.drawContours(current_contour, contours, contour_count, (255, 255, 255), -1)
height, width = current_contour.shape[:2]
# all_white_pixels = []
current_white_pixels = []
for i in range(0, height):
for j in range(0, width):
if (current_contour.item(i, j) == 255):
current_white_pixels.append([i, j])
matrix = np.array(current_white_pixels)
"""Finding covariance matrix"""
C = np.cov(matrix.T)
eigenvalues, eigenvectors = np.linalg.eig(C)
"""Finding max eigenvalue"""
# max_ev = max(eigenvalues)
"""Finding index of max eigenvalue"""
max_index = eigenvalues.argmax(axis=0)
"""The largest eigen value gives the approximate length of the bounding
ellipse around the largest word. If we follow the index of the largest
eigen value and find the eigen vectors in the column of that index,
we'll get the x and y coordinates of it's centre."""
y = eigenvectors[1, max_index]
x = eigenvectors[0, max_index]
angle = (np.arctan2(y, x)) * (180 / np.pi)
all_angles.append(angle)
contour_count += 1
logging.debug(contour_count)
logging.debug(all_angles)
angle = np.mean(all_angles)
logging.debug(angle)
k = 0
non_zero_angles = []
for i in all_angles:
if ((i != 0) and (i != 90.0)):
non_zero_angles.append(i)
logging.debug(non_zero_angles)
rounded_angles = []
for i in non_zero_angles:
rounded_angles.append(np.round(i, 0))
logging.debug(rounded_angles)
logging.debug("mode is")
# logging.debug(np.mode(rounded_angles))
# angle = np.mean(non_zero_angles)
# angle = np.mode(rounded_angles)
mode_angle = mode(rounded_angles)[0][0]
logging.debug(mode_angle)
precision_angles = []
for i in non_zero_angles:
if (np.round(i, 0) == mode_angle):
precision_angles.append(i)
logging.debug('precision angles:')
logging.debug(precision_angles)
angle = np.mean(precision_angles)
logging.debug('Finally, the required angle is:')
logging.debug(angle)
# M = cv2.getRotationMatrix2D((width/2,height/2),-(90+angle),1)
M = cv2.getRotationMatrix2D((width / 2, height / 2), -(90 + angle), 1)
dst = cv2.warpAffine(img, M, (width_orig, height_orig))
# cv2.imshow('final',dst)
cv2.imwrite('images/skewcorrected2.jpg', dst)
return dst
def preprocess(img):
return skew_correction(img)
# Does not work with linux:
# cv2.destroyAllWindows()
#detecting characters on image creating key points on characters.
#Detecting characters on image using keypoints
#detecting keypoints caharacter characters on the image
#this process draws keypoints on all characters available on the image
#the image to be processed is passsed in here, such that cv2.imread = 'image_resize.png'
img = cv2.imread('image_resize.png') #pass the image
gray= cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
orb = cv2.ORB_create(edgeThreshold=5,nfeatures=10000, scoreType=cv2.ORB_HARRIS_SCORE,scaleFactor=1.2)
kp ,des= orb.detectAndCompute(gray,None)
img=cv2.drawKeypoints(gray,kp,None)
cv2.imwrite('processed/images/keypoints.jpg',img)
cv2.imshow('threshold image', img)
# Maintain output window until
# user presses a key
cv2.waitKey(0)
# Destroying present windows on screen
cv2.destroyAllWindows()
# import libraries
import csv
import cv2
import pytesseract
def pre_processing(image):
"""
This function take one argument as
input. this function will convert
input image to binary image
:param image: image
:return: thresholded image
"""
gray_image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
# converting it to binary image
threshold_img = cv2.threshold(gray_image, 0, 255, cv2.THRESH_BINARY | cv2.THRESH_OTSU)[1]
# saving image to view threshold image
cv2.imwrite('processed/images/thresholded.png', threshold_img)
cv2.imshow('threshold image', threshold_img)
# Maintain output window until
# user presses a key
cv2.waitKey(0)
# Destroying present windows on screen
cv2.destroyAllWindows()
return threshold_img
def parse_text(threshold_img):
"""
This function take one argument as
input. this function will feed input
image to tesseract to predict text.
:param threshold_img: image
return: meta-data dictionary
"""
# configuring parameters for tesseract
tesseract_config = r' --oem 3 -l eng+chi_sim+chi_tra+spa+por+grc+deu+ell+fas+fil+heb+hin+ita+jpn+kor+lat+nep+osd+pol+rus+spa+swa+tel+tha+yor --psm 6'
# now feeding image to tesseract
details = pytesseract.image_to_data(threshold_img, output_type=pytesseract.Output.DICT,
config=tesseract_config, lang='eng')
return details
def draw_boxes(image, details, threshold_point):
"""
This function takes three argument as
input. it draw boxes on text area detected
by Tesseract. it also writes resulted image to
your local disk so that you can view it.
:param image: image
:param details: dictionary
:param threshold_point: integer
:return: None
"""
total_boxes = len(details['text'])
for sequence_number in range(total_boxes):
#if int(details['conf'][sequence_number]) > threshold_point:
(x, y, w, h) = (details['left'][sequence_number], details['top'][sequence_number],
details['width'][sequence_number], details['height'][sequence_number])
image = cv2.rectangle(image, (x, y), (x + w, y + h), (0, 255, 0), 2)
# saving image to local
cv2.imwrite('processed/images/captured_text_area.png', image)
# display image
cv2.imshow('captured text', image)
# Maintain output window until user presses a key
cv2.waitKey(0)
# Destroying present windows on screen
cv2.destroyAllWindows()
def format_text(details):
"""
This function take one argument as
input.This function will arrange
resulted text into proper format.
:param details: dictionary
:return: list
"""
parse_text = []
word_list = []
last_word = ''
for word in details['text']:
if word != '':
word_list.append(word)
last_word = word
if (last_word != '' and word == '') or (word == details['text'][-1]):
parse_text.append(word_list)
word_list = []
return parse_text
def write_text(formatted_text):
"""
This function take one argument.
it will write arranged text into
a file.
:param formatted_text: list
:return: None
"""
with open('processed/text_detected/text_detected.txt', 'w', newline="") as file:
csv.writer(file, delimiter=" ").writerows(formatted_text)
if __name__ == "__main__":
# reading image from local
image = cv2.imread('image_resize.png')
# calling pre_processing function to perform pre-processing on input image.
thresholds_image = pre_processing(image)
# calling parse_text function to get text from image by Tesseract.
parsed_data = parse_text(thresholds_image)
# defining threshold for draw box
accuracy_threshold = 30
# calling draw_boxes function which will draw dox around text area.
draw_boxes(thresholds_image, parsed_data, accuracy_threshold)
# calling format_text function which will format text according to input image
arranged_text = format_text(parsed_data)
# calling write_text function which will write arranged text into file
write_text(arranged_text)