Delete cas_match.py
Browse files- cas_match.py +0 -266
cas_match.py
DELETED
@@ -1,266 +0,0 @@
|
|
1 |
-
# -*- coding: utf-8 -*-
|
2 |
-
"""
|
3 |
-
Created on Sun Jun 4 15:55:12 2023
|
4 |
-
|
5 |
-
@author: wooji
|
6 |
-
"""
|
7 |
-
import re
|
8 |
-
import streamlit as st
|
9 |
-
import numpy as np
|
10 |
-
import pandas as pd
|
11 |
-
import streamlit as st
|
12 |
-
import pandas as pd
|
13 |
-
from io import StringIO
|
14 |
-
import pdfplumber
|
15 |
-
import xlrd
|
16 |
-
import re
|
17 |
-
import pdfplumber
|
18 |
-
import pandas as pd
|
19 |
-
import time
|
20 |
-
import os
|
21 |
-
import numpy as np
|
22 |
-
#import pythoncom
|
23 |
-
#import win32com
|
24 |
-
#from win32com.client import Dispatch
|
25 |
-
import docx2pdf
|
26 |
-
import docx
|
27 |
-
#import win32com.client as wc
|
28 |
-
#import win32com.client as win32
|
29 |
-
import pytesseract
|
30 |
-
from PIL import Image
|
31 |
-
import os
|
32 |
-
from pdf2image import convert_from_path,convert_from_bytes
|
33 |
-
from io import BytesIO
|
34 |
-
|
35 |
-
import openpyxl
|
36 |
-
import base64
|
37 |
-
st.title("MSDS报告CAS号提取程序")
|
38 |
-
#%%
|
39 |
-
from docx import Document
|
40 |
-
|
41 |
-
def get_tables(docx_path):
|
42 |
-
docStr = Document(docx_path)
|
43 |
-
numTables = docStr.tables
|
44 |
-
my_list = []
|
45 |
-
for table in numTables:
|
46 |
-
row_count = len(table.rows)
|
47 |
-
col_count = len(table.columns)
|
48 |
-
for i in range(row_count):
|
49 |
-
row = table.rows[i].cells
|
50 |
-
for j in range(col_count):
|
51 |
-
content = row[j].text
|
52 |
-
my_list.append(content)
|
53 |
-
my_list = ';'.join(my_list).strip('')
|
54 |
-
return my_list
|
55 |
-
|
56 |
-
|
57 |
-
def get_paragraphs(docx_path):
|
58 |
-
#打开word文档
|
59 |
-
document = Document(docx_path)
|
60 |
-
#获取所有段落
|
61 |
-
all_paragraphs = document.paragraphs
|
62 |
-
paragraph_texts = []
|
63 |
-
# 循环读取列表
|
64 |
-
for paragraph in all_paragraphs:
|
65 |
-
paragraph_texts.append(paragraph.text)
|
66 |
-
paragraph_texts = ';'.join(paragraph_texts).strip('')
|
67 |
-
return paragraph_texts
|
68 |
-
|
69 |
-
|
70 |
-
|
71 |
-
#%% 函数二、打开pdf文件,输出每一页pdf中的所有文字
|
72 |
-
def openpdf(path):
|
73 |
-
with pdfplumber.open(path) as pdf:
|
74 |
-
# pdf = pdfplumber.open(path)
|
75 |
-
item = []
|
76 |
-
for page in pdf.pages:
|
77 |
-
text = page.extract_text()
|
78 |
-
item.append(text)
|
79 |
-
# item = [''.join(i) for i in item]
|
80 |
-
item = ';'.join(item).strip('')
|
81 |
-
return item
|
82 |
-
|
83 |
-
#%% 函数三、将目标CAS号,和pdf中的内容进行比对。返回什么?
|
84 |
-
def extract(text,cas):
|
85 |
-
pattern = re.compile(cas,re.S)
|
86 |
-
r_list = pattern.findall(text)
|
87 |
-
return r_list
|
88 |
-
#%%
|
89 |
-
# data = pd.DataFrame(columns=['CAS','名称','匹配结果','备注'])
|
90 |
-
st.write('使用说明')
|
91 |
-
st.caption('支持解析的格式:.pdf(扫描版或非扫描版均支持)和.docx。可将MSDS文件夹直接拖拽到下方上传区域')
|
92 |
-
st.write('excel输出内容详解')
|
93 |
-
st.caption('第一列为文件名称,所有上传的文件均会显示在第一列,即便该文件格式不支持提取')
|
94 |
-
st.caption('第二列为文件中提取的CAS号,若为空则表明未提取到')
|
95 |
-
st.caption('第三列为化学物质名称,仅支持显示与清单匹配成功的化学物质的名称')
|
96 |
-
st.caption('第四列为匹配结果,共3种结果:3960种、优评优控、重点管控')
|
97 |
-
st.caption('第五列为备注,共3种结果:1、不支持该格式文件,请手动查看:说明此类文件不支持解析,请手动查看;2、图片pdf,建议人工复核:说明该pdf为图片,提取正确率较低,视情况可进行人工复核;3、未检测到CAS,请手动检查:说明在该文件中未检测到CAS,请人工确认')
|
98 |
-
|
99 |
-
st.caption('提取速度:提取一个电子pdf大约耗时4s,一个扫描版pdf大约耗时10~20s。具体速度由pdf的页数决定')
|
100 |
-
st.divider()
|
101 |
-
uploaded_file = st.file_uploader("请上传MSDS报告,可直接往里拖拽文件夹",accept_multiple_files=True)
|
102 |
-
@st.cache_data
|
103 |
-
def main(uploaded_file):
|
104 |
-
data = pd.DataFrame(columns=['CAS','名称','匹配结果','备注'])
|
105 |
-
|
106 |
-
begin = time.time()
|
107 |
-
# openpdf(uploaded_file)
|
108 |
-
cas = r'[0-9]+-[0-9][0-9]-[0-9][^0-9]'
|
109 |
-
# st.write(extract(openpdf(uploaded_file),cas))
|
110 |
-
for file in range(len(uploaded_file)):
|
111 |
-
if uploaded_file[file].name[-4:] == 'docx':
|
112 |
-
text = get_paragraphs(uploaded_file[file])
|
113 |
-
# text(get_tables(uploaded_file[file]))
|
114 |
-
# text = ';'.join(text).strip('')
|
115 |
-
elif uploaded_file[file].name[-3:] == 'pdf' or uploaded_file[file].name[-3:] == 'PDF':
|
116 |
-
text = openpdf(uploaded_file[file])
|
117 |
-
else:
|
118 |
-
cas_set = pd.DataFrame({'备注':{uploaded_file[file].name:'不支持该格式文件,请手动查看'}})
|
119 |
-
data = pd.concat([data,cas_set],axis=0)
|
120 |
-
continue
|
121 |
-
cas_extract = extract(text,cas)
|
122 |
-
if cas_extract != []:
|
123 |
-
for item in range(len(cas_extract)):
|
124 |
-
cas_iso = cas_extract[item]
|
125 |
-
cas_iso = cas_iso[0:len(cas_iso)-1]
|
126 |
-
cas_set = pd.DataFrame({'CAS':{uploaded_file[file].name:cas_iso}})
|
127 |
-
data = pd.concat([data,cas_set],axis=0)
|
128 |
-
#提取docx表格内的内容
|
129 |
-
elif uploaded_file[file].name[-4:] == 'docx':
|
130 |
-
text = get_tables(uploaded_file[file])
|
131 |
-
# text = ';'.join(text).strip('')
|
132 |
-
cas_extract = extract(text,cas)
|
133 |
-
if cas_extract != []:
|
134 |
-
for item in range(len(cas_extract)):
|
135 |
-
cas_iso = cas_extract[item]
|
136 |
-
cas_iso = cas_iso[0:len(cas_iso)-1]
|
137 |
-
cas_set = pd.DataFrame({'CAS':{uploaded_file[file].name:cas_iso}})
|
138 |
-
data = pd.concat([data,cas_set],axis=0)
|
139 |
-
else:
|
140 |
-
cas_set = pd.DataFrame({'备注':{uploaded_file[file].name:'未检测到CAS,请手动检查'}})
|
141 |
-
data = pd.concat([data,cas_set],axis=0)
|
142 |
-
else:
|
143 |
-
pages = convert_from_bytes(uploaded_file[file].getvalue()) # 上传的内容是什么?
|
144 |
-
text = []
|
145 |
-
for i,page in enumerate(pages):
|
146 |
-
buf = BytesIO()
|
147 |
-
page.save(buf,format="JPEG")
|
148 |
-
buf.seek(0)
|
149 |
-
img_page=Image.open(buf)
|
150 |
-
# st.write('here')
|
151 |
-
txt=pytesseract.image_to_string(img_page,lang='chi_sim')
|
152 |
-
text.append(txt)
|
153 |
-
text = ';'.join(text).strip('')
|
154 |
-
cas_extract = extract(text,cas)
|
155 |
-
if cas_extract != []:
|
156 |
-
cas_extract = extract(text,cas)
|
157 |
-
for item in range(len(cas_extract)):
|
158 |
-
cas_iso = cas_extract[item]
|
159 |
-
cas_iso = cas_iso[0:len(cas_iso)-1]
|
160 |
-
print(cas_iso)
|
161 |
-
# cas_set = pd.Series({uploaded_file[file].name:cas_iso+'图片pdf,请手动检查'}) #在这里加备注提示是扫描版pdf
|
162 |
-
#用dataframe承载
|
163 |
-
cas_set = pd.DataFrame({'CAS':{uploaded_file[file].name:cas_iso},'备注':{uploaded_file[file].name:'图片pdf,建议人工复核'}})
|
164 |
-
data = pd.concat([data,cas_set],axis=0)
|
165 |
-
else:
|
166 |
-
cas_set = pd.DataFrame({'备注':{uploaded_file[file].name:'未检测到CAS,请手动检查'}})
|
167 |
-
data = pd.concat([data,cas_set],axis=0)
|
168 |
-
|
169 |
-
# st.write(uploaded_file)
|
170 |
-
# convert_from_bytes(open('/home/belval/example.pdf','rb').read())
|
171 |
-
|
172 |
-
|
173 |
-
|
174 |
-
#%%数据整理
|
175 |
-
data_reset_index = data.reset_index(drop=False)
|
176 |
-
#修改列名
|
177 |
-
data_rename = data_reset_index.rename(columns={'index':'MSDS文件名称'})
|
178 |
-
#去除重复行
|
179 |
-
data_output = data_rename.drop_duplicates() #subset='pdf名称'可以查看是不是所有文件都包含在表格里
|
180 |
-
# target_data_base = pd.read_excel('C:/Users/wooji/Nutstore/1/Jiho华南所/鉴定中心-工作/MSDS/102-104物质清单.xlsx',sheet_name='基102-3960种',index_col=0)
|
181 |
-
# # target_data_pri = pd.read_excel('C:/Users/wooji/Nutstore/1/Jiho华南所/鉴定中心-工作/MSDS/物质清单.xlsx',sheet_name='优评优控',index_col=0)
|
182 |
-
# # target_data_key = pd.read_excel('C:/Users/wooji/Nutstore/1/Jiho华南所/鉴定中心-工作/MSDS/物质清单.xlsx',sheet_name='重点管控',index_col=0)
|
183 |
-
# target_cas_base = target_data_base[['CAS','名称']]
|
184 |
-
# target_cas_pri = target_data_pri[['CAS','名称']]
|
185 |
-
# target_cas_key = target_data_key[['CAS','名称']]
|
186 |
-
# target_cas_base = target_cas_base.reset_index(drop=True)
|
187 |
-
# target_cas_pri = target_cas_pri.reset_index(drop=True)
|
188 |
-
# target_cas_key = target_cas_key.reset_index(drop=True)
|
189 |
-
target_data = pd.read_excel('C:/Users/wooji/Nutstore/1/Jiho华南所/鉴定中心-工作/MSDS/物质清单.xlsx',sheet_name='总表',index_col=0)
|
190 |
-
target_cas = target_data[['CAS','名称','清单']]
|
191 |
-
target_cas = target_cas.reset_index(drop=True)
|
192 |
-
|
193 |
-
|
194 |
-
#%%
|
195 |
-
for row in data_output.index:
|
196 |
-
# print(data_output.loc[row]['CAS号提取'])
|
197 |
-
for b in target_cas.index:
|
198 |
-
if data_output.loc[row]['CAS'] == target_cas.loc[b]['CAS']:
|
199 |
-
data_output.loc[row]['匹配结果'] =target_cas.loc[b]['清单']
|
200 |
-
data_output.loc[row]['名称'] = target_cas.loc[b]['名称']
|
201 |
-
|
202 |
-
|
203 |
-
data_final = data_output
|
204 |
-
# [['pdf名称','匹配结果','CAS号提取','名称','备注']]
|
205 |
-
end = time.time()
|
206 |
-
run_time = end - begin
|
207 |
-
st.write('运行耗时:'+ str(round(run_time,2))+'秒')
|
208 |
-
return data_final
|
209 |
-
|
210 |
-
|
211 |
-
if uploaded_file == []:
|
212 |
-
st.stop()
|
213 |
-
else:
|
214 |
-
data_final = main(uploaded_file)
|
215 |
-
data_final
|
216 |
-
data_final.to_excel('resuls.xlsx')
|
217 |
-
wb2 = openpyxl.load_workbook('resuls.xlsx')
|
218 |
-
wb2.save('results.xlsx')#注意!文件此时保存在内存中且为字节格式文件
|
219 |
-
data=open('results.xlsx','rb').read()#以只读模式读取且读取为二进制文件
|
220 |
-
b64 = base64.b64encode(data).decode('UTF-8')#解码并加密为base64
|
221 |
-
excel_name = st.text_input(':blue[请输入本次导入的文件所属企业名称,若为空则导出的excel默认取名为myresult.xlsx]')
|
222 |
-
st.warning('建议示例:广西xx企业-原辅料 or 广西xx企业-产品 ------- 输入完请按回车 ', icon="🚨")
|
223 |
-
if excel_name:
|
224 |
-
excel_name = excel_name + '.xlsx'
|
225 |
-
href = f'<a href="data:file/data;base64,{b64}" download={excel_name}>导出excel</a>'#定义下载链接,���认的下载文件名是myresults.xlsx
|
226 |
-
st.markdown(href, unsafe_allow_html=True)#输出到浏览器
|
227 |
-
wb2.close()
|
228 |
-
else:
|
229 |
-
href = f'<a href="data:file/data;base64,{b64}" download=myresult.xlsx>导出excel</a>'#定义下载链接,默认的下载文件名是myresults.xlsx
|
230 |
-
st.markdown(href, unsafe_allow_html=True)#输出到浏览器
|
231 |
-
wb2.close()
|
232 |
-
|
233 |
-
|
234 |
-
st.subheader('!!!单次使用完请刷新页面后再上传新的文件')
|
235 |
-
|
236 |
-
|
237 |
-
# else:
|
238 |
-
# excel_name = excel_name + '.xlsx'
|
239 |
-
# href = f'<a href="data:file/data;base64,{b64}" download={excel_name}>Download xlsx file</a>'#定义下载链接,默认的下载文件名是myresults.xlsx
|
240 |
-
# st.markdown(href, unsafe_allow_html=True)#输出到浏览器
|
241 |
-
# wb2.close()
|
242 |
-
|
243 |
-
|
244 |
-
|
245 |
-
|
246 |
-
####直接写识别图片的代码
|
247 |
-
# stringio = StringIO(uploaded_file[file].getvalue().decode("utf-8"))
|
248 |
-
# st.write(stringio) ##这句是对的
|
249 |
-
# bytes_data = uploaded_file[file].read()
|
250 |
-
# st.write(bytes_data)
|
251 |
-
# st.write(uploaded_file[file])
|
252 |
-
# st.write(bytes_data)
|
253 |
-
# =============================================================================
|
254 |
-
# ####
|
255 |
-
# stringio = StringIO(uploaded_file[file].getvalue().decode("utf-8"))
|
256 |
-
# st.write(stringio)
|
257 |
-
# # To read file as string:
|
258 |
-
# string_data = stringio.read()
|
259 |
-
# st.write(string_data)
|
260 |
-
# ###
|
261 |
-
# =============================================================================
|
262 |
-
|
263 |
-
|
264 |
-
|
265 |
-
|
266 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|