woo3 commited on
Commit
3df763b
·
1 Parent(s): c92a2cf

Delete cas_match.py

Browse files
Files changed (1) hide show
  1. cas_match.py +0 -266
cas_match.py DELETED
@@ -1,266 +0,0 @@
1
- # -*- coding: utf-8 -*-
2
- """
3
- Created on Sun Jun 4 15:55:12 2023
4
-
5
- @author: wooji
6
- """
7
- import re
8
- import streamlit as st
9
- import numpy as np
10
- import pandas as pd
11
- import streamlit as st
12
- import pandas as pd
13
- from io import StringIO
14
- import pdfplumber
15
- import xlrd
16
- import re
17
- import pdfplumber
18
- import pandas as pd
19
- import time
20
- import os
21
- import numpy as np
22
- #import pythoncom
23
- #import win32com
24
- #from win32com.client import Dispatch
25
- import docx2pdf
26
- import docx
27
- #import win32com.client as wc
28
- #import win32com.client as win32
29
- import pytesseract
30
- from PIL import Image
31
- import os
32
- from pdf2image import convert_from_path,convert_from_bytes
33
- from io import BytesIO
34
-
35
- import openpyxl
36
- import base64
37
- st.title("MSDS报告CAS号提取程序")
38
- #%%
39
- from docx import Document
40
-
41
- def get_tables(docx_path):
42
- docStr = Document(docx_path)
43
- numTables = docStr.tables
44
- my_list = []
45
- for table in numTables:
46
- row_count = len(table.rows)
47
- col_count = len(table.columns)
48
- for i in range(row_count):
49
- row = table.rows[i].cells
50
- for j in range(col_count):
51
- content = row[j].text
52
- my_list.append(content)
53
- my_list = ';'.join(my_list).strip('')
54
- return my_list
55
-
56
-
57
- def get_paragraphs(docx_path):
58
- #打开word文档
59
- document = Document(docx_path)
60
- #获取所有段落
61
- all_paragraphs = document.paragraphs
62
- paragraph_texts = []
63
- # 循环读取列表
64
- for paragraph in all_paragraphs:
65
- paragraph_texts.append(paragraph.text)
66
- paragraph_texts = ';'.join(paragraph_texts).strip('')
67
- return paragraph_texts
68
-
69
-
70
-
71
- #%% 函数二、打开pdf文件,输出每一页pdf中的所有文字
72
- def openpdf(path):
73
- with pdfplumber.open(path) as pdf:
74
- # pdf = pdfplumber.open(path)
75
- item = []
76
- for page in pdf.pages:
77
- text = page.extract_text()
78
- item.append(text)
79
- # item = [''.join(i) for i in item]
80
- item = ';'.join(item).strip('')
81
- return item
82
-
83
- #%% 函数三、将目标CAS号,和pdf中的内容进行比对。返回什么?
84
- def extract(text,cas):
85
- pattern = re.compile(cas,re.S)
86
- r_list = pattern.findall(text)
87
- return r_list
88
- #%%
89
- # data = pd.DataFrame(columns=['CAS','名称','匹配结果','备注'])
90
- st.write('使用说明')
91
- st.caption('支持解析的格式:.pdf(扫描版或非扫描版均支持)和.docx。可将MSDS文件夹直接拖拽到下方上传区域')
92
- st.write('excel输出内容详解')
93
- st.caption('第一列为文件名称,所有上传的文件均会显示在第一列,即便该文件格式不支持提取')
94
- st.caption('第二列为文件中提取的CAS号,若为空则表明未提取到')
95
- st.caption('第三列为化学物质名称,仅支持显示与清单匹配成功的化学物质的名称')
96
- st.caption('第四列为匹配结果,共3种结果:3960种、优评优控、重点管控')
97
- st.caption('第五列为备注,共3种结果:1、不支持该格式文件,请手动查看:说明此类文件不支持解析,请手动查看;2、图片pdf,建议人工复核:说明该pdf为图片,提取正确率较低,视情况可进行人工复核;3、未检测到CAS,请手动检查:说明在该文件中未检测到CAS,请人工确认')
98
-
99
- st.caption('提取速度:提取一个电子pdf大约耗时4s,一个扫描版pdf大约耗时10~20s。具体速度由pdf的页数决定')
100
- st.divider()
101
- uploaded_file = st.file_uploader("请上传MSDS报告,可直接往里拖拽文件夹",accept_multiple_files=True)
102
- @st.cache_data
103
- def main(uploaded_file):
104
- data = pd.DataFrame(columns=['CAS','名称','匹配结果','备注'])
105
-
106
- begin = time.time()
107
- # openpdf(uploaded_file)
108
- cas = r'[0-9]+-[0-9][0-9]-[0-9][^0-9]'
109
- # st.write(extract(openpdf(uploaded_file),cas))
110
- for file in range(len(uploaded_file)):
111
- if uploaded_file[file].name[-4:] == 'docx':
112
- text = get_paragraphs(uploaded_file[file])
113
- # text(get_tables(uploaded_file[file]))
114
- # text = ';'.join(text).strip('')
115
- elif uploaded_file[file].name[-3:] == 'pdf' or uploaded_file[file].name[-3:] == 'PDF':
116
- text = openpdf(uploaded_file[file])
117
- else:
118
- cas_set = pd.DataFrame({'备注':{uploaded_file[file].name:'不支持该格式文件,请手动查看'}})
119
- data = pd.concat([data,cas_set],axis=0)
120
- continue
121
- cas_extract = extract(text,cas)
122
- if cas_extract != []:
123
- for item in range(len(cas_extract)):
124
- cas_iso = cas_extract[item]
125
- cas_iso = cas_iso[0:len(cas_iso)-1]
126
- cas_set = pd.DataFrame({'CAS':{uploaded_file[file].name:cas_iso}})
127
- data = pd.concat([data,cas_set],axis=0)
128
- #提取docx表格内的内容
129
- elif uploaded_file[file].name[-4:] == 'docx':
130
- text = get_tables(uploaded_file[file])
131
- # text = ';'.join(text).strip('')
132
- cas_extract = extract(text,cas)
133
- if cas_extract != []:
134
- for item in range(len(cas_extract)):
135
- cas_iso = cas_extract[item]
136
- cas_iso = cas_iso[0:len(cas_iso)-1]
137
- cas_set = pd.DataFrame({'CAS':{uploaded_file[file].name:cas_iso}})
138
- data = pd.concat([data,cas_set],axis=0)
139
- else:
140
- cas_set = pd.DataFrame({'备注':{uploaded_file[file].name:'未检测到CAS,请手动检查'}})
141
- data = pd.concat([data,cas_set],axis=0)
142
- else:
143
- pages = convert_from_bytes(uploaded_file[file].getvalue()) # 上传的内容是什么?
144
- text = []
145
- for i,page in enumerate(pages):
146
- buf = BytesIO()
147
- page.save(buf,format="JPEG")
148
- buf.seek(0)
149
- img_page=Image.open(buf)
150
- # st.write('here')
151
- txt=pytesseract.image_to_string(img_page,lang='chi_sim')
152
- text.append(txt)
153
- text = ';'.join(text).strip('')
154
- cas_extract = extract(text,cas)
155
- if cas_extract != []:
156
- cas_extract = extract(text,cas)
157
- for item in range(len(cas_extract)):
158
- cas_iso = cas_extract[item]
159
- cas_iso = cas_iso[0:len(cas_iso)-1]
160
- print(cas_iso)
161
- # cas_set = pd.Series({uploaded_file[file].name:cas_iso+'图片pdf,请手动检查'}) #在这里加备注提示是扫描版pdf
162
- #用dataframe承载
163
- cas_set = pd.DataFrame({'CAS':{uploaded_file[file].name:cas_iso},'备注':{uploaded_file[file].name:'图片pdf,建议人工复核'}})
164
- data = pd.concat([data,cas_set],axis=0)
165
- else:
166
- cas_set = pd.DataFrame({'备注':{uploaded_file[file].name:'未检测到CAS,请手动检查'}})
167
- data = pd.concat([data,cas_set],axis=0)
168
-
169
- # st.write(uploaded_file)
170
- # convert_from_bytes(open('/home/belval/example.pdf','rb').read())
171
-
172
-
173
-
174
- #%%数据整理
175
- data_reset_index = data.reset_index(drop=False)
176
- #修改列名
177
- data_rename = data_reset_index.rename(columns={'index':'MSDS文件名称'})
178
- #去除重复行
179
- data_output = data_rename.drop_duplicates() #subset='pdf名称'可以查看是不是所有文件都包含在表格里
180
- # target_data_base = pd.read_excel('C:/Users/wooji/Nutstore/1/Jiho华南所/鉴定中心-工作/MSDS/102-104物质清单.xlsx',sheet_name='基102-3960种',index_col=0)
181
- # # target_data_pri = pd.read_excel('C:/Users/wooji/Nutstore/1/Jiho华南所/鉴定中心-工作/MSDS/物质清单.xlsx',sheet_name='优评优控',index_col=0)
182
- # # target_data_key = pd.read_excel('C:/Users/wooji/Nutstore/1/Jiho华南所/鉴定中心-工作/MSDS/物质清单.xlsx',sheet_name='重点管控',index_col=0)
183
- # target_cas_base = target_data_base[['CAS','名称']]
184
- # target_cas_pri = target_data_pri[['CAS','名称']]
185
- # target_cas_key = target_data_key[['CAS','名称']]
186
- # target_cas_base = target_cas_base.reset_index(drop=True)
187
- # target_cas_pri = target_cas_pri.reset_index(drop=True)
188
- # target_cas_key = target_cas_key.reset_index(drop=True)
189
- target_data = pd.read_excel('C:/Users/wooji/Nutstore/1/Jiho华南所/鉴定中心-工作/MSDS/物质清单.xlsx',sheet_name='总表',index_col=0)
190
- target_cas = target_data[['CAS','名称','清单']]
191
- target_cas = target_cas.reset_index(drop=True)
192
-
193
-
194
- #%%
195
- for row in data_output.index:
196
- # print(data_output.loc[row]['CAS号提取'])
197
- for b in target_cas.index:
198
- if data_output.loc[row]['CAS'] == target_cas.loc[b]['CAS']:
199
- data_output.loc[row]['匹配结果'] =target_cas.loc[b]['清单']
200
- data_output.loc[row]['名称'] = target_cas.loc[b]['名称']
201
-
202
-
203
- data_final = data_output
204
- # [['pdf名称','匹配结果','CAS号提取','名称','备注']]
205
- end = time.time()
206
- run_time = end - begin
207
- st.write('运行耗时:'+ str(round(run_time,2))+'秒')
208
- return data_final
209
-
210
-
211
- if uploaded_file == []:
212
- st.stop()
213
- else:
214
- data_final = main(uploaded_file)
215
- data_final
216
- data_final.to_excel('resuls.xlsx')
217
- wb2 = openpyxl.load_workbook('resuls.xlsx')
218
- wb2.save('results.xlsx')#注意!文件此时保存在内存中且为字节格式文件
219
- data=open('results.xlsx','rb').read()#以只读模式读取且读取为二进制文件
220
- b64 = base64.b64encode(data).decode('UTF-8')#解码并加密为base64
221
- excel_name = st.text_input(':blue[请输入本次导入的文件所属企业名称,若为空则导出的excel默认取名为myresult.xlsx]')
222
- st.warning('建议示例:广西xx企业-原辅料 or 广西xx企业-产品 ------- 输入完请按回车 ', icon="🚨")
223
- if excel_name:
224
- excel_name = excel_name + '.xlsx'
225
- href = f'<a href="data:file/data;base64,{b64}" download={excel_name}>导出excel</a>'#定义下载链接,���认的下载文件名是myresults.xlsx
226
- st.markdown(href, unsafe_allow_html=True)#输出到浏览器
227
- wb2.close()
228
- else:
229
- href = f'<a href="data:file/data;base64,{b64}" download=myresult.xlsx>导出excel</a>'#定义下载链接,默认的下载文件名是myresults.xlsx
230
- st.markdown(href, unsafe_allow_html=True)#输出到浏览器
231
- wb2.close()
232
-
233
-
234
- st.subheader('!!!单次使用完请刷新页面后再上传新的文件')
235
-
236
-
237
- # else:
238
- # excel_name = excel_name + '.xlsx'
239
- # href = f'<a href="data:file/data;base64,{b64}" download={excel_name}>Download xlsx file</a>'#定义下载链接,默认的下载文件名是myresults.xlsx
240
- # st.markdown(href, unsafe_allow_html=True)#输出到浏览器
241
- # wb2.close()
242
-
243
-
244
-
245
-
246
- ####直接写识别图片的代码
247
- # stringio = StringIO(uploaded_file[file].getvalue().decode("utf-8"))
248
- # st.write(stringio) ##这句是对的
249
- # bytes_data = uploaded_file[file].read()
250
- # st.write(bytes_data)
251
- # st.write(uploaded_file[file])
252
- # st.write(bytes_data)
253
- # =============================================================================
254
- # ####
255
- # stringio = StringIO(uploaded_file[file].getvalue().decode("utf-8"))
256
- # st.write(stringio)
257
- # # To read file as string:
258
- # string_data = stringio.read()
259
- # st.write(string_data)
260
- # ###
261
- # =============================================================================
262
-
263
-
264
-
265
-
266
-