vlff李飞飞 commited on
Commit
6bca16b
1 Parent(s): f973cc6
browser_qwen/background.js CHANGED
@@ -29,9 +29,9 @@ chrome.runtime.onMessage.addListener(async (msg, sender) => {
29
  url = tabs[0].url;
30
  console.log(url);
31
  if (msg.data) {
32
- chrome.storage.sync.get(['data','access_token'], function(result) {
33
  chrome.storage.sync.set({ data: result.data }, function() {
34
- send_data({ 'content' : msg.data, 'query': '', 'url': result.access_token + "_" + url, 'task':'cache', 'type':msg.type});
35
  });
36
  });
37
  }
@@ -41,7 +41,7 @@ chrome.runtime.onMessage.addListener(async (msg, sender) => {
41
  if (msg.data) {
42
  chrome.storage.sync.get(['data','access_token'], function(result) {
43
  chrome.storage.sync.set({ data: result.data }, function() {
44
- send_data({ 'url' : result.access_token + "_" + msg.data, 'task':'pop_url'});
45
  });
46
  });
47
  }
 
29
  url = tabs[0].url;
30
  console.log(url);
31
  if (msg.data) {
32
+ chrome.storage.sync.get(['data'], function(result) {
33
  chrome.storage.sync.set({ data: result.data }, function() {
34
+ send_data({ 'content' : msg.data, 'query': '', 'url': url, 'task':'cache', 'type':msg.type});
35
  });
36
  });
37
  }
 
41
  if (msg.data) {
42
  chrome.storage.sync.get(['data','access_token'], function(result) {
43
  chrome.storage.sync.set({ data: result.data }, function() {
44
+ send_data({ 'url' : msg.data, 'task':'pop_url'});
45
  });
46
  });
47
  }
qwen_server/assistant_server.py CHANGED
@@ -58,7 +58,7 @@ def set_page_url(access_token):
58
  lines = {access_token: []}
59
  assert os.path.exists(cache_file_popup_url)
60
  for line in jsonlines.open(cache_file_popup_url):
61
- _access_token = cache_file_popup_url.split("_")[0]
62
  if _access_token not in lines:
63
  lines[_access_token] = []
64
  lines[_access_token].append(line)
@@ -85,7 +85,7 @@ def bot(history, access_token):
85
  gr.Info("Please add this page to LLMBB's Reading List first!")
86
  else:
87
  for line in jsonlines.open(cache_file):
88
- if line['url'] == PAGE_URL[access_token][-1]:
89
  now_page = line
90
 
91
  if not now_page:
@@ -119,7 +119,7 @@ def bot(history, access_token):
119
  now_page['session'] = history
120
  lines = []
121
  for line in jsonlines.open(cache_file):
122
- if line['url'] != PAGE_URL[access_token][-1]:
123
  lines.append(line)
124
 
125
  lines.append(now_page)
@@ -134,7 +134,7 @@ def load_history_session(history, access_token):
134
  gr.Info("Please add this page to LLMBB's Reading List first!")
135
  return []
136
  for line in jsonlines.open(cache_file):
137
- if line['url'] == PAGE_URL[access_token][-1]:
138
  now_page = line
139
  if not now_page:
140
  gr.Info("Please add this page to LLMBB's Reading List first!")
@@ -151,7 +151,7 @@ def clear_session(access_token):
151
  now_page = None
152
  lines = []
153
  for line in jsonlines.open(cache_file):
154
- if line['url'] == PAGE_URL[access_token][-1]:
155
  now_page = line
156
  else:
157
  lines.append(line)
 
58
  lines = {access_token: []}
59
  assert os.path.exists(cache_file_popup_url)
60
  for line in jsonlines.open(cache_file_popup_url):
61
+ _access_token = line['access_token']
62
  if _access_token not in lines:
63
  lines[_access_token] = []
64
  lines[_access_token].append(line)
 
85
  gr.Info("Please add this page to LLMBB's Reading List first!")
86
  else:
87
  for line in jsonlines.open(cache_file):
88
+ if line["access_token"] == access_token and line['url'] == PAGE_URL[access_token][-1]:
89
  now_page = line
90
 
91
  if not now_page:
 
119
  now_page['session'] = history
120
  lines = []
121
  for line in jsonlines.open(cache_file):
122
+ if line["access_token"] == access_token and line['url'] != PAGE_URL[access_token][-1]:
123
  lines.append(line)
124
 
125
  lines.append(now_page)
 
134
  gr.Info("Please add this page to LLMBB's Reading List first!")
135
  return []
136
  for line in jsonlines.open(cache_file):
137
+ if line["access_token"] == access_token and line['url'] == PAGE_URL[access_token][-1]:
138
  now_page = line
139
  if not now_page:
140
  gr.Info("Please add this page to LLMBB's Reading List first!")
 
151
  now_page = None
152
  lines = []
153
  for line in jsonlines.open(cache_file):
154
+ if line["access_token"] == access_token and line['url'] == PAGE_URL[access_token][-1]:
155
  now_page = line
156
  else:
157
  lines.append(line)
qwen_server/database_server.py CHANGED
@@ -46,9 +46,8 @@ app.mount('/static',
46
  name='static')
47
 
48
 
49
- def update_pop_url(data, cache_file_popup_url):
50
- _access_token = data['url'].split("_")[0]
51
- new_line = {'url': data['url']}
52
  lines = []
53
  for line in jsonlines.open(cache_file_popup_url):
54
  lines.append(line)
@@ -60,12 +59,12 @@ def update_pop_url(data, cache_file_popup_url):
60
  return response
61
 
62
 
63
- def change_checkbox_state(text, cache_file):
64
  if not os.path.exists(cache_file):
65
  return {'result': 'no file'}
66
  lines = []
67
  for line in jsonlines.open(cache_file):
68
- if line['url'] == text[3:]:
69
  if line['checked']:
70
  line['checked'] = False
71
  else:
@@ -107,16 +106,16 @@ async def cache_data(file_name: str):
107
  async def web_listening(request: Request):
108
  data = await request.json()
109
  msg_type = data['task']
110
-
111
  cache_file_popup_url = os.path.join(server_config.path.cache_root, 'popup_url.jsonl')
112
  cache_file = os.path.join(server_config.path.cache_root, 'browse.jsonl')
113
 
114
  if msg_type == 'change_checkbox':
115
- rsp = change_checkbox_state(data['ckid'], cache_file)
116
  elif msg_type == 'cache':
117
  cache_obj = multiprocessing.Process(
118
  target=extract_and_cache_document,
119
- args=(data, cache_file, server_config.path.cache_root))
120
  cache_obj.start()
121
  # rsp = cache_data(data, cache_file)
122
  rsp = 'caching'
 
46
  name='static')
47
 
48
 
49
+ def update_pop_url(data, cache_file_popup_url, access_token):
50
+ new_line = {'url': data['url'], "access_token": access_token}
 
51
  lines = []
52
  for line in jsonlines.open(cache_file_popup_url):
53
  lines.append(line)
 
59
  return response
60
 
61
 
62
+ def change_checkbox_state(text, cache_file,access_token):
63
  if not os.path.exists(cache_file):
64
  return {'result': 'no file'}
65
  lines = []
66
  for line in jsonlines.open(cache_file):
67
+ if line['url'] == text[3:] and line['access_token'] == access_token:
68
  if line['checked']:
69
  line['checked'] = False
70
  else:
 
106
  async def web_listening(request: Request):
107
  data = await request.json()
108
  msg_type = data['task']
109
+ access_token = request.session.get("access_token")
110
  cache_file_popup_url = os.path.join(server_config.path.cache_root, 'popup_url.jsonl')
111
  cache_file = os.path.join(server_config.path.cache_root, 'browse.jsonl')
112
 
113
  if msg_type == 'change_checkbox':
114
+ rsp = change_checkbox_state(data['ckid'], cache_file, access_token)
115
  elif msg_type == 'cache':
116
  cache_obj = multiprocessing.Process(
117
  target=extract_and_cache_document,
118
+ args=(data, cache_file, server_config.path.cache_root,access_token))
119
  cache_obj.start()
120
  # rsp = cache_data(data, cache_file)
121
  rsp = 'caching'
qwen_server/utils.py CHANGED
@@ -44,7 +44,7 @@ def sanitize_chrome_file_path(file_path: str) -> str:
44
  return file_path
45
 
46
 
47
- def extract_and_cache_document(data, cache_file, cache_root):
48
  logger.info('Starting cache pages...')
49
  if data['url'].split('.')[-1].lower() in ['pdf', 'docx', 'pptx']:
50
  date1 = datetime.datetime.now()
@@ -55,6 +55,7 @@ def extract_and_cache_document(data, cache_file, cache_root):
55
  type=data['type'],
56
  raw=[],
57
  extract='',
 
58
  topic='',
59
  checked=False,
60
  session=[]).to_dict()
@@ -95,6 +96,7 @@ def extract_and_cache_document(data, cache_file, cache_root):
95
  type=data['type'],
96
  raw=[],
97
  extract='',
 
98
  topic='',
99
  checked=False,
100
  session=[]).to_dict()
@@ -120,6 +122,7 @@ def extract_and_cache_document(data, cache_file, cache_root):
120
  type=data['type'],
121
  raw=data['content'],
122
  extract=extract,
 
123
  topic='',
124
  checked=True,
125
  session=[])
 
44
  return file_path
45
 
46
 
47
+ def extract_and_cache_document(data, cache_file, cache_root, access_token):
48
  logger.info('Starting cache pages...')
49
  if data['url'].split('.')[-1].lower() in ['pdf', 'docx', 'pptx']:
50
  date1 = datetime.datetime.now()
 
55
  type=data['type'],
56
  raw=[],
57
  extract='',
58
+ access_token=access_token,
59
  topic='',
60
  checked=False,
61
  session=[]).to_dict()
 
96
  type=data['type'],
97
  raw=[],
98
  extract='',
99
+ access_token=access_token,
100
  topic='',
101
  checked=False,
102
  session=[]).to_dict()
 
122
  type=data['type'],
123
  raw=data['content'],
124
  extract=extract,
125
+ access_token=access_token,
126
  topic='',
127
  checked=True,
128
  session=[])