LukasHug commited on
Commit
7771cfc
β€’
1 Parent(s): 6262027

Upload 5 files

Browse files
Files changed (3) hide show
  1. README.md +6 -6
  2. app.py +20 -77
  3. builder.py +167 -0
README.md CHANGED
@@ -1,13 +1,13 @@
1
  ---
2
- title: LlavaGuard
3
- emoji: πŸ–Ό
4
- colorFrom: purple
5
- colorTo: red
6
  sdk: gradio
7
- sdk_version: 4.16.0
8
  app_file: app.py
9
  pinned: false
10
- license: mit
11
  ---
12
 
13
  Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
1
  ---
2
+ title: LLaVA 1.6
3
+ emoji: πŸ‘
4
+ colorFrom: green
5
+ colorTo: yellow
6
  sdk: gradio
7
+ sdk_version: 4.36.1
8
  app_file: app.py
9
  pinned: false
10
+ license: apache-2.0
11
  ---
12
 
13
  Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
app.py CHANGED
@@ -11,11 +11,11 @@ import gradio as gr
11
  import spaces
12
  import torch
13
 
 
14
  from llava.constants import IMAGE_TOKEN_INDEX
15
  from llava.constants import LOGDIR
16
  from llava.conversation import (default_conversation, conv_templates)
17
  from llava.mm_utils import KeywordsStoppingCriteria, tokenizer_image_token
18
- from llava.model.builder import load_pretrained_model
19
  from llava.utils import (build_logger, violates_moderation, moderation_msg)
20
  from taxonomy import wrap_taxonomy, default_taxonomy
21
 
@@ -67,24 +67,28 @@ def run_llava(prompt, pil_image):
67
  return outputs[0].strip()
68
 
69
 
 
 
 
 
 
 
 
 
 
 
 
 
70
  def get_conv_log_filename():
71
  t = datetime.datetime.now()
72
  name = os.path.join(LOGDIR, f"{t.year}-{t.month:02d}-{t.day:02d}-conv.json")
73
  return name
74
 
75
-
76
  def get_model_list():
77
- # ret = requests.post(args.controller_url + "/refresh_all_workers")
78
- # assert ret.status_code == 200
79
- # ret = requests.post(args.controller_url + "/list_models")
80
- # logger.info(f"get_model_list: {ret.json()}")
81
- # models = ret.json()["models"]
82
- # models.sort(key=lambda x: priority.get(x, x))
83
- # logger.info(f"Models: {models}")
84
  models = [
85
- 'LukasHug/LlavaGuard-7B-hf',
86
- 'LukasHug/LlavaGuard-13B-hf',
87
- 'LukasHug/LlavaGuard-34B-hf', ][:1]
88
  return models
89
 
90
 
@@ -245,18 +249,6 @@ def llava_bot(state, model_selector, temperature, top_p, max_new_tokens, request
245
  new_state.append_message(new_state.roles[1], None)
246
  state = new_state
247
 
248
- # Query worker address
249
- # controller_url = args.controller_url
250
- # ret = requests.post(controller_url + "/get_worker_address",
251
- # json={"model": model_name})
252
- # worker_addr = ret.json()["address"]
253
- # logger.info(f"model_name: {model_name}, worker_addr: {worker_addr}")
254
-
255
- # No available worker
256
- # if worker_addr == "":
257
- # state.messages[-1][-1] = server_error_msg
258
- # yield (state, state.to_gradio_chatbot(), disable_btn, disable_btn, disable_btn, enable_btn, enable_btn)
259
- # return
260
 
261
  # Construct prompt
262
  prompt = state.get_prompt()
@@ -274,51 +266,8 @@ def llava_bot(state, model_selector, temperature, top_p, max_new_tokens, request
274
 
275
  state.messages[-1][-1] = output
276
 
277
- # Make requests
278
- # pload = {
279
- # "model": model_name,
280
- # "prompt": prompt,
281
- # "temperature": float(temperature),
282
- # "top_p": float(top_p),
283
- # # "num_beams": 2,
284
- # # "top_k": 50,
285
- # "max_new_tokens": min(int(max_new_tokens), 1536),
286
- # "stop": state.sep if state.sep_style in [SeparatorStyle.SINGLE, SeparatorStyle.MPT] else state.sep2,
287
- # "images": f'List of {len(state.get_images())} images: {all_image_hash}',
288
- # }
289
- # logger.info(f"==== request ====\n{pload}")
290
- #
291
- # pload['images'] = state.get_images()
292
-
293
- # state.messages[-1][-1] = "β–Œ"
294
-
295
  yield (state, state.to_gradio_chatbot()) + (disable_btn,) * 5
296
 
297
- # try:
298
- # # Stream output
299
- # response = requests.post(worker_addr + "/worker_generate_stream",
300
- # headers=headers, json=pload, stream=True, timeout=10)
301
- # for chunk in response.iter_lines(decode_unicode=False, delimiter=b"\0"):
302
- # if chunk:
303
- # data = json.loads(chunk.decode())
304
- # if data["error_code"] == 0:
305
- # output = data["text"][len(prompt):].strip()
306
- # state.messages[-1][-1] = output + "β–Œ"
307
- # yield (state, state.to_gradio_chatbot()) + (disable_btn,) * 5
308
- # else:
309
- # output = data["text"] + f" (error_code: {data['error_code']})"
310
- # state.messages[-1][-1] = output
311
- # yield (state, state.to_gradio_chatbot()) + (
312
- # disable_btn, disable_btn, disable_btn, enable_btn, enable_btn)
313
- # return
314
- # time.sleep(0.03)
315
- # except requests.exceptions.RequestException as e:
316
- # state.messages[-1][-1] = server_error_msg
317
- # yield (state, state.to_gradio_chatbot()) + (disable_btn, disable_btn, disable_btn, enable_btn, enable_btn)
318
- # return
319
- #
320
- # state.messages[-1][-1] = state.messages[-1][-1][:-1]
321
- # yield (state, state.to_gradio_chatbot()) + (enable_btn,) * 5
322
 
323
  finish_tstamp = time.time()
324
  logger.info(f"{output}")
@@ -457,6 +406,8 @@ def build_demo(embed_mode, cur_dir=None, concurrency_count=10):
457
  [textbox, upvote_btn, downvote_btn, flag_btn]
458
  )
459
 
 
 
460
  regenerate_btn.click(
461
  regenerate,
462
  [state, image_process_mode],
@@ -540,10 +491,7 @@ Set the environment variable `model` to change the model:
540
  ['AIML-TUDA/LlavaGuard-13B'](https://huggingface.co/AIML-TUDA/LlavaGuard-13B),
541
  ['AIML-TUDA/LlavaGuard-34B'](https://huggingface.co/AIML-TUDA/LlavaGuard-34B),
542
  """
543
- # set_up_env_and_token(read=True)
544
  print(f"args: {args}")
545
- # set the huggingface login token
546
- # controller_proc = start_controller()
547
  concurrency_count = int(os.getenv("concurrency_count", 5))
548
  api_key = os.getenv("token")
549
  if api_key:
@@ -561,19 +509,14 @@ Set the environment variable `model` to change the model:
561
  'LukasHug/LlavaGuard-13B-hf',
562
  'LukasHug/LlavaGuard-34B-hf', ]
563
  bits = int(os.getenv("bits", 16))
564
- model = os.getenv("model", models[0])
565
  available_devices = os.getenv("CUDA_VISIBLE_DEVICES", "0")
566
- model_path, model_name = model, model.split("/")[1]
567
- # model_path = '/common-repos/LlavaGuard/models/LlavaGuard-v1.1-7b-full/smid_and_crawled_v2_with_augmented_policies/json-v12/llava'
568
-
569
 
570
  print(f"Loading model {model_path}")
571
  tokenizer, model, image_processor, context_len = load_pretrained_model(model_path, None, model_name, token=api_key)
572
-
573
  model.config.tokenizer_model_max_length = 2048 * 2
574
 
575
- # Wait for worker and controller to start
576
- # time.sleep(10)
577
 
578
  exit_status = 0
579
  try:
 
11
  import spaces
12
  import torch
13
 
14
+ from builder import load_pretrained_model
15
  from llava.constants import IMAGE_TOKEN_INDEX
16
  from llava.constants import LOGDIR
17
  from llava.conversation import (default_conversation, conv_templates)
18
  from llava.mm_utils import KeywordsStoppingCriteria, tokenizer_image_token
 
19
  from llava.utils import (build_logger, violates_moderation, moderation_msg)
20
  from taxonomy import wrap_taxonomy, default_taxonomy
21
 
 
67
  return outputs[0].strip()
68
 
69
 
70
+ def load_selected_model(model_path):
71
+ model_name = model_path.split("/")[-1]
72
+ global tokenizer, model, image_processor, context_len
73
+ with warnings.catch_warnings(record=True) as w:
74
+ warnings.simplefilter("always")
75
+ tokenizer, model, image_processor, context_len = load_pretrained_model(model_path, None, model_name)
76
+ for warning in w:
77
+ if "vision" not in str(warning.message).lower():
78
+ print(warning.message)
79
+ model.config.tokenizer_model_max_length = 2048 * 2
80
+
81
+
82
  def get_conv_log_filename():
83
  t = datetime.datetime.now()
84
  name = os.path.join(LOGDIR, f"{t.year}-{t.month:02d}-{t.day:02d}-conv.json")
85
  return name
86
 
 
87
  def get_model_list():
 
 
 
 
 
 
 
88
  models = [
89
+ 'LukasHug/LlavaGuard-7B-hf',
90
+ 'LukasHug/LlavaGuard-13B-hf',
91
+ 'LukasHug/LlavaGuard-34B-hf', ]
92
  return models
93
 
94
 
 
249
  new_state.append_message(new_state.roles[1], None)
250
  state = new_state
251
 
 
 
 
 
 
 
 
 
 
 
 
 
252
 
253
  # Construct prompt
254
  prompt = state.get_prompt()
 
266
 
267
  state.messages[-1][-1] = output
268
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
269
  yield (state, state.to_gradio_chatbot()) + (disable_btn,) * 5
270
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
271
 
272
  finish_tstamp = time.time()
273
  logger.info(f"{output}")
 
406
  [textbox, upvote_btn, downvote_btn, flag_btn]
407
  )
408
 
409
+ model_selector.change(load_selected_model)
410
+
411
  regenerate_btn.click(
412
  regenerate,
413
  [state, image_process_mode],
 
491
  ['AIML-TUDA/LlavaGuard-13B'](https://huggingface.co/AIML-TUDA/LlavaGuard-13B),
492
  ['AIML-TUDA/LlavaGuard-34B'](https://huggingface.co/AIML-TUDA/LlavaGuard-34B),
493
  """
 
494
  print(f"args: {args}")
 
 
495
  concurrency_count = int(os.getenv("concurrency_count", 5))
496
  api_key = os.getenv("token")
497
  if api_key:
 
509
  'LukasHug/LlavaGuard-13B-hf',
510
  'LukasHug/LlavaGuard-34B-hf', ]
511
  bits = int(os.getenv("bits", 16))
512
+ model = os.getenv("model", models[1])
513
  available_devices = os.getenv("CUDA_VISIBLE_DEVICES", "0")
514
+ model_path, model_name = model, model.split("/")[0]
 
 
515
 
516
  print(f"Loading model {model_path}")
517
  tokenizer, model, image_processor, context_len = load_pretrained_model(model_path, None, model_name, token=api_key)
 
518
  model.config.tokenizer_model_max_length = 2048 * 2
519
 
 
 
520
 
521
  exit_status = 0
522
  try:
builder.py ADDED
@@ -0,0 +1,167 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2023 Haotian Liu
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+
16
+ import os
17
+ import warnings
18
+ import shutil
19
+
20
+ from transformers import AutoTokenizer, AutoModelForCausalLM, AutoConfig, BitsAndBytesConfig
21
+ import torch
22
+ from llava.model import *
23
+ from llava.constants import DEFAULT_IMAGE_PATCH_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
24
+
25
+
26
+ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, load_4bit=False, device_map="auto", device="cuda", use_flash_attn=False, **kwargs):
27
+ kwargs = {"device_map": device_map, **kwargs}
28
+
29
+ if device != "cuda":
30
+ kwargs['device_map'] = {"": device}
31
+
32
+ if load_8bit:
33
+ kwargs['load_in_8bit'] = True
34
+ elif load_4bit:
35
+ kwargs['load_in_4bit'] = True
36
+ kwargs['quantization_config'] = BitsAndBytesConfig(
37
+ load_in_4bit=True,
38
+ bnb_4bit_compute_dtype=torch.float16,
39
+ bnb_4bit_use_double_quant=True,
40
+ bnb_4bit_quant_type='nf4'
41
+ )
42
+ else:
43
+ kwargs['torch_dtype'] = torch.float16
44
+
45
+ if use_flash_attn:
46
+ kwargs['attn_implementation'] = 'flash_attention_2'
47
+ token = kwargs.get('token', None)
48
+ if 'llava' in model_name.lower():
49
+ # Load LLaVA model
50
+ if 'lora' in model_name.lower() and model_base is None:
51
+ warnings.warn('There is `lora` in model name but no `model_base` is provided. If you are loading a LoRA model, please provide the `model_base` argument. Detailed instruction: https://github.com/haotian-liu/LLaVA#launch-a-model-worker-lora-weights-unmerged.')
52
+ if 'lora' in model_name.lower() and model_base is not None:
53
+ from llava.model.language_model.llava_llama import LlavaConfig
54
+ lora_cfg_pretrained = LlavaConfig.from_pretrained(model_path)
55
+ tokenizer = AutoTokenizer.from_pretrained(model_base, use_fast=False)
56
+ print('Loading LLaVA from base model...')
57
+ model = LlavaLlamaForCausalLM.from_pretrained(model_base, low_cpu_mem_usage=True, config=lora_cfg_pretrained, **kwargs)
58
+ token_num, tokem_dim = model.lm_head.out_features, model.lm_head.in_features
59
+ if model.lm_head.weight.shape[0] != token_num:
60
+ model.lm_head.weight = torch.nn.Parameter(torch.empty(token_num, tokem_dim, device=model.device, dtype=model.dtype))
61
+ model.model.embed_tokens.weight = torch.nn.Parameter(torch.empty(token_num, tokem_dim, device=model.device, dtype=model.dtype))
62
+
63
+ print('Loading additional LLaVA weights...')
64
+ if os.path.exists(os.path.join(model_path, 'non_lora_trainables.bin')):
65
+ non_lora_trainables = torch.load(os.path.join(model_path, 'non_lora_trainables.bin'), map_location='cpu')
66
+ else:
67
+ # this is probably from HF Hub
68
+ from huggingface_hub import hf_hub_download
69
+ def load_from_hf(repo_id, filename, subfolder=None):
70
+ cache_file = hf_hub_download(
71
+ repo_id=repo_id,
72
+ filename=filename,
73
+ subfolder=subfolder)
74
+ return torch.load(cache_file, map_location='cpu')
75
+ non_lora_trainables = load_from_hf(model_path, 'non_lora_trainables.bin')
76
+ non_lora_trainables = {(k[11:] if k.startswith('base_model.') else k): v for k, v in non_lora_trainables.items()}
77
+ if any(k.startswith('model.model.') for k in non_lora_trainables):
78
+ non_lora_trainables = {(k[6:] if k.startswith('model.') else k): v for k, v in non_lora_trainables.items()}
79
+ model.load_state_dict(non_lora_trainables, strict=False)
80
+
81
+ from peft import PeftModel
82
+ print('Loading LoRA weights...')
83
+ model = PeftModel.from_pretrained(model, model_path)
84
+ print('Merging LoRA weights...')
85
+ model = model.merge_and_unload()
86
+ print('Model is loaded...')
87
+ elif model_base is not None:
88
+ # this may be mm projector only
89
+ print('Loading LLaVA from base model...')
90
+ if 'mpt' in model_name.lower():
91
+ if not os.path.isfile(os.path.join(model_path, 'configuration_mpt.py')):
92
+ shutil.copyfile(os.path.join(model_base, 'configuration_mpt.py'), os.path.join(model_path, 'configuration_mpt.py'))
93
+ tokenizer = AutoTokenizer.from_pretrained(model_base, use_fast=True)
94
+ cfg_pretrained = AutoConfig.from_pretrained(model_path, trust_remote_code=True)
95
+ model = LlavaMptForCausalLM.from_pretrained(model_base, low_cpu_mem_usage=True, config=cfg_pretrained, **kwargs)
96
+ else:
97
+ tokenizer = AutoTokenizer.from_pretrained(model_base, use_fast=False)
98
+ cfg_pretrained = AutoConfig.from_pretrained(model_path)
99
+ model = LlavaLlamaForCausalLM.from_pretrained(model_base, low_cpu_mem_usage=True, config=cfg_pretrained, **kwargs)
100
+
101
+ mm_projector_weights = torch.load(os.path.join(model_path, 'mm_projector.bin'), map_location='cpu')
102
+ mm_projector_weights = {k: v.to(torch.float16) for k, v in mm_projector_weights.items()}
103
+ model.load_state_dict(mm_projector_weights, strict=False)
104
+ else:
105
+ if 'mpt' in model_name.lower():
106
+ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True)
107
+ model = LlavaMptForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, **kwargs)
108
+ elif 'mistral' in model_name.lower():
109
+ tokenizer = AutoTokenizer.from_pretrained(model_path)
110
+ model = LlavaMistralForCausalLM.from_pretrained(
111
+ model_path,
112
+ low_cpu_mem_usage=True,
113
+ **kwargs
114
+ )
115
+ else:
116
+ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False, token=token)
117
+ model = LlavaLlamaForCausalLM.from_pretrained(
118
+ model_path,
119
+ low_cpu_mem_usage=True,
120
+ **kwargs
121
+ )
122
+ else:
123
+ # Load language model
124
+ if model_base is not None:
125
+ # PEFT model
126
+ from peft import PeftModel
127
+ tokenizer = AutoTokenizer.from_pretrained(model_base, use_fast=False)
128
+ model = AutoModelForCausalLM.from_pretrained(model_base, low_cpu_mem_usage=True, **kwargs)
129
+ print(f"Loading LoRA weights from {model_path}")
130
+ model = PeftModel.from_pretrained(model, model_path)
131
+ print(f"Merging weights")
132
+ model = model.merge_and_unload()
133
+ print('Convert to FP16...')
134
+ model.to(torch.float16)
135
+ else:
136
+ use_fast = False
137
+ if 'mpt' in model_name.lower():
138
+ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True)
139
+ model = AutoModelForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, trust_remote_code=True, **kwargs)
140
+ else:
141
+ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
142
+ model = AutoModelForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, **kwargs)
143
+
144
+ image_processor = None
145
+
146
+ if 'llava' in model_name.lower():
147
+ mm_use_im_start_end = getattr(model.config, "mm_use_im_start_end", False)
148
+ mm_use_im_patch_token = getattr(model.config, "mm_use_im_patch_token", True)
149
+ if mm_use_im_patch_token:
150
+ tokenizer.add_tokens([DEFAULT_IMAGE_PATCH_TOKEN], special_tokens=True)
151
+ if mm_use_im_start_end:
152
+ tokenizer.add_tokens([DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN], special_tokens=True)
153
+ model.resize_token_embeddings(len(tokenizer))
154
+
155
+ vision_tower = model.get_vision_tower()
156
+ if not vision_tower.is_loaded:
157
+ vision_tower.load_model(device_map=device_map)
158
+ if device_map != 'auto':
159
+ vision_tower.to(device=device_map, dtype=torch.float16)
160
+ image_processor = vision_tower.image_processor
161
+
162
+ if hasattr(model.config, "max_sequence_length"):
163
+ context_len = model.config.max_sequence_length
164
+ else:
165
+ context_len = 2048
166
+
167
+ return tokenizer, model, image_processor, context_len