MrYxJ commited on
Commit
62d807b
1 Parent(s): 39eac5d
Files changed (4) hide show
  1. __init__.py +15 -0
  2. app.py +107 -0
  3. model_utils.py +151 -0
  4. requirements.txt +7 -0
__init__.py ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # !usr/bin/env python
2
+ # -*- coding:utf-8 -*-
3
+
4
+ '''
5
+ Description :
6
+ Version : 1.0
7
+ Author : MrYXJ
8
+ Mail : yxj2017@gmail.com
9
+ Github : https://github.com/MrYxJ
10
+ Date : 2023-09-05 23:26:00
11
+ LastEditTime : 2023-09-05 23:26:02
12
+ Copyright (C) 2023 mryxj. All rights reserved.
13
+ '''
14
+
15
+
app.py ADDED
@@ -0,0 +1,107 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # !usr/bin/env python
2
+ # -*- coding:utf-8 -*-
3
+
4
+ '''
5
+ Description :
6
+ Version : 1.0
7
+ Author : MrYXJ
8
+ Mail : yxj2017@gmail.com
9
+ Github : https://github.com/MrYxJ
10
+ Date : 2023-09-05 23:25:28
11
+ LastEditTime : 2023-09-09 19:24:09
12
+ Copyright (C) 2023 mryxj. All rights reserved.
13
+ '''
14
+ import gradio as gr
15
+ import pandas as pd
16
+
17
+ from model_utils import calculate_flops_in_hugging_space
18
+ from model_utils import get_mode_from_hf
19
+
20
+
21
+ def get_calclops_result(model_name: str,
22
+ input_shape: tuple,
23
+ access_token: str,
24
+ bp_factor: float,
25
+ output_unit: str):
26
+
27
+ try:
28
+ input_shape = eval(input_shape)
29
+ if not isinstance(input_shape, tuple):
30
+ raise Exception("Not a tuple.")
31
+ except Exception as e:
32
+ raise gr.Error(f"Input shape must be a tuple.")
33
+
34
+ try:
35
+ bp_factor = float(bp_factor)
36
+ except Exception as e:
37
+ raise gr.Error(f"The Factor of Backward as multiple as Forward must be a float.")
38
+
39
+ if output_unit not in ["", "auto", "T", "G", "M", "K", "m", "u"]:
40
+ raise gr.Error(
41
+ f"The Output Unit only can be leaved blank or 'T'、'G'、'M'、'K'、'm'、'u'."
42
+ )
43
+
44
+ model = get_mode_from_hf(model_name=model_name,
45
+ library="auto",
46
+ access_token=access_token)
47
+
48
+ title = f"## Calculate FLOPs and Params of '{model_name}' "
49
+ data, return_print = calculate_flops_in_hugging_space(model_name=model_name,
50
+ empty_model=model,
51
+ access_token=access_token,
52
+ input_shape=input_shape,
53
+ bp_factor=bp_factor,
54
+ output_unit=output_unit)
55
+
56
+ return [title, gr.update(visible=True, value=pd.DataFrame(data)), gr.update(visible=True, value=return_print)]
57
+
58
+
59
+ with gr.Blocks() as demo:
60
+ with gr.Column():
61
+ gr.Markdown(
62
+ """<img src="https://raw.githubusercontent.com/MrYxJ/calculate-flops.pytorch/main/screenshot/calflops_hf3.png?raw=true" style="float: left;" width="250" height="250"><h1> ⛽️Model(Transformers) FLOPs and Parameter Calculator</h1>
63
+ This tool is based on the python package [calflops](https://pypi.org/project/calflops/) developed in 🤗Huggingface Space specifically for mainly transformers model to compute FLOPs and Parameters num more convenient.
64
+ And calflops is designed to compute the theoretical amount of FLOPs(floating-point operations)、MACs(multiply-add operations) and Parameters in all various neural networks, also including **Transformers(Bert、LlaMA etc Large Language Model) Models** as long as based on the Pytorch implementation,
65
+ , which also open source on github, welcome to [star🌟 and use🌟](https://raw.githubusercontent.com/MrYxJ/calculate-flops.pytorch).
66
+
67
+ This tool can support the calculation of FLOPs without downloading the parameters of the whole model to the local which is very friendly for large models, but at the same time, this method can only support forward propagation of the model in the `meta` device (Some model implemented operators are not supported on meta device).
68
+ Therefore, it is possible that the model operator cannot calculate FLOPs in this way because it does not support calculation on `meta` device, which is that you can also calculate FLOPs by ```pip install calflops``` and download the parameters of the full model to the local.
69
+
70
+ This tool also support to calculate the FLOPs of model once complete forward + backward pass. According to [epochai report](https://epochai.org/blog/backward-forward-FLOP-ratio), the backward pass of the model FLOPs is default **2** times as much computation as forward pass.
71
+ Considering the distributed training of large models, according to [paper](https://arxiv.org/pdf/2205.05198.pdf), if you use parameter activate the recalculation in order to save gpu memory which would make backward pass of models FLOPs would be **3** times as much computation as forward pass.
72
+ All in all, you can set the param `Factor of Backward as multiple as Forward` to determine how many times backward pass as much computation as forward pass. Meanwhile this tool supports the printing of FLOPS, Parameter calculation value and proportion of each submodule of the model, it is convient for users to understand the performance consumption of each part of the model and make targeted optimization.
73
+
74
+ To use this tool pass in the URL or model name on the 🤗HuggingFace Models that you want to calculate the FLOPs and Parameters, you can
75
+ select what input shape of model in one step、what times of model backward pass as much comsuption as forward pass and what unit of the calculation result. Finally, this tool is learned from anther good tool [model-memory-usage](https://huggingface.co/spaces/hf-accelerate/model-memory-usage) in the implementation process, 🙏thanks."""
76
+ )
77
+ out_text = gr.Markdown()
78
+ calculate_out = gr.DataFrame(
79
+ headers = ["Total Training Params", "Forward FLOPs", "Forward MACs", "Forward+Backward FLOPs", "Forward+Backward MACs"],
80
+ interactive=False,
81
+ visible=False
82
+ )
83
+ print_model = gr.Textbox(label=f"Print Each Modules FLOPs、Params", visible=False,
84
+ show_legend=True, show_copy_button=True)
85
+
86
+ with gr.Row():
87
+ inp = gr.Textbox(label="Model Name or URL", value="baichuan-inc/Baichuan2-13B-Chat")
88
+ access_token = gr.Textbox(label="API Token", placeholder="Optional (for gated models)")
89
+
90
+ with gr.Row():
91
+ input_shape = gr.Textbox(label="Input Shape, such: (batch, seq_len)", value="(1, 128)")
92
+ bp_factor = gr.Textbox(label="Factor of Backward as multiple as Forward", value="2.0")
93
+ output_unit = gr.Textbox(label="Outputs Unit, blank means not using unit", value="auto",
94
+ placeholder="Optional, leave blank or 'auto','T','G','M','K','m','u'.")
95
+
96
+ with gr.Row():
97
+ btn = gr.Button("Calculate FLOPs and Params")
98
+ post_to_hub = gr.Button(
99
+ value="Report results in this model repo's discussions!\n(Will open in a new tab)", visible=False
100
+ )
101
+ btn.click(
102
+ get_calclops_result,
103
+ inputs=[inp, input_shape, access_token, bp_factor, output_unit],
104
+ outputs=[out_text, calculate_out, print_model]
105
+ )
106
+
107
+ demo.launch()
model_utils.py ADDED
@@ -0,0 +1,151 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # !usr/bin/env python
2
+ # -*- coding:utf-8 -*-
3
+
4
+ '''
5
+ Description :
6
+ Version : 1.0
7
+ Author : MrYXJ
8
+ Mail : yxj2017@gmail.com
9
+ Github : https://github.com/MrYxJ
10
+ Date : 2023-09-05 23:28:32
11
+ LastEditTime : 2023-09-09 19:14:20
12
+ Copyright (C) 2023 mryxj. All rights reserved.
13
+ '''
14
+
15
+
16
+ import gradio as gr
17
+ import torch
18
+
19
+ from accelerate.commands.estimate import check_has_model
20
+ from urllib.parse import urlparse
21
+ from huggingface_hub.utils import GatedRepoError
22
+ from huggingface_hub.utils import RepositoryNotFoundError
23
+
24
+ from calflops import create_empty_model
25
+ from calflops import calculate_flops_hf
26
+ from calflops import flops_to_string
27
+ from calflops import macs_to_string
28
+ from calflops import params_to_string
29
+
30
+ def calculate_flops_in_hugging_space(model_name: str,
31
+ empty_model: torch.nn.modules,
32
+ access_token: str,
33
+ input_shape: tuple,
34
+ bp_factor: float,
35
+ output_unit: str):
36
+
37
+ "Calculates the FLOPs and Params usage for a model init on `meta` device"
38
+
39
+ try:
40
+ flops, macs, params, return_print = calculate_flops_hf(model_name=model_name,
41
+ empty_model=empty_model,
42
+ access_token=access_token,
43
+ input_shape=input_shape,
44
+ return_results=True,
45
+ output_as_string=False)
46
+ except Exception as e:
47
+ print("Error info:", e)
48
+ raise gr.Error(
49
+ f"Model `{model_name}` does not support inference on the meta device, You can download the complete model parameters to your local and using the python package calflops to calculate FLOPs and Params of model `{model_name}`."
50
+ )
51
+
52
+ fw_bp_flops = flops * (1.0 + bp_factor)
53
+ fw_bp_macs = macs * (1.0 + bp_factor)
54
+
55
+ if output_unit == "":
56
+ pass
57
+ elif output_unit == "auto":
58
+ params = params_to_string(params, units=None, precision=3)
59
+ flops = flops_to_string(flops, units=None, precision=3)
60
+ macs = macs_to_string(macs, units=None, precision=3)
61
+ fw_bp_flops = flops_to_string(fw_bp_flops, units=None, precision=3)
62
+ fw_bp_macs = macs_to_string(fw_bp_macs, units=None, precision=3)
63
+ elif output_unit == "T" or output_unit == "G" or output_unit == "M" or output_unit == "K" or output_unit == "m" or output_unit == "u":
64
+ params = params_to_string(params, units=output_unit, precision=3)
65
+ flops = flops_to_string(flops, units=output_unit, precision=3)
66
+ macs = macs_to_string(macs, units=output_unit, precision=3)
67
+ fw_bp_flops = flops_to_string(fw_bp_flops, units=output_unit, precision=3)
68
+ fw_bp_macs = macs_to_string(fw_bp_macs, units=output_unit, precision=3)
69
+
70
+ return_lines = return_print.split("\n")
71
+ return_start = False
72
+ return_print = ""
73
+ for line in return_lines[:-2]:
74
+ if return_start:
75
+ return_print += line + "\n"
76
+ if "Detailed" in line:
77
+ return_start = True
78
+
79
+ data = []
80
+ data.append(
81
+ { "Total Training Params": params,
82
+ "Forward FLOPs": flops,
83
+ "Forward MACs": macs,
84
+ "Forward+Backward FLOPs": fw_bp_flops,
85
+ "Forward+Backward MACs": fw_bp_macs
86
+ }
87
+ )
88
+ return data, return_print
89
+
90
+
91
+ def extract_from_url(name: str):
92
+ "Checks if `name` is a URL, and if so converts it to a model name"
93
+ is_url = False
94
+ try:
95
+ result = urlparse(name)
96
+ is_url = all([result.scheme, result.netloc])
97
+ except Exception:
98
+ is_url = False
99
+ # Pass through if not a URL
100
+ if not is_url:
101
+ return name
102
+ else:
103
+ path = result.path
104
+ return path[1:]
105
+
106
+
107
+ def translate_llama2(text):
108
+ "Translates llama-2 to its hf counterpart"
109
+ if not text.endswith("-hf"):
110
+ return text + "-hf"
111
+ return text
112
+
113
+
114
+ def get_mode_from_hf(model_name: str, library: str, access_token: str):
115
+ "Finds and grabs model from the Hub, and initializes on `meta`"
116
+ if "meta-llama" in model_name:
117
+ model_name = translate_llama2(model_name)
118
+ if library == "auto":
119
+ library = None
120
+ model_name = extract_from_url(model_name)
121
+ try:
122
+ model = create_empty_model(model_name, library_name=library, trust_remote_code=True, access_token=access_token)
123
+ except GatedRepoError:
124
+ raise gr.Error(
125
+ f"Model `{model_name}` is a gated model, please ensure to pass in your access token and try again if you have access. You can find your access token here : https://huggingface.co/settings/tokens. "
126
+ )
127
+ except RepositoryNotFoundError:
128
+ raise gr.Error(f"Model `{model_name}` was not found on the Hub, please try another model name.")
129
+ except ValueError:
130
+ raise gr.Error(
131
+ f"Model `{model_name}` does not have any library metadata on the Hub, please manually select a library_name to use (such as `transformers`)"
132
+ )
133
+ except (RuntimeError, OSError) as e:
134
+ library = check_has_model(e)
135
+ if library != "unknown":
136
+ raise gr.Error(
137
+ f"Tried to load `{model_name}` with `{library}` but a possible model to load was not found inside the repo."
138
+ )
139
+ raise gr.Error(
140
+ f"Model `{model_name}` had an error, please open a discussion on the model's page with the error message and name: `{e}`"
141
+ )
142
+ except ImportError:
143
+ # hacky way to check if it works with `trust_remote_code=False`
144
+ model = create_empty_model(
145
+ model_name, library_name=library, trust_remote_code=False, access_token=access_token
146
+ )
147
+ except Exception as e:
148
+ raise gr.Error(
149
+ f"Model `{model_name}` had an error, please open a discussion on the model's page with the error message and name: `{e}`"
150
+ )
151
+ return model
requirements.txt ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ calflops @ git+https://github.com/MrYxJ/calculate-flops.pytorch
2
+ accelerate @ git+https://github.com/huggingface/accelerate
3
+ transformers
4
+ timm
5
+ huggingface_hub
6
+ tabulate
7
+ pandas