avi-pipable
commited on
Commit
•
2d58f3f
1
Parent(s):
ada74c3
Update README.md
Browse files
README.md
CHANGED
@@ -30,7 +30,8 @@ widget:
|
|
30 |
## What have we built?
|
31 |
|
32 |
A 1.3 bn code documentation model that outperforms most models on documenting codes and making your in-house libs ready for LLM and RAG pipelines.
|
33 |
-
We have also open sourced a [
|
|
|
34 |
This is a further trained version of pip-sql-1.3b.
|
35 |
|
36 |
## How we built it?
|
@@ -46,42 +47,9 @@ The model is open source under apache 2.0. License
|
|
46 |
|
47 |
|
48 |
### Library use
|
49 |
-
```python
|
50 |
-
!pip3 install git+https://github.com/PipableAI/pip-library-parser
|
51 |
-
!pip3 install atlassian-python-api
|
52 |
-
|
53 |
-
|
54 |
-
from pip_library_parser import CodeToDocGenerator
|
55 |
-
|
56 |
-
# Replace 'your_module' and 'YourModule' with the actual module and module name
|
57 |
-
module_name = 'your_module'
|
58 |
-
module = __import__(module_name)
|
59 |
-
|
60 |
-
# Instantiate the CodeToDocGenerator
|
61 |
-
generator = CodeToDocGenerator()
|
62 |
-
|
63 |
-
# Generate docstrings for the module's functions and methods
|
64 |
-
docs = generator.generate_module_docs(module, module_name)
|
65 |
-
|
66 |
-
# 'docs' now contains a dictionary mapping function/method names to their generated docstrings
|
67 |
-
|
68 |
-
```
|
69 |
|
70 |
-
|
71 |
-
|
72 |
-
|
73 |
-
# Instantiate the CodeToDocGenerator
|
74 |
-
generator = CodeToDocGenerator()
|
75 |
-
|
76 |
-
code_snippet = """
|
77 |
-
def example_function(x):
|
78 |
-
return x * 2
|
79 |
-
"""
|
80 |
-
|
81 |
-
docstring = generator.generate_docstring_from_pip_model(code_snippet)
|
82 |
-
print("Generated Docstring:")
|
83 |
-
print(docstring)
|
84 |
-
```
|
85 |
|
86 |
### Installation
|
87 |
|
@@ -104,11 +72,11 @@ prompt = f"""<example_response>{example of some --question: , --query}</example
|
|
104 |
```python
|
105 |
from transformers import AutoModelForCausalLM, AutoTokenizer
|
106 |
device = "cuda"
|
107 |
-
model = AutoModelForCausalLM.from_pretrained("PipableAI/pip-
|
108 |
-
tokenizer = AutoTokenizer.from_pretrained("PipableAI/pip-
|
109 |
prompt = f"""<example_response>
|
110 |
--code:def function_2(x): return x / 2
|
111 |
-
--question:Document the code
|
112 |
--doc:
|
113 |
Description:This function takes a number and divides it by 2.
|
114 |
Parameters:
|
@@ -125,8 +93,19 @@ def example_function(x):
|
|
125 |
<question>Document the python code above giving function description ,parameters and return type and example how to call the function.</question>
|
126 |
<doc>"""
|
127 |
inputs = tokenizer(prompt, return_tensors="pt")
|
128 |
-
outputs = model.generate(**inputs, max_new_tokens=
|
129 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
130 |
```
|
131 |
|
132 |
|
@@ -138,7 +117,7 @@ tokenizer.decode(outputs[0], skip_special_tokens=True).split('<doc>')[-1].split(
|
|
138 |
```python
|
139 |
text=''' <example_response>
|
140 |
--code:def function_2(x): return x / 2
|
141 |
-
--question:Document the code
|
142 |
--doc:
|
143 |
Description:This function takes a number and divides it by 2.
|
144 |
Parameters:
|
|
|
30 |
## What have we built?
|
31 |
|
32 |
A 1.3 bn code documentation model that outperforms most models on documenting codes and making your in-house libs ready for LLM and RAG pipelines.
|
33 |
+
We have also open sourced a [pip library_etl](https://github.com/PipableAI/pip-library-etl.git) for the same, together the lib and model can turn your codebase to functional parse tree ready to be consumed by LLMs to execute complex tasks.
|
34 |
+
This model is also capable of generating SQL queries with accuracies on par with those of [pip-sql-1.3b](https://huggingface.co/PipableAI/pip-sql-1.3b), with additional capabilities of providing extra examples, instructions ,and column descriptions as context.
|
35 |
This is a further trained version of pip-sql-1.3b.
|
36 |
|
37 |
## How we built it?
|
|
|
47 |
|
48 |
|
49 |
### Library use
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
50 |
|
51 |
+
For directly using the capabilities of model without putting extra efforts on schems and prompts try to use [pip library_etl](https://github.com/PipableAI/pip-library-etl.git).
|
52 |
+
For detaied usage refer to the [colab_notebook](https://colab.research.google.com/drive/17PyMU_3QN9LROy7x-jmaema0cuLRzBvc?usp=sharing)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
53 |
|
54 |
### Installation
|
55 |
|
|
|
72 |
```python
|
73 |
from transformers import AutoModelForCausalLM, AutoTokenizer
|
74 |
device = "cuda"
|
75 |
+
model = AutoModelForCausalLM.from_pretrained("PipableAI/pip-library-etl-1.3b ").to(device)
|
76 |
+
tokenizer = AutoTokenizer.from_pretrained("PipableAI/pip-library-etl-1.3b b")
|
77 |
prompt = f"""<example_response>
|
78 |
--code:def function_2(x): return x / 2
|
79 |
+
--question:Document the python code above giving function description ,parameters and return type and example how to call the function.
|
80 |
--doc:
|
81 |
Description:This function takes a number and divides it by 2.
|
82 |
Parameters:
|
|
|
93 |
<question>Document the python code above giving function description ,parameters and return type and example how to call the function.</question>
|
94 |
<doc>"""
|
95 |
inputs = tokenizer(prompt, return_tensors="pt")
|
96 |
+
outputs = model.generate(**inputs, max_new_tokens=450)
|
97 |
+
doc = (
|
98 |
+
self.tokenizer.decode(outputs[0], skip_special_tokens=True)
|
99 |
+
.split("<doc>")[-1]
|
100 |
+
.split("</doc>")[0]
|
101 |
+
)
|
102 |
+
doc = (
|
103 |
+
doc.replace("<p>", "")
|
104 |
+
.replace("</p>", "")
|
105 |
+
.replace("<function_description>", "")
|
106 |
+
.replace("</function_description>", "")
|
107 |
+
)
|
108 |
+
print(doc)
|
109 |
```
|
110 |
|
111 |
|
|
|
117 |
```python
|
118 |
text=''' <example_response>
|
119 |
--code:def function_2(x): return x / 2
|
120 |
+
--question:Document the python code above giving function description ,parameters and return type and example how to call the function.
|
121 |
--doc:
|
122 |
Description:This function takes a number and divides it by 2.
|
123 |
Parameters:
|