File size: 6,328 Bytes
1fd7b67
d8410b4
095b5f1
f8e2c8b
12d3e1a
dc376b6
 
1286e81
12d3e1a
d07865c
 
 
d8410b4
d07865c
12d3e1a
d07865c
12d3e1a
1286e81
 
 
5ea29d8
ca8a144
a1c6081
 
d8410b4
ca8a144
 
 
 
 
 
 
 
 
 
 
 
 
 
c625f4c
 
ca8a144
1fd7b67
c625f4c
 
 
1fd7b67
c625f4c
 
1fd7b67
55f46c1
78209bc
55f46c1
c625f4c
f8e2c8b
 
 
 
 
 
 
 
baeaaa5
d07865c
f8e2c8b
 
d07865c
f8e2c8b
 
 
 
 
 
 
 
 
 
 
 
 
 
12d3e1a
f8e2c8b
 
d8410b4
 
 
 
095b5f1
12d3e1a
f8e2c8b
 
 
d8410b4
 
f8e2c8b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
d8410b4
 
 
 
 
f8e2c8b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
a1c6081
 
 
d8410b4
 
 
 
 
 
 
 
f8e2c8b
 
 
 
 
 
a1c6081
f8e2c8b
d8410b4
f8e2c8b
 
 
 
a1c6081
d8410b4
f8e2c8b
 
 
 
 
a1c6081
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
import os
from typing import cast
from _utils.bubble_integrations.enviar_resposta_final import enviar_resposta_final
from _utils.custom_exception_handler import custom_exception_handler_wihout_api_handler
from _utils.gerar_relatorio_modelo_usuario.prompts import prompt_auxiliar_SEM_CONTEXT
from _utils.gerar_relatorio_modelo_usuario.GerarDocumento import (
    GerarDocumento,
)
from _utils.gerar_relatorio_modelo_usuario.contextual_retriever import (
    ContextualRetriever,
)
from _utils.gerar_relatorio_modelo_usuario.utils import (
    generate_document_title,
    gerar_resposta_compilada,
    get_full_text_and_all_PDFs_chunks,
    get_response_from_auxiliar_contextual_prompt,
)
from _utils.models.gerar_relatorio import (
    RetrievalConfig,
)
import markdown

from _utils.utils import convert_markdown_to_HTML


def reciprocal_rank_fusion(result_lists, weights=None):
    """Combine multiple ranked lists using reciprocal rank fusion"""
    fused_scores = {}
    num_lists = len(result_lists)
    if weights is None:
        weights = [1.0] * num_lists

    for i in range(num_lists):
        for doc_id, score in result_lists[i]:
            if doc_id not in fused_scores:
                fused_scores[doc_id] = 0
            fused_scores[doc_id] += weights[i] * score

    # Sort by score in descending order
    sorted_results = sorted(fused_scores.items(), key=lambda x: x[1], reverse=True)

    return sorted_results


os.environ["LANGCHAIN_TRACING_V2"] = "true"
os.environ["LANGCHAIN_ENDPOINT"] = "https://api.smith.langchain.com"
os.environ.get("LANGCHAIN_API_KEY")
os.environ["LANGCHAIN_PROJECT"] = "VELLA"


async def get_llm_summary_answer_by_cursor_complete(
    serializer, listaPDFs, isBubble=False
):
    """Parâmetro "contexto" só deve ser passado quando quiser utilizar o teste com ragas, e assim, não quiser passar PDFs"""
    try:
        # Configuration
        config = RetrievalConfig(
            num_chunks=serializer["num_chunks_retrieval"],
            embedding_weight=serializer["embedding_weight"],
            bm25_weight=serializer["bm25_weight"],
            context_window=serializer["context_window"],
            chunk_overlap=serializer["chunk_overlap"],
        )

        contextual_retriever = ContextualRetriever(
            config, serializer["claude_context_model"]
        )

        # Initialize enhanced summarizer
        summarizer = GerarDocumento(
            config=config,
            embedding_model=serializer["hf_embedding"],
            chunk_overlap=serializer["chunk_overlap"],
            chunk_size=serializer["chunk_size"],
            num_k_rerank=serializer["num_k_rerank"],
            model_cohere_rerank=serializer["model_cohere_rerank"],
            # prompt_auxiliar=serializer["prompt_auxiliar"],
            gpt_model=serializer["model"],
            gpt_temperature=serializer["gpt_temperature"],
            prompt_gerar_documento=serializer["prompt_gerar_documento"],
            reciprocal_rank_fusion=reciprocal_rank_fusion,
        )

        all_PDFs_chunks, full_text_as_array = await get_full_text_and_all_PDFs_chunks(
            listaPDFs,
            summarizer.splitter,
            serializer["should_use_llama_parse"],
            isBubble,
        )

        is_contextualized_chunk = serializer["should_have_contextual_chunks"]

        if is_contextualized_chunk:
            response_auxiliar_summary = (
                await get_response_from_auxiliar_contextual_prompt(full_text_as_array)
            )

            print("\nCOMEÇANDO A FAZER AS REQUISIÇÕES DO CONTEXTUAL")
            contextualized_chunks = await contextual_retriever.contextualize_all_chunks(
                all_PDFs_chunks, response_auxiliar_summary
            )
            print("\nTERMINOU DE FAZER TODAS AS REQUISIÇÕES DO CONTEXTUAL")
            chunks_processados = contextualized_chunks
        else:
            chunks_processados = all_PDFs_chunks

        # Create enhanced vector store and BM25 index
        vector_store, bm25, chunk_ids = (
            summarizer.vector_store.create_enhanced_vector_store(
                chunks_processados, is_contextualized_chunk
            )
        )

        llm_ultimas_requests = serializer["llm_ultimas_requests"]
        print("\nCOMEÇANDO A FAZER ÚLTIMA REQUISIÇÃO")
        structured_summaries = await summarizer.gerar_documento_final(
            vector_store,
            bm25,
            chunk_ids,
            llm_ultimas_requests,
            prompt_auxiliar_SEM_CONTEXT,
        )
        print("\nTERMINOU DE FAZER A ÚLTIMA REQUISIÇÃO")

        if not isinstance(structured_summaries, list):
            from rest_framework.response import Response

            return Response({"erro": structured_summaries})

        texto_completo = summarizer.resumo_gerado + "\n\n"

        for x in structured_summaries:
            texto_completo = texto_completo + x["content"] + "\n"
            x["source"]["text"] = x["source"]["text"][0:200]
            x["source"]["context"] = x["source"]["context"][0:200]

        texto_completo_como_html = convert_markdown_to_HTML(texto_completo)
        print("\ntexto_completo_como_html", texto_completo_como_html)

        if is_contextualized_chunk:
            prompt_titulo_do_documento = response_auxiliar_summary
        else:
            prompt_titulo_do_documento = texto_completo_como_html
        titulo_do_documento = await generate_document_title(
            cast(str, prompt_titulo_do_documento)
        )

        if isBubble:
            print("COMEÇANDO A REQUISIÇÃO FINAL PARA O BUBBLE")
            enviar_resposta_final(
                serializer["doc_id"],
                serializer["form_response_id"],
                serializer["version"],
                texto_completo_como_html,
                False,
                cast(str, titulo_do_documento),
            )
            print("TERMINOU A REQUISIÇÃO FINAL PARA O BUBBLE")

        return {
            "texto_completo": texto_completo_como_html,
            "titulo_do_documento": titulo_do_documento,
            "resultado": structured_summaries,
            "parametros-utilizados": gerar_resposta_compilada(serializer),
        }
    except Exception as e:
        custom_exception_handler_wihout_api_handler(e, serializer)
        raise