jbilcke-hf HF staff commited on
Commit
274b152
β€’
2 Parent(s): e52146b 67f4bb9
Files changed (4) hide show
  1. .gitignore +2 -0
  2. README.md +16 -3
  3. package.json +1 -0
  4. src/app/queries/predict.ts +47 -1
.gitignore CHANGED
@@ -33,3 +33,5 @@ yarn-error.log*
33
  # typescript
34
  *.tsbuildinfo
35
  next-env.d.ts
 
 
 
33
  # typescript
34
  *.tsbuildinfo
35
  next-env.d.ts
36
+
37
+ pnpm-lock.yaml
README.md CHANGED
@@ -92,14 +92,27 @@ HF_INFERENCE_ENDPOINT_URL="path to your inference endpoint url"
92
 
93
  To run this kind of LLM locally, you can use [TGI](https://github.com/huggingface/text-generation-inference) (Please read [this post](https://github.com/huggingface/text-generation-inference/issues/726) for more information about the licensing).
94
 
95
- ### Option 3: Fork and modify the code to use a different LLM system
96
 
97
- Another option could be to disable the LLM completely and replace it with another LLM protocol and/or provider (eg. OpenAI, Replicate), or a human-generated story instead (by returning mock or static data).
98
 
 
 
 
 
 
 
 
 
 
 
 
 
 
99
 
100
  ### Notes
101
 
102
- It is possible that I modify the AI Comic Factory to make it easier in the future (eg. add support for OpenAI or Replicate)
103
 
104
  ## The Rendering API
105
 
 
92
 
93
  To run this kind of LLM locally, you can use [TGI](https://github.com/huggingface/text-generation-inference) (Please read [this post](https://github.com/huggingface/text-generation-inference/issues/726) for more information about the licensing).
94
 
95
+ ### Option 3: Use an OpenAI API Key
96
 
97
+ This is a new option added recently, where you can use OpenAI API with an OpenAI API Key.
98
 
99
+ To activate it, create a `.env.local` configuration file:
100
+
101
+ ```bash
102
+ LLM_ENGINE="OPENAI"
103
+ # default openai api base url is: https://api.openai.com/v1
104
+ OPENAI_API_BASE_URL="Your OpenAI API Base URL"
105
+ OPENAI_API_KEY="Your OpenAI API Key"
106
+ OPENAI_API_MODEL="gpt-3.5-turbo"
107
+ ```
108
+
109
+ ### Option 4: Fork and modify the code to use a different LLM system
110
+
111
+ Another option could be to disable the LLM completely and replace it with another LLM protocol and/or provider (eg. Claude, Replicate), or a human-generated story instead (by returning mock or static data).
112
 
113
  ### Notes
114
 
115
+ It is possible that I modify the AI Comic Factory to make it easier in the future (eg. add support for Claude or Replicate)
116
 
117
  ## The Rendering API
118
 
package.json CHANGED
@@ -43,6 +43,7 @@
43
  "html2canvas": "^1.4.1",
44
  "lucide-react": "^0.260.0",
45
  "next": "13.4.10",
 
46
  "pick": "^0.0.1",
47
  "postcss": "8.4.26",
48
  "react": "18.2.0",
 
43
  "html2canvas": "^1.4.1",
44
  "lucide-react": "^0.260.0",
45
  "next": "13.4.10",
46
+ "openai": "^4.10.0",
47
  "pick": "^0.0.1",
48
  "postcss": "8.4.26",
49
  "react": "18.2.0",
src/app/queries/predict.ts CHANGED
@@ -1,8 +1,11 @@
1
  "use server"
2
 
3
- import { LLMEngine } from "@/types"
4
  import { HfInference, HfInferenceEndpoint } from "@huggingface/inference"
5
 
 
 
 
 
6
  const hf = new HfInference(process.env.HF_API_TOKEN)
7
 
8
 
@@ -10,6 +13,8 @@ const hf = new HfInference(process.env.HF_API_TOKEN)
10
  const llmEngine = `${process.env.LLM_ENGINE || ""}` as LLMEngine
11
  const inferenceEndpoint = `${process.env.LLM_HF_INFERENCE_ENDPOINT_URL || ""}`
12
  const inferenceModel = `${process.env.LLM_HF_INFERENCE_API_MODEL || ""}`
 
 
13
 
14
  let hfie: HfInferenceEndpoint
15
 
@@ -34,6 +39,16 @@ switch (llmEngine) {
34
  throw new Error(error)
35
  }
36
  break;
 
 
 
 
 
 
 
 
 
 
37
 
38
  default:
39
  const error = "No Inference Endpoint URL or Inference API Model defined"
@@ -45,6 +60,10 @@ export async function predict(inputs: string) {
45
 
46
  console.log(`predict: `, inputs)
47
 
 
 
 
 
48
  const api = llmEngine ==="INFERENCE_ENDPOINT" ? hfie : hf
49
 
50
  let instructions = ""
@@ -92,4 +111,31 @@ export async function predict(inputs: string) {
92
  .replaceAll("<|assistant|>", "")
93
  .replaceAll('""', '"')
94
  )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
95
  }
 
1
  "use server"
2
 
 
3
  import { HfInference, HfInferenceEndpoint } from "@huggingface/inference"
4
 
5
+ import type { ChatCompletionMessage } from "openai/resources/chat"
6
+ import { LLMEngine } from "@/types"
7
+ import OpenAI from "openai"
8
+
9
  const hf = new HfInference(process.env.HF_API_TOKEN)
10
 
11
 
 
13
  const llmEngine = `${process.env.LLM_ENGINE || ""}` as LLMEngine
14
  const inferenceEndpoint = `${process.env.LLM_HF_INFERENCE_ENDPOINT_URL || ""}`
15
  const inferenceModel = `${process.env.LLM_HF_INFERENCE_API_MODEL || ""}`
16
+ const openaiApiKey = `${process.env.LLM_OPENAI_API_KEY || ""}`
17
+
18
 
19
  let hfie: HfInferenceEndpoint
20
 
 
39
  throw new Error(error)
40
  }
41
  break;
42
+
43
+ case "OPENAI":
44
+ if (openaiApiKey) {
45
+ console.log("Using an OpenAI API Key")
46
+ } else {
47
+ const error = "No OpenAI API key defined"
48
+ console.error(error)
49
+ throw new Error(error)
50
+ }
51
+ break;
52
 
53
  default:
54
  const error = "No Inference Endpoint URL or Inference API Model defined"
 
60
 
61
  console.log(`predict: `, inputs)
62
 
63
+ if (llmEngine==="OPENAI") {
64
+ return predictWithOpenAI(inputs)
65
+ }
66
+
67
  const api = llmEngine ==="INFERENCE_ENDPOINT" ? hfie : hf
68
 
69
  let instructions = ""
 
111
  .replaceAll("<|assistant|>", "")
112
  .replaceAll('""', '"')
113
  )
114
+ }
115
+
116
+ async function predictWithOpenAI(inputs: string) {
117
+ const openaiApiBaseUrl = `${process.env.OPENAI_API_BASE_URL || "https://api.openai.com/v1"}`
118
+ const openaiApiModel = `${process.env.OPENAI_API_MODEL || "gpt-3.5-turbo"}`
119
+
120
+ const openai = new OpenAI({
121
+ apiKey: openaiApiKey,
122
+ baseURL: openaiApiBaseUrl,
123
+ })
124
+
125
+ const messages: ChatCompletionMessage[] = [
126
+ { role: "system", content: inputs },
127
+ ]
128
+
129
+ try {
130
+ const res = await openai.chat.completions.create({
131
+ messages: messages,
132
+ stream: false,
133
+ model: openaiApiModel,
134
+ temperature: 0.8
135
+ })
136
+
137
+ return res.choices[0].message.content
138
+ } catch (err) {
139
+ console.error(`error during generation: ${err}`)
140
+ }
141
  }