jbilcke-hf HF staff commited on
Commit
9bfb451
β€’
1 Parent(s): faf4ba4

add new "fast LLM" optimization

Browse files
src/app/main.tsx CHANGED
@@ -7,7 +7,6 @@ import { TopMenu } from "./interface/top-menu"
7
  import { fonts } from "@/lib/fonts"
8
  import { useStore } from "./store"
9
  import { Zoom } from "./interface/zoom"
10
- import { getStory } from "./queries/getStory"
11
  import { BottomBar } from "./interface/bottom-bar"
12
  import { Page } from "./interface/page"
13
  import { GeneratedPanel } from "@/types"
@@ -88,7 +87,8 @@ export default function Main() {
88
  nbPanelsToGenerate,
89
  existingPanels,
90
  })
91
- console.log("LLM generated some new panels:", candidatePanels)
 
92
 
93
  existingPanels.push(...candidatePanels)
94
 
@@ -114,7 +114,7 @@ export default function Main() {
114
  }
115
 
116
  // update the frontend
117
- console.log("updating the frontend..")
118
  setCaptions(newCaptions)
119
  setPanels(newPanelsPrompts)
120
 
 
7
  import { fonts } from "@/lib/fonts"
8
  import { useStore } from "./store"
9
  import { Zoom } from "./interface/zoom"
 
10
  import { BottomBar } from "./interface/bottom-bar"
11
  import { Page } from "./interface/page"
12
  import { GeneratedPanel } from "@/types"
 
87
  nbPanelsToGenerate,
88
  existingPanels,
89
  })
90
+ console.log("LLM generated some new panels:")
91
+ console.table(candidatePanels)
92
 
93
  existingPanels.push(...candidatePanels)
94
 
 
114
  }
115
 
116
  // update the frontend
117
+ // console.log("updating the frontend..")
118
  setCaptions(newCaptions)
119
  setPanels(newPanelsPrompts)
120
 
src/app/queries/getStory.ts CHANGED
@@ -17,7 +17,7 @@ export const getStory = async ({
17
  prompt: string;
18
  nbTotalPanels: number;
19
  }): Promise<GeneratedPanels> => {
20
- // throw new Error("Planned maintenance")
21
 
22
  // In case you need to quickly debug the RENDERING engine you can uncomment this:
23
  // return mockGeneratedPanels
 
17
  prompt: string;
18
  nbTotalPanels: number;
19
  }): Promise<GeneratedPanels> => {
20
+ throw new Error("legacy, deprecated")
21
 
22
  // In case you need to quickly debug the RENDERING engine you can uncomment this:
23
  // return mockGeneratedPanels
src/app/queries/getStoryContinuation.ts CHANGED
@@ -32,23 +32,22 @@ export const getStoryContinuation = async ({
32
  existingPanels,
33
  })
34
 
35
- console.log("LLM responded with panels:", panelCandidates)
36
-
37
 
38
  // we clean the output from the LLM
39
  // most importantly, we need to adjust the panel index,
40
  // to start from where we last finished
41
- for (let p = startAt; p < endAt; p++) {
42
  panels.push({
43
- panel: p,
44
- instructions: `${panelCandidates[p]?.instructions || ""}`,
45
- caption: `${panelCandidates[p]?.caption || ""}`,
46
  })
47
  }
48
 
49
  } catch (err) {
50
- console.log("LLM step failed due to:", err)
51
- console.log("we are now switching to a degraded mode, using 4 similar panels")
52
  panels = []
53
  for (let p = startAt; p < endAt; p++) {
54
  panels.push({
@@ -61,7 +60,7 @@ export const getStoryContinuation = async ({
61
  caption: "(Sorry, LLM generation failed: using degraded mode)"
62
  })
63
  }
64
- console.error(err)
65
  } finally {
66
  return panels
67
  }
 
32
  existingPanels,
33
  })
34
 
35
+ // console.log("LLM responded with panelCandidates:", panelCandidates)
 
36
 
37
  // we clean the output from the LLM
38
  // most importantly, we need to adjust the panel index,
39
  // to start from where we last finished
40
+ for (let i = 0; i < nbPanelsToGenerate; i++) {
41
  panels.push({
42
+ panel: startAt + i,
43
+ instructions: `${panelCandidates[i]?.instructions || ""}`,
44
+ caption: `${panelCandidates[i]?.caption || ""}`,
45
  })
46
  }
47
 
48
  } catch (err) {
49
+ // console.log("LLM step failed due to:", err)
50
+ // console.log("we are now switching to a degraded mode, using 4 similar panels")
51
  panels = []
52
  for (let p = startAt; p < endAt; p++) {
53
  panels.push({
 
60
  caption: "(Sorry, LLM generation failed: using degraded mode)"
61
  })
62
  }
63
+ // console.error(err)
64
  } finally {
65
  return panels
66
  }
src/app/queries/predictNextPanels.ts CHANGED
@@ -18,6 +18,7 @@ export const predictNextPanels = async ({
18
  nbPanelsToGenerate: number;
19
  existingPanels: GeneratedPanel[];
20
  }): Promise<GeneratedPanel[]> => {
 
21
  // throw new Error("Planned maintenance")
22
 
23
  // In case you need to quickly debug the RENDERING engine you can uncomment this:
 
18
  nbPanelsToGenerate: number;
19
  existingPanels: GeneratedPanel[];
20
  }): Promise<GeneratedPanel[]> => {
21
+ // console.log("predictNextPanels: ", { prompt, nbPanelsToGenerate })
22
  // throw new Error("Planned maintenance")
23
 
24
  // In case you need to quickly debug the RENDERING engine you can uncomment this:
src/app/queries/predictWithHuggingFace.ts CHANGED
@@ -58,7 +58,7 @@ export async function predict(inputs: string, nbPanels: number): Promise<string>
58
  }
59
  })) {
60
  instructions += output.token.text
61
- process.stdout.write(output.token.text)
62
  if (
63
  instructions.includes("</s>") ||
64
  instructions.includes("<s>") ||
@@ -78,7 +78,7 @@ export async function predict(inputs: string, nbPanels: number): Promise<string>
78
  }
79
  }
80
  } catch (err) {
81
- console.error(`error during generation: ${err}`)
82
 
83
  // a common issue with Llama-2 might be that the model receives too many requests
84
  if (`${err}` === "Error: Model is overloaded") {
 
58
  }
59
  })) {
60
  instructions += output.token.text
61
+ // process.stdout.write(output.token.text)
62
  if (
63
  instructions.includes("</s>") ||
64
  instructions.includes("<s>") ||
 
78
  }
79
  }
80
  } catch (err) {
81
+ // console.error(`error during generation: ${err}`)
82
 
83
  // a common issue with Llama-2 might be that the model receives too many requests
84
  if (`${err}` === "Error: Model is overloaded") {
src/lib/dirtyLLMResponseCleaner.ts CHANGED
@@ -2,12 +2,14 @@ export function dirtyLLMResponseCleaner(input: string): string {
2
  let str = (
3
  `${input || ""}`
4
  // a summary of all the weird hallucinations I saw it make..
 
5
  .replaceAll(`"]`, `"}]`)
6
- .replaceAll(`" ]`, `"}]`)
7
- .replaceAll(`" ]`, `"}]`)
8
- .replaceAll(`"\n]`, `"}]`)
9
- .replaceAll(`"\n ]`, `"}]`)
10
- .replaceAll(`"\n ]`, `"}]`)
 
11
  .replaceAll("}}", "}")
12
  .replaceAll("]]", "]")
13
  .replaceAll("[[", "[")
 
2
  let str = (
3
  `${input || ""}`
4
  // a summary of all the weird hallucinations I saw it make..
5
+ .replaceAll(`"\n`, `",\n`) // fix missing commas at the end of a line
6
  .replaceAll(`"]`, `"}]`)
7
+ .replaceAll(/"\S*,?\S*\]/gi, `"}]`)
8
+ .replaceAll(/"\S*,?\S*\}\S*]/gi, `"}]`)
9
+
10
+ // this removes the trailing commas (which are valid in JS but not JSON)
11
+ .replace(/,(?=\s*?[\}\]])/g, '')
12
+
13
  .replaceAll("}}", "}")
14
  .replaceAll("]]", "]")
15
  .replaceAll("[[", "[")