starting space

.gitignore

@@ -0,0 +1,5 @@
+*.pyc
+.DS_Store
+output
+code/data/init
+code/data/arabic-fonts/*.ttf

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "diffvg"]
+	path = diffvg
+	url = https://github.com/BachiLi/diffvg.git

README.md

@@ -1,8 +1,8 @@
 ---
 title: Word To Image
-emoji: 📚
-colorFrom: purple
-colorTo: green
+emoji: ✒️  ➡️ 🎨
+colorFrom: blue
+colorTo: pink
 sdk: gradio
 sdk_version: 3.29.0
 app_file: app.py

app.py

@@ -0,0 +1,368 @@
+import gradio as gr
+import os
+import argparse
+from easydict import EasyDict as edict
+import yaml
+import os.path as osp
+import random
+import numpy.random as npr
+import sys
+
+# sys.path.append('./code')
+
+sys.path.append('/home/user/app/code')
+
+# set up diffvg
+
+# os.system('git clone https://github.com/BachiLi/diffvg.git')
+os.system('git submodule update --init')
+os.chdir('diffvg')
+print(os.getcwd())
+os.system('git submodule update --init --recursive')
+print(os.getcwd())
+os.system('python setup.py install --user')
+sys.path.append("/home/user/.local/lib/python3.8/site-packages/diffvg-0.0.1-py3.8-linux-x86_64.egg")
+
+os.chdir('/home/user/app')
+
+import torch
+from diffusers import StableDiffusionPipeline
+
+
+device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+
+model = None 
+model = StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5",
+                                               torch_dtype=torch.float16).to(device)
+
+from typing import Mapping
+from tqdm import tqdm
+import torch
+from torch.optim.lr_scheduler import LambdaLR
+import pydiffvg
+import save_svg
+from losses import SDSLoss, ToneLoss, ConformalLoss
+from utils import (
+    edict_2_dict,
+    update,
+    check_and_create_dir,
+    get_data_augs,
+    save_image,
+    preprocess,
+    learning_rate_decay,
+    combine_word)
+import warnings
+
+TITLE="""<h1 style="font-size: 42px;" align="center">Word-As-Image for Semantic Typography</h1>"""
+DESCRIPTION="""A demo for [Word-As-Image for Semantic Typography](https://wordasimage.github.io/Word-As-Image-Page/). By using Word-as-Image, a visual representation of the meaning of the word is created while maintaining legibility of the text and font style. 
+Please select a semantic concept word and a letter you wish to generate, it will take ~5 minutes to perform 500 iterations."""
+
+DESCRIPTION += '\n<p>This demo is licensed under a <a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/"> Creative Commons Attribution-ShareAlike 4.0 International License</a>.</p>'
+
+if (SPACE_ID := os.getenv('SPACE_ID')) is not None:
+    DESCRIPTION += f'\n<p>For faster inference without waiting in queue, you may duplicate the space and upgrade to GPU in settings. <a href="https://huggingface.co/spaces/{SPACE_ID}?duplicate=true"><img style="display: inline; margin-top: 0em; margin-bottom: 0em" src="https://bit.ly/3gLdBN6" alt="Duplicate Space" /></a></p>'
+
+
+warnings.filterwarnings("ignore")
+
+pydiffvg.set_print_timing(False)
+gamma = 1.0
+
+
+def set_config(semantic_concept, word, letter, font_name, num_steps):
+    
+    cfg_d = edict()
+    cfg_d.config = "code/config/base.yaml"
+    cfg_d.experiment = "demo"
+
+    with open(cfg_d.config, 'r') as f:
+        cfg_full = yaml.load(f, Loader=yaml.FullLoader)
+
+    cfg_key = cfg_d.experiment
+    cfgs = [cfg_d]
+    while cfg_key:
+        cfgs.append(cfg_full[cfg_key])
+        cfg_key = cfgs[-1].get('parent_config', 'baseline')
+  
+    cfg = edict()
+    for options in reversed(cfgs):
+        update(cfg, options)
+    del cfgs
+
+    cfg.semantic_concept = semantic_concept
+    cfg.word = word
+    cfg.optimized_letter = letter
+    cfg.font = font_name
+    cfg.seed = 0
+    cfg.num_iter = num_steps
+    
+    if ' ' in cfg.word:
+        raise gr.Error(f'should be only one word')
+    cfg.caption = f"a {cfg.semantic_concept}. {cfg.prompt_suffix}"
+    cfg.log_dir = f"output/{cfg.experiment}_{cfg.word}"
+    if cfg.optimized_letter in cfg.word:
+        cfg.optimized_letter = cfg.optimized_letter
+    else:
+        raise gr.Error(f'letter should be in word')
+
+    cfg.letter = f"{cfg.font}_{cfg.optimized_letter}_scaled"
+    cfg.target = f"code/data/init/{cfg.letter}"
+
+    # set experiment dir
+    signature = f"{cfg.letter}_concept_{cfg.semantic_concept}_seed_{cfg.seed}"
+    cfg.experiment_dir = \
+        osp.join(cfg.log_dir, cfg.font, signature)
+    configfile = osp.join(cfg.experiment_dir, 'config.yaml')
+
+    # create experiment dir and save config
+    check_and_create_dir(configfile)
+    with open(osp.join(configfile), 'w') as f:
+        yaml.dump(edict_2_dict(cfg), f)
+
+    if cfg.seed is not None:
+        random.seed(cfg.seed)
+        npr.seed(cfg.seed)
+        torch.manual_seed(cfg.seed)
+        torch.backends.cudnn.benchmark = False
+    else:
+        assert False
+    return cfg
+
+
+def init_shapes(svg_path, trainable: Mapping[str, bool]):
+    svg = f'{svg_path}.svg'
+    canvas_width, canvas_height, shapes_init, shape_groups_init = pydiffvg.svg_to_scene(svg)
+
+    parameters = edict()
+
+    # path points
+    if trainable.point:
+        parameters.point = []
+        for path in shapes_init:
+            path.points.requires_grad = True
+            parameters.point.append(path.points)
+
+    return shapes_init, shape_groups_init, parameters
+
+
+def run_main_ex(semantic_concept, word, letter, font_name, num_steps):
+    return list(next(run_main_app(semantic_concept, word, letter, font_name, num_steps, 1)))
+                        
+def run_main_app(semantic_concept, word, letter, font_name, num_steps, example=0):
+    
+    cfg = set_config(semantic_concept, word, letter, font_name, num_steps)
+
+    pydiffvg.set_use_gpu(torch.cuda.is_available())
+
+    print("preprocessing")
+    preprocess(cfg.font, cfg.word, cfg.optimized_letter, cfg.level_of_cc)
+    filename_init = os.path.join("code/data/init/", f"{cfg.font}_{cfg.word}_scaled.svg").replace(" ", "_")
+    if not example:
+        yield gr.update(value=filename_init,visible=True),gr.update(visible=False),gr.update(visible=False)
+
+    sds_loss = SDSLoss(cfg, device, model)
+
+    h, w = cfg.render_size, cfg.render_size
+
+    data_augs = get_data_augs(cfg.cut_size)
+
+    render = pydiffvg.RenderFunction.apply
+
+    # initialize shape
+    print('initializing shape')
+    shapes, shape_groups, parameters = init_shapes(svg_path=cfg.target, trainable=cfg.trainable)
+
+    scene_args = pydiffvg.RenderFunction.serialize_scene(w, h, shapes, shape_groups)
+    img_init = render(w, h, 2, 2, 0, None, *scene_args)
+    img_init = img_init[:, :, 3:4] * img_init[:, :, :3] + \
+               torch.ones(img_init.shape[0], img_init.shape[1], 3, device=device) * (1 - img_init[:, :, 3:4])
+    img_init = img_init[:, :, :3]
+
+    tone_loss = ToneLoss(cfg)
+    tone_loss.set_image_init(img_init)
+
+    num_iter = cfg.num_iter
+    pg = [{'params': parameters["point"], 'lr': cfg.lr_base["point"]}]
+    optim = torch.optim.Adam(pg, betas=(0.9, 0.9), eps=1e-6)
+
+    conformal_loss = ConformalLoss(parameters, device, cfg.optimized_letter, shape_groups)
+
+    lr_lambda = lambda step: learning_rate_decay(step, cfg.lr.lr_init, cfg.lr.lr_final, num_iter,
+                                                 lr_delay_steps=cfg.lr.lr_delay_steps,
+                                                 lr_delay_mult=cfg.lr.lr_delay_mult) / cfg.lr.lr_init
+
+    scheduler = LambdaLR(optim, lr_lambda=lr_lambda, last_epoch=-1)  # lr.base * lrlambda_f
+
+    print("start training")
+    # training loop
+    t_range = tqdm(range(num_iter))
+    for step in t_range:
+        optim.zero_grad()
+
+        # render image
+        scene_args = pydiffvg.RenderFunction.serialize_scene(w, h, shapes, shape_groups)
+        img = render(w, h, 2, 2, step, None, *scene_args)
+
+        # compose image with white background
+        img = img[:, :, 3:4] * img[:, :, :3] + torch.ones(img.shape[0], img.shape[1], 3, device=device) * (
+                    1 - img[:, :, 3:4])
+        img = img[:, :, :3]
+
+        filename = os.path.join(
+            cfg.experiment_dir, "video-svg", f"iter{step:04d}.svg")
+        check_and_create_dir(filename)
+        save_svg.save_svg(filename, w, h, shapes, shape_groups)
+        if not example:
+            yield gr.update(visible=True),gr.update(value=filename, label=f'iters: {step} / {num_iter}', visible=True),gr.update(visible=False)
+
+        x = img.unsqueeze(0).permute(0, 3, 1, 2)  # HWC -> NCHW
+        x = x.repeat(cfg.batch_size, 1, 1, 1)
+        x_aug = data_augs.forward(x)
+
+        # compute diffusion loss per pixel
+        loss = sds_loss(x_aug)
+
+        tone_loss_res = tone_loss(x, step)
+        loss = loss + tone_loss_res
+
+        loss_angles = conformal_loss()
+        loss_angles = cfg.loss.conformal.angeles_w * loss_angles
+        loss = loss + loss_angles
+
+        loss.backward()
+        optim.step()
+        scheduler.step()
+
+            
+    filename = os.path.join(
+        cfg.experiment_dir, "output-svg", "output.svg")
+    check_and_create_dir(filename)
+    save_svg.save_svg(
+        filename, w, h, shapes, shape_groups)
+
+    combine_word(cfg.word, cfg.optimized_letter, cfg.font, cfg.experiment_dir)
+
+    image = os.path.join(cfg.experiment_dir,f"{cfg.font}_{cfg.word}_{cfg.optimized_letter}.svg")
+    yield gr.update(value=filename_init,visible=True),gr.update(visible=False),gr.update(value=image,visible=True)
+ 
+
+with gr.Blocks() as demo:
+
+    gr.HTML(TITLE)
+    gr.Markdown(DESCRIPTION)
+    
+    with gr.Row():
+        with gr.Column():
+
+            semantic_concept = gr.Text(
+                label='Semantic Concept',
+                max_lines=1,
+                placeholder=
+                'Enter a semantic concept. For example: BUNNY'
+            )
+
+            word = gr.Text(
+                label='Word',
+                max_lines=1,
+                placeholder=
+                'Enter a word. For example: BUNNY'
+            )
+           
+            letter = gr.Text(
+                label='Letter',
+                max_lines=1,
+                placeholder=
+                'Choose a letter in the word to optimize. For example: Y'
+            )
+
+            num_steps = gr.Slider(label='Optimization Iterations',
+                      minimum=0,
+                      maximum=500,
+                      step=10,
+                      value=500)
+            
+            font_name = gr.Text(value=None,visible=False,label="Font Name")
+        
+            
+            def on_select(evt: gr.SelectData):
+                return evt.value
+                
+            font_name = "ArefRuqaa.ttf"
+          
+            run = gr.Button('Generate')
+
+        with gr.Column():
+            result0 = gr.Image(type="filepath", label="Initial Word").style(height=333)
+            result1 = gr.Image(type="filepath", label="Optimization Process").style(height=110)
+            result2 = gr.Image(type="filepath", label="Final Result",visible=False).style(height=333)
+            
+        
+    with gr.Row():
+        # examples
+        examples = [
+            [
+            "BUNNY",
+            "BUNNY",
+            "Y",
+            "KaushanScript-Regular",
+            500
+            ],
+            [
+            "LION",
+            "LION",
+            "O",
+            "Quicksand",
+            500
+            ],
+            [
+            "FROG",
+            "FROG",
+            "G",
+            "IndieFlower-Regular",
+            500
+            ],
+            [
+            "CAT",
+            "CAT",
+            "C",
+            "LuckiestGuy-Regular",
+            500
+            ],
+        ]
+        demo.queue(max_size=10, concurrency_count=2)
+        # gr.Examples(examples=examples,
+        #         inputs=[
+        #             semantic_concept,
+        #             word,
+        #             letter, 
+        #             font_name,
+        #             num_steps
+        #         ],
+        #         outputs=[
+        #             result0,
+        #             result1,
+        #             result2
+        #         ],
+        #         fn=run_main_ex,
+        #         cache_examples=True)
+        
+        
+    # inputs
+    inputs = [
+        semantic_concept,
+        word,
+        letter, 
+        font_name,
+        num_steps
+    ]
+
+    outputs = [
+        result0,
+        result1,
+        result2
+    ]
+    
+    run.click(fn=run_main_app, inputs=inputs, outputs=outputs, queue=True)
+
+
+demo.launch(share=False)

code/bezier.py

@@ -0,0 +1,122 @@
+import numpy as np
+import matplotlib.pyplot as plt
+from scipy.special import binom
+from numpy.linalg import norm
+
+def num_bezier(n_ctrl, degree=3):
+    if type(n_ctrl) == np.ndarray:
+        n_ctrl = len(n_ctrl)
+    return int((n_ctrl - 1) / degree)
+
+def bernstein(n, i):
+    bi = binom(n, i)
+    return lambda t, bi=bi, n=n, i=i: bi * t**i * (1 - t)**(n - i)
+
+def bezier(P, t, d=0):
+    '''Bezier curve of degree len(P)-1. d is the derivative order (0 gives positions)'''
+    n = P.shape[0] - 1
+    if d > 0:
+        Q = np.diff(P, axis=0)*n
+        return bezier(Q, t, d-1)
+    B = np.vstack([bernstein(n, i)(t) for i, p in enumerate(P)])
+    return (P.T @ B).T
+
+def cubic_bezier(P, t):
+    return (1.0-t)**3*P[0] + 3*(1.0-t)**2*t*P[1] + 3*(1.0-t)*t**2*P[2] + t**3*P[3]
+
+def bezier_piecewise(Cp, subd=100, degree=3, d=0):
+    ''' sample a piecewise Bezier curve given a sequence of control points'''
+    num = num_bezier(Cp.shape[0], degree)
+    X = []
+    for i in range(num):
+        P = Cp[i*degree:i*degree+degree+1, :]
+        t = np.linspace(0, 1., subd)[:-1]
+        Y = bezier(P, t, d)
+        X += [Y]
+    X.append(Cp[-1])
+    X = np.vstack(X)
+    return X
+
+def compute_beziers(beziers, subd=100, degree=3):
+    chain = beziers_to_chain(beziers)
+    return bezier_piecewise(chain, subd, degree)
+
+def plot_control_polygon(Cp, degree=3, lw=0.5, linecolor=np.ones(3)*0.1):
+    n_bezier = num_bezier(len(Cp), degree)
+    for i in range(n_bezier):
+        cp = Cp[i*degree:i*degree+degree+1, :]
+        if degree==3:
+            plt.plot(cp[0:2,0], cp[0:2, 1], ':', color=linecolor, linewidth=lw)
+            plt.plot(cp[2:,0], cp[2:,1], ':', color=linecolor, linewidth=lw)
+            plt.plot(cp[:,0], cp[:,1], 'o', color=[0, 0.5, 1.], markersize=4)
+        else:
+            plt.plot(cp[:,0], cp[:,1], ':', color=linecolor, linewidth=lw)
+            plt.plot(cp[:,0], cp[:,1], 'o', color=[0, 0.5, 1.])
+
+
+def chain_to_beziers(chain, degree=3):
+    ''' Convert Bezier chain to list of curve segments (4 control points each)'''
+    num = num_bezier(chain.shape[0], degree)
+    beziers = []
+    for i in range(num):
+        beziers.append(chain[i*degree:i*degree+degree+1,:])
+    return beziers
+
+
+def beziers_to_chain(beziers):
+    ''' Convert list of Bezier curve segments to a piecewise bezier chain (shares vertices)'''
+    n = len(beziers)
+    chain = []
+    for i in range(n):
+        chain.append(list(beziers[i][:-1]))
+    chain.append([beziers[-1][-1]])
+    return np.array(sum(chain, []))
+
+
+def split_cubic(bez, t):
+    p1, p2, p3, p4 = bez
+
+    p12 = (p2 - p1) * t + p1
+    p23 = (p3 - p2) * t + p2
+    p34 = (p4 - p3) * t + p3
+
+    p123 = (p23 - p12) * t + p12
+    p234 = (p34 - p23) * t + p23
+
+    p1234 = (p234 - p123) * t + p123
+
+    return np.array([p1, p12, p123, p1234]), np.array([p1234, p234, p34, p4])
+
+
+def approx_arc_length(bez):
+    c0, c1, c2, c3 = bez
+    v0 = norm(c1-c0)*0.15
+    v1 = norm(-0.558983582205757*c0 + 0.325650248872424*c1 + 0.208983582205757*c2 + 0.024349751127576*c3)
+    v2 = norm(c3-c0+c2-c1)*0.26666666666666666
+    v3 = norm(-0.024349751127576*c0 - 0.208983582205757*c1 - 0.325650248872424*c2 + 0.558983582205757*c3)
+    v4 = norm(c3-c2)*.15
+    return v0 + v1 + v2 + v3 + v4
+
+
+def subdivide_bezier(bez, thresh):
+    stack = [bez]
+    res = []
+    while stack:
+        bez = stack.pop()
+        l = approx_arc_length(bez)
+        if l < thresh:
+            res.append(bez)
+        else:
+            b1, b2 = split_cubic(bez, 0.5)
+            stack += [b2, b1]
+    return res
+
+def subdivide_bezier_chain(C, thresh):
+    beziers = chain_to_beziers(C)
+    res = []
+    for bez in beziers:
+        res += subdivide_bezier(bez, thresh)
+    return beziers_to_chain(res)
+
+
+

code/collage.py

@@ -0,0 +1,41 @@
+import os
+import imageio 
+import numpy as np
+from glob import glob 
+from PIL import Image, ImageSequence
+
+if __name__ == "__main__":
+    
+    path = "/Users/bkhmsi/Desktop/Animal-Words/*.gif"
+    save_path = os.path.join(os.path.dirname(path), "collage.gif")
+
+
+    width, height = 400, 400
+    nx, ny = 5, 5
+    n_frames = 67
+    collage = np.ones((n_frames+10, width*nx, height*ny)).astype(np.uint8)
+
+    filenames = [p for p in glob(path) if os.path.basename(p)[:-4] not in ["palestine", "amin", "collage"]]
+    print(f"> {len(filenames)} Files Found")
+    for file in filenames:
+        print(os.path.basename(file))
+
+    assert nx*ny <= len(filenames)
+
+    for i in range(nx):
+        for j in range(ny):
+            image = Image.open(filenames[i*ny+j])
+            assert image.is_animated
+            idx = 0
+            for frame_idx in range(image.n_frames):
+                image.seek(frame_idx)
+                frame = image.convert('L').copy()
+                if frame_idx == 0 or frame_idx == image.n_frames-1:
+                    for _ in range(5): 
+                        collage[idx, i*width:(i+1)*width,j*height:(j+1)*height] = np.asarray(frame)[100:500, 100:500]
+                        idx += 1
+                else:
+                    collage[idx, i*width:(i+1)*width,j*height:(j+1)*height] = np.asarray(frame)[100:500, 100:500]
+                    idx += 1
+
+    imageio.mimsave(save_path, collage)

code/config.py

@@ -0,0 +1,108 @@
+import argparse
+import os.path as osp
+import yaml
+import random
+from easydict import EasyDict as edict
+import numpy.random as npr
+import torch
+from utils import (
+    edict_2_dict,
+    check_and_create_dir,
+    update)
+import wandb
+import warnings
+warnings.filterwarnings("ignore")
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--config", type=str, default="code/config/base.yaml")
+    parser.add_argument("--experiment", type=str, default="conformal_0.5_dist_pixel_100_kernel201")
+    parser.add_argument("--seed", type=int, default=0)
+    parser.add_argument('--log_dir', metavar='DIR', default="output")
+    parser.add_argument('--font', type=str, default="none", help="font name")
+    parser.add_argument('--semantic_concept', type=str, help="the semantic concept to insert")
+    parser.add_argument('--word', type=str, default="none", help="the text to work on")
+    parser.add_argument('--prompt_suffix', type=str, default="minimal flat 2d vector. lineal color."
+                                                             " trending on artstation")
+    parser.add_argument('--optimized_letter', type=str, default="none", help="the letter in the word to optimize")
+    parser.add_argument('--batch_size', type=int, default=1)
+    parser.add_argument('--use_wandb', type=int, default=0)
+    parser.add_argument('--wandb_user', type=str, default="none")
+
+    cfg = edict()
+    args = parser.parse_args()
+    with open('TOKEN', 'r') as f:
+        setattr(args, 'token', f.read().replace('\n', ''))
+    cfg.config = args.config
+    cfg.experiment = args.experiment
+    cfg.seed = args.seed
+    cfg.font = args.font
+    cfg.semantic_concept = args.semantic_concept
+    cfg.word = cfg.semantic_concept if args.word == "none" else args.word
+    if " " in cfg.word:
+      raise ValueError(f'no spaces are allowed')
+    if "jpeg" in args.semantic_concept:
+        cfg.caption = args.semantic_concept 
+    else:
+        cfg.caption = f"a {args.semantic_concept}. {args.prompt_suffix}"
+        
+    cfg.log_dir = f"{args.log_dir}/{args.experiment}_{cfg.word}"
+    if args.optimized_letter in cfg.word:
+        cfg.optimized_letter = args.optimized_letter
+    else:
+      raise ValueError(f'letter should be in word')
+    cfg.batch_size = args.batch_size
+    cfg.token = args.token
+    cfg.use_wandb = args.use_wandb
+    cfg.wandb_user = args.wandb_user
+    cfg.letter = f"{args.font}_{args.optimized_letter}_scaled"
+    cfg.target = f"code/data/init/{cfg.letter}"
+
+    return cfg
+
+
+def set_config():
+
+    cfg_arg = parse_args()
+    with open(cfg_arg.config, 'r') as f:
+        cfg_full = yaml.load(f, Loader=yaml.FullLoader)
+
+    # recursively traverse parent_config pointers in the config dicts
+    cfg_key = cfg_arg.experiment
+    cfgs = [cfg_arg]
+    while cfg_key:
+        cfgs.append(cfg_full[cfg_key])
+        cfg_key = cfgs[-1].get('parent_config', 'baseline')
+
+    # allowing children configs to override their parents
+    cfg = edict()
+    for options in reversed(cfgs):
+        update(cfg, options)
+    del cfgs
+
+    # set experiment dir
+    signature = f"{cfg.letter}_concept_{cfg.semantic_concept}_seed_{cfg.seed}"
+    cfg.experiment_dir = \
+        osp.join(cfg.log_dir, cfg.font, signature)
+    configfile = osp.join(cfg.experiment_dir, 'config.yaml')
+    print('Config:', cfg)
+
+    # create experiment dir and save config
+    check_and_create_dir(configfile)
+    with open(osp.join(configfile), 'w') as f:
+        yaml.dump(edict_2_dict(cfg), f)
+
+    if cfg.use_wandb:
+        wandb.init(project="Word-As-Image", entity=cfg.wandb_user,
+                   config=cfg, name=f"{signature}", id=wandb.util.generate_id())
+
+    if cfg.seed is not None:
+        random.seed(cfg.seed)
+        npr.seed(cfg.seed)
+        torch.manual_seed(cfg.seed)
+        torch.backends.cudnn.benchmark = False
+    else:
+        assert False
+
+    return cfg

code/config/base.yaml

@@ -0,0 +1,59 @@
+baseline:
+  parent_config: ''
+  save:
+    init: true
+    image: true
+    video: true
+    video_frame_freq: 1
+  trainable:
+    point: true
+  lr_base:
+    point: 1
+  lr:
+    lr_init: 0.002
+    lr_final: 0.0008
+    lr_delay_mult: 0.1
+    lr_delay_steps: 100
+  num_iter: 500
+  render_size: 600
+  cut_size: 512
+  level_of_cc: 0 # 0 - original number of cc / 1 - recommended / 2 - more control points
+  seed: 0
+  diffusion:
+    model: "runwayml/stable-diffusion-v1-5" #"stabilityai/stable-diffusion-2-1"
+    timesteps: 1000
+    guidance_scale: 100
+  loss:
+    use_sds_loss: true
+    tone:
+      use_tone_loss: false
+    conformal:
+      use_conformal_loss: false
+
+conformal_0.5_dist_pixel_100_kernel201:
+  parent_config: baseline
+  level_of_cc: 1
+  loss:
+    tone:
+      use_tone_loss: true
+      dist_loss_weight: 100
+      pixel_dist_kernel_blur: 201
+      pixel_dist_sigma: 30
+    conformal:
+      use_conformal_loss: true
+      angeles_w: 0.5
+
+Animals:
+  parent_config: baseline
+  level_of_cc: 1
+  num_iter: 500
+  loss:
+    tone:
+      use_tone_loss: true
+      dist_loss_weight: 100
+      pixel_dist_kernel_blur: 201
+      pixel_dist_sigma: 30
+    conformal:
+      use_conformal_loss: true
+      angeles_w: 0.5
+

code/data/arabic-fonts/dl-fonts.sh

@@ -0,0 +1,20 @@
+wget https://arbfonts.com//wp-content/fonts/diwany-arabic-fonts//mcs-diwany-jaly-s-u.ttf -O 01.ttf
+wget https://arbfonts.com//wp-content/fonts/new-arabic-fonts//46-Diwani-Bent.ttf -O 02.ttf
+wget https://arbfonts.com//wp-content/fonts/diwany-arabic-fonts//diwany-edited.ttf -O 03.ttf
+wget https://arbfonts.com/wp-content/fonts/diwany-arabic-fonts//arbfonts-samt-7017.ttf -O 04.ttf
+wget https://arbfonts.com//wp-content/fonts/kufi-arabic-fonts//QadasiRegular.ttf -O 05.ttf
+wget https://arbfonts.com//wp-content/fonts/kufi-arabic-fonts//Spirit-Of-Doha-Black.otf -O 06.ttf
+wget https://arbfonts.com//wp-content/fonts/new-arabic-fonts//AlQalam-alavi.ttf -O 07.ttf
+wget https://arbfonts.com//wp-content/fonts/new-arabic-fonts//22-andlso.ttf -O 08.ttf
+wget https://arbfonts.com//wp-content/fonts/new-arabic-fonts//ArefRuqaa-Bold-1.ttf -O 09.ttf 
+wget https://arbfonts.com//wp-content/fonts/new-arabic-fonts//ArefRuqaa-Regular-1.ttf -O 10.ttf
+wget https://arbfonts.com//wp-content/fonts/new-arabic-fonts//117-Barada-Reqa.ttf -O 11.ttf
+wget https://arbfonts.com/wp-content/fonts/diwany-arabic-fonts//arbfonts-diwany-thuluth.ttf -O 12.ttf
+wget https://arbfonts.com//wp-content/fonts/unlimited-free-arabic-fonts//UthmanicHafs1-Ver09_2.otf -O 13.ttf
+wget https://arbfonts.com//wp-content/fonts/new-arabic-fonts//ALAMERAH-Thin.ttf -O 14.ttf
+wget https://arbfonts.com/font_files/horr/unicode/Boahmed%20Alhour.ttf -O 15.ttf
+wget https://arbfonts.com//wp-content/fonts/misc//K-Kamran.ttf -O 16.ttf
+wget https://arbfonts.com//wp-content/fonts/farsi-free-fonts//Jamil-nory.ttf -O 17.ttf
+wget https://arbfonts.com//wp-content/fonts/brands-arasbic-fonts//Mobily.ttf -O 18.ttf
+wget https://arbfonts.com//wp-content/fonts/new-arabic-fonts//QTSManga-Regular-1.ttf -O 19.ttf
+wget https://arbfonts.com//wp-content/fonts/arabic-fonts/new//Al-Jazeera-Arabic-Regular.ttf -O 20.ttf

code/data/arabic-fonts/font_names.txt

@@ -0,0 +1,20 @@
+ديواني جلي
+ديواني مشكل
+ديواني طويل
+ديواني بسيط
+كوفي بسيط
+كوفي منحني
+فارسي بسيط
+مغربي اندلس
+رقعة مدبب
+رقعة بسيط
+رقعة سريع
+ثلث ديواني
+ثلث بسيط
+مربع بسيط
+حر مدبب
+حر بسيط
+حر طويل
+موبايلي
+منجا
+الجزيرة

code/harfbuzz_test.py

@@ -0,0 +1,33 @@
+import vharfbuzz as hv
+
+animal_names = [
+  {"english": "cat", "arabic": "قطة"},
+  {"english": "Lion", "arabic": "أسد"},
+  {"english": "Elephant", "arabic": "فيل"},
+  {"english": "Tiger", "arabic": "نمر"},
+  {"english": "Cheetah", "arabic": "فهد"},
+  {"english": "Monkey", "arabic": "قرد"},
+  {"english": "Dolphin", "arabic": "دلفين"},
+  {"english": "Penguin", "arabic": "بطريق"},
+  {"english": "Kangaroo", "arabic": "كنغر"},
+  {"english": "Fox", "arabic": "ثعلب"},
+  {"english": "Eagle", "arabic": "نسر"},
+  {"english": "Wolf", "arabic": "ذئب"},
+  {"english": "Turtle", "arabic": "سلحفاة"},
+  {"english": "Panda", "arabic": "باندا"},
+  {"english": "Giraffe", "arabic": "زرافة"},
+  {"english": "Bear", "arabic": "دب"},
+  {"english": "Owl", "arabic": "بومة"}
+]
+
+fontpath = './data/fonts/ArefRuqaa.ttf'
+vhb = hv.Vharfbuzz(fontpath)
+
+path_templ = "/Users/bkhmsi/Desktop/Animal-Words/correct/{}.svg"
+
+for animal in animal_names:
+    txt = animal["arabic"]
+    buf = vhb.shape(txt, {"features": {"kern": True, "liga": True}})
+    svg = vhb.buf_to_svg(buf)
+    with open(path_templ.format(animal["english"]), 'w') as fout:
+        fout.write(svg)

code/losses.py

@@ -0,0 +1,198 @@
+import torch.nn as nn
+import torchvision
+from scipy.spatial import Delaunay
+import torch
+import numpy as np
+from torch.nn import functional as nnf
+from easydict import EasyDict
+from shapely.geometry import Point
+from shapely.geometry.polygon import Polygon
+from torchvision import transforms
+from PIL import Image
+from transformers import CLIPProcessor, CLIPModel
+
+from diffusers import StableDiffusionPipeline
+
+class SDSLoss(nn.Module):
+    def __init__(self, cfg, device):
+        super(SDSLoss, self).__init__()
+        self.cfg = cfg
+        self.device = device
+        self.pipe = StableDiffusionPipeline.from_pretrained(cfg.diffusion.model,
+                                                       torch_dtype=torch.float16, use_auth_token=cfg.token)
+        self.pipe = self.pipe.to(self.device)
+
+        self.clip_model = CLIPModel.from_pretrained("openai/clip-vit-large-patch14").to(self.device)
+        self.clip_processor = CLIPProcessor.from_pretrained("openai/clip-vit-large-patch14")
+    
+        # default scheduler: PNDMScheduler(beta_start=0.00085, beta_end=0.012,
+        # beta_schedule="scaled_linear", num_train_timesteps=1000)
+        self.alphas = self.pipe.scheduler.alphas_cumprod.to(self.device)
+        self.sigmas = (1 - self.pipe.scheduler.alphas_cumprod).to(self.device)
+
+        self.text_embeddings = None
+        self.embed_text()
+
+    def embed_text(self):
+        # tokenizer and embed text
+
+        if "jpeg" not in self.cfg.caption:
+          text_input = self.pipe.tokenizer(self.cfg.caption, padding="max_length",
+                                          max_length=self.pipe.tokenizer.model_max_length,
+                                          truncation=True, return_tensors="pt")
+          uncond_input = self.pipe.tokenizer([""], padding="max_length",
+                                          max_length=text_input.input_ids.shape[-1],
+                                          return_tensors="pt")
+          with torch.no_grad():
+              text_embeddings = self.pipe.text_encoder(text_input.input_ids.to(self.device))[0]
+              uncond_embeddings = self.pipe.text_encoder(uncond_input.input_ids.to(self.device))[0]
+        else:
+            print(f"> Reading Image {self.cfg.caption}")
+            with torch.no_grad():
+                image = Image.open(self.cfg.caption)
+                inputs = self.clip_processor(images=image, return_tensors="pt").to(self.device)
+                img_emb = self.clip_model.get_image_features(**inputs)
+            text_embeddings = img_emb
+            uncond_embeddings = img_emb
+
+        print(text_embeddings.size())
+        print(uncond_embeddings.size())
+        self.text_embeddings = torch.cat([uncond_embeddings, text_embeddings])
+        self.text_embeddings = self.text_embeddings.repeat_interleave(self.cfg.batch_size, 0)
+        del self.pipe.tokenizer
+        del self.pipe.text_encoder
+
+
+    def forward(self, x_aug):
+        sds_loss = 0
+
+        # encode rendered image
+        x = x_aug * 2. - 1.
+        with torch.cuda.amp.autocast():
+            init_latent_z = (self.pipe.vae.encode(x).latent_dist.sample())
+        latent_z = 0.18215 * init_latent_z  # scaling_factor * init_latents
+
+        with torch.inference_mode():
+            # sample timesteps
+            timestep = torch.randint(
+                low=50,
+                high=min(950, self.cfg.diffusion.timesteps) - 1,  # avoid highest timestep | diffusion.timesteps=1000
+                size=(latent_z.shape[0],),
+                device=self.device, dtype=torch.long)
+
+            # add noise
+            eps = torch.randn_like(latent_z)
+            # zt = alpha_t * latent_z + sigma_t * eps
+            noised_latent_zt = self.pipe.scheduler.add_noise(latent_z, eps, timestep)
+
+            # denoise
+            z_in = torch.cat([noised_latent_zt] * 2)  # expand latents for classifier free guidance
+            timestep_in = torch.cat([timestep] * 2)
+            with torch.autocast(device_type="cuda", dtype=torch.float16):
+                eps_t_uncond, eps_t = self.pipe.unet(z_in, timestep, encoder_hidden_states=self.text_embeddings).sample.float().chunk(2)
+
+            eps_t = eps_t_uncond + self.cfg.diffusion.guidance_scale * (eps_t - eps_t_uncond)
+
+            # w = alphas[timestep]^0.5 * (1 - alphas[timestep]) = alphas[timestep]^0.5 * sigmas[timestep]
+            grad_z = self.alphas[timestep]**0.5 * self.sigmas[timestep] * (eps_t - eps)
+            assert torch.isfinite(grad_z).all()
+            grad_z = torch.nan_to_num(grad_z.detach().float(), 0.0, 0.0, 0.0)
+
+        sds_loss = grad_z.clone() * latent_z
+        del grad_z
+
+        sds_loss = sds_loss.sum(1).mean()
+        return sds_loss
+
+
+class ToneLoss(nn.Module):
+    def __init__(self, cfg):
+        super(ToneLoss, self).__init__()
+        self.dist_loss_weight = cfg.loss.tone.dist_loss_weight
+        self.im_init = None
+        self.cfg = cfg
+        self.mse_loss = nn.MSELoss()
+        self.blurrer = torchvision.transforms.GaussianBlur(kernel_size=(cfg.loss.tone.pixel_dist_kernel_blur,
+                                                                        cfg.loss.tone.pixel_dist_kernel_blur), sigma=(cfg.loss.tone.pixel_dist_sigma))
+
+    def set_image_init(self, im_init):
+        self.im_init = im_init.permute(2, 0, 1).unsqueeze(0)
+        self.init_blurred = self.blurrer(self.im_init)
+
+
+    def get_scheduler(self, step=None):
+        if step is not None:
+            return self.dist_loss_weight * np.exp(-(1/5)*((step-300)/(20)) ** 2)
+        else:
+            return self.dist_loss_weight
+
+    def forward(self, cur_raster, step=None):
+        blurred_cur = self.blurrer(cur_raster)
+        return self.mse_loss(self.init_blurred.detach(), blurred_cur) * self.get_scheduler(step)
+            
+
+class ConformalLoss:
+    def __init__(self, parameters: EasyDict, device: torch.device, target_letter: str, shape_groups):
+        self.parameters = parameters
+        self.target_letter = target_letter
+        self.shape_groups = shape_groups
+        self.faces = self.init_faces(device)
+        self.faces_roll_a = [torch.roll(self.faces[i], 1, 1) for i in range(len(self.faces))]
+
+        with torch.no_grad():
+            self.angles = []
+            self.reset()
+
+
+    def get_angles(self, points: torch.Tensor) -> torch.Tensor:
+        angles_ = []
+        for i in range(len(self.faces)):
+            triangles = points[self.faces[i]]
+            triangles_roll_a = points[self.faces_roll_a[i]]
+            edges = triangles_roll_a - triangles
+            length = edges.norm(dim=-1)
+            edges = edges / (length + 1e-1)[:, :, None]
+            edges_roll = torch.roll(edges, 1, 1)
+            cosine = torch.einsum('ned,ned->ne', edges, edges_roll)
+            angles = torch.arccos(cosine)
+            angles_.append(angles)
+        return angles_
+    
+    def get_letter_inds(self, letter_to_insert):
+        for group, l in zip(self.shape_groups, self.target_letter):
+            if l == letter_to_insert:
+                letter_inds = group.shape_ids
+                return letter_inds[0], letter_inds[-1], len(letter_inds)
+
+    def reset(self):
+        points = torch.cat([point.clone().detach() for point in self.parameters.point])
+        self.angles = self.get_angles(points)
+
+    def init_faces(self, device: torch.device) -> torch.tensor:
+        faces_ = []
+        for j, c in enumerate(self.target_letter):
+            points_np = [self.parameters.point[i].clone().detach().cpu().numpy() for i in range(len(self.parameters.point))]
+            start_ind, end_ind, shapes_per_letter = self.get_letter_inds(c)
+            print(c, start_ind, end_ind)
+            holes = []
+            if shapes_per_letter > 1:
+                holes = points_np[start_ind+1:end_ind]
+            poly = Polygon(points_np[start_ind], holes=holes)
+            poly = poly.buffer(0)
+            points_np = np.concatenate(points_np)
+            faces = Delaunay(points_np).simplices
+            is_intersect = np.array([poly.contains(Point(points_np[face].mean(0))) for face in faces], dtype=np.bool_)
+            faces_.append(torch.from_numpy(faces[is_intersect]).to(device, dtype=torch.int64))
+        return faces_
+
+    def __call__(self) -> torch.Tensor:
+        loss_angles = 0
+        points = torch.cat(self.parameters.point)
+        angles = self.get_angles(points)
+        for i in range(len(self.faces)):
+            loss_angles += (nnf.mse_loss(angles[i], self.angles[i]))
+        return loss_angles
+
+
+
+

code/main.py

@@ -0,0 +1,184 @@
+from typing import Mapping
+import os
+from tqdm import tqdm
+from easydict import EasyDict as edict
+import matplotlib.pyplot as plt
+import torch
+from torch.optim.lr_scheduler import LambdaLR
+import pydiffvg
+import save_svg
+from losses import SDSLoss, ToneLoss, ConformalLoss
+from config import set_config
+from utils import (
+    check_and_create_dir,
+    get_data_augs,
+    save_image,
+    preprocess,
+    learning_rate_decay,
+    combine_word,
+    create_video)
+import wandb
+import warnings
+warnings.filterwarnings("ignore")
+
+pydiffvg.set_print_timing(False)
+gamma = 1.0
+
+
+def init_shapes(svg_path, trainable: Mapping[str, bool]):
+
+    svg = f'{svg_path}.svg'
+    canvas_width, canvas_height, shapes_init, shape_groups_init = pydiffvg.svg_to_scene(svg)
+
+    parameters = edict()
+
+    # path points
+    if trainable.point:
+        parameters.point = []
+        for path in shapes_init:
+            path.points.requires_grad = True
+            parameters.point.append(path.points)
+
+    return shapes_init, shape_groups_init, parameters
+
+
+if __name__ == "__main__":
+    
+    cfg = set_config()
+
+    # use GPU if available
+    pydiffvg.set_use_gpu(torch.cuda.is_available())
+    device = pydiffvg.get_device()
+
+    # cfg.word = cfg.word[::-1]
+
+    print("preprocessing")
+    preprocess(cfg.font, cfg.word, cfg.optimized_letter, cfg.level_of_cc)
+
+    if cfg.loss.use_sds_loss:
+        sds_loss = SDSLoss(cfg, device)
+
+    h, w = cfg.render_size, cfg.render_size
+
+    data_augs = get_data_augs(cfg.cut_size)
+
+    render = pydiffvg.RenderFunction.apply
+
+    # initialize shape
+    print('initializing shape')
+    shapes, shape_groups, parameters = init_shapes(svg_path=cfg.target, trainable=cfg.trainable)
+
+    scene_args = pydiffvg.RenderFunction.serialize_scene(w, h, shapes, shape_groups)
+    img_init = render(w, h, 2, 2, 0, None, *scene_args)
+    img_init = img_init[:, :, 3:4] * img_init[:, :, :3] + \
+               torch.ones(img_init.shape[0], img_init.shape[1], 3, device=device) * (1 - img_init[:, :, 3:4])
+    img_init = img_init[:, :, :3]
+    if cfg.use_wandb:
+        plt.imshow(img_init.detach().cpu())
+        wandb.log({"init": wandb.Image(plt)}, step=0)
+        plt.close()
+
+    if cfg.loss.tone.use_tone_loss:
+        tone_loss = ToneLoss(cfg)
+        tone_loss.set_image_init(img_init)
+
+    if cfg.save.init:
+        print('saving init')
+        filename = os.path.join(
+            cfg.experiment_dir, "svg-init", "init.svg")
+        check_and_create_dir(filename)
+        save_svg.save_svg(filename, w, h, shapes, shape_groups)
+
+    num_iter = cfg.num_iter
+    pg = [{'params': parameters["point"], 'lr': cfg.lr_base["point"]}]
+    optim = torch.optim.Adam(pg, betas=(0.9, 0.9), eps=1e-6)
+
+    if cfg.loss.conformal.use_conformal_loss:
+        conformal_loss = ConformalLoss(parameters, device, cfg.optimized_letter, shape_groups)
+
+    lr_lambda = lambda step: learning_rate_decay(step, cfg.lr.lr_init, cfg.lr.lr_final, num_iter,
+                                                 lr_delay_steps=cfg.lr.lr_delay_steps,
+                                                 lr_delay_mult=cfg.lr.lr_delay_mult) / cfg.lr.lr_init
+
+    scheduler = LambdaLR(optim, lr_lambda=lr_lambda, last_epoch=-1)  # lr.base * lrlambda_f
+
+    print("start training")
+    # training loop
+    t_range = tqdm(range(num_iter))
+    for step in t_range:
+        if cfg.use_wandb:
+            wandb.log({"learning_rate": optim.param_groups[0]['lr']}, step=step)
+        optim.zero_grad()
+
+        # render image
+        scene_args = pydiffvg.RenderFunction.serialize_scene(w, h, shapes, shape_groups)
+        img = render(w, h, 2, 2, step, None, *scene_args)
+
+        # compose image with white background
+        img = img[:, :, 3:4] * img[:, :, :3] + torch.ones(img.shape[0], img.shape[1], 3, device=device) * (1 - img[:, :, 3:4])
+        img = img[:, :, :3]
+
+        if cfg.save.video and (step % cfg.save.video_frame_freq == 0 or step == num_iter - 1):
+            save_image(img, os.path.join(cfg.experiment_dir, "video-png", f"iter{step:04d}.png"), gamma)
+            filename = os.path.join(
+                cfg.experiment_dir, "video-svg", f"iter{step:04d}.svg")
+            check_and_create_dir(filename)
+            save_svg.save_svg(
+                filename, w, h, shapes, shape_groups)
+            if cfg.use_wandb:
+                plt.imshow(img.detach().cpu())
+                wandb.log({"img": wandb.Image(plt)}, step=step)
+                plt.close()
+
+        x = img.unsqueeze(0).permute(0, 3, 1, 2)  # HWC -> NCHW
+        x = x.repeat(cfg.batch_size, 1, 1, 1)
+        x_aug = data_augs.forward(x)
+
+        # compute diffusion loss per pixel
+        loss = sds_loss(x_aug)
+        if cfg.use_wandb:
+            wandb.log({"sds_loss": loss.item()}, step=step)
+
+        if cfg.loss.tone.use_tone_loss:
+            tone_loss_res = tone_loss(x, step)
+            if cfg.use_wandb:
+                wandb.log({"dist_loss": tone_loss_res}, step=step)
+            loss = loss + tone_loss_res
+
+        if cfg.loss.conformal.use_conformal_loss:
+            loss_angles = conformal_loss()
+            loss_angles = cfg.loss.conformal.angeles_w * loss_angles
+            if cfg.use_wandb:
+                wandb.log({"loss_angles": loss_angles}, step=step)
+            loss = loss + loss_angles
+
+        t_range.set_postfix({'loss': loss.item()})
+        loss.backward()
+        optim.step()
+        scheduler.step()
+
+    filename = os.path.join(
+        cfg.experiment_dir, "output-svg", "output.svg")
+    check_and_create_dir(filename)
+    save_svg.save_svg(
+        filename, w, h, shapes, shape_groups)
+
+    combine_word(cfg.word, cfg.optimized_letter, cfg.font, cfg.experiment_dir)
+
+    if cfg.save.image:
+        filename = os.path.join(
+            cfg.experiment_dir, "output-png", "output.png")
+        check_and_create_dir(filename)
+        imshow = img.detach().cpu()
+        pydiffvg.imwrite(imshow, filename, gamma=gamma)
+        if cfg.use_wandb:
+            plt.imshow(img.detach().cpu())
+            wandb.log({"img": wandb.Image(plt)}, step=step)
+            plt.close()
+
+    if cfg.save.video:
+        print("saving video")
+        create_video(cfg.num_iter, cfg.experiment_dir, cfg.save.video_frame_freq)
+
+    if cfg.use_wandb:
+        wandb.finish()

code/save_svg.py

@@ -0,0 +1,155 @@
+import torch
+import pydiffvg
+import xml.etree.ElementTree as etree
+from xml.dom import minidom
+def prettify(elem):
+    """Return a pretty-printed XML string for the Element.
+    """
+    rough_string = etree.tostring(elem, 'utf-8')
+    reparsed = minidom.parseString(rough_string)
+    return reparsed.toprettyxml(indent="  ")
+def save_svg(filename, width, height, shapes, shape_groups, use_gamma = False, background=None):
+    root = etree.Element('svg')
+    root.set('version', '1.1')
+    root.set('xmlns', 'http://www.w3.org/2000/svg')
+    root.set('width', str(width))
+    root.set('height', str(height))
+    if background is not None:
+        print(f"setting background to {background}")
+        root.set('style', str(background))
+    defs = etree.SubElement(root, 'defs')
+    g = etree.SubElement(root, 'g')
+    if use_gamma:
+        f = etree.SubElement(defs, 'filter')
+        f.set('id', 'gamma')
+        f.set('x', '0')
+        f.set('y', '0')
+        f.set('width', '100%')
+        f.set('height', '100%')
+        gamma = etree.SubElement(f, 'feComponentTransfer')
+        gamma.set('color-interpolation-filters', 'sRGB')
+        feFuncR = etree.SubElement(gamma, 'feFuncR')
+        feFuncR.set('type', 'gamma')
+        feFuncR.set('amplitude', str(1))
+        feFuncR.set('exponent', str(1/2.2))
+        feFuncG = etree.SubElement(gamma, 'feFuncG')
+        feFuncG.set('type', 'gamma')
+        feFuncG.set('amplitude', str(1))
+        feFuncG.set('exponent', str(1/2.2))
+        feFuncB = etree.SubElement(gamma, 'feFuncB')
+        feFuncB.set('type', 'gamma')
+        feFuncB.set('amplitude', str(1))
+        feFuncB.set('exponent', str(1/2.2))
+        feFuncA = etree.SubElement(gamma, 'feFuncA')
+        feFuncA.set('type', 'gamma')
+        feFuncA.set('amplitude', str(1))
+        feFuncA.set('exponent', str(1/2.2))
+        g.set('style', 'filter:url(#gamma)')
+    # Store color
+    for i, shape_group in enumerate(shape_groups):
+        def add_color(shape_color, name):
+            if isinstance(shape_color, pydiffvg.LinearGradient):
+                lg = shape_color
+                color = etree.SubElement(defs, 'linearGradient')
+                color.set('id', name)
+                color.set('x1', str(lg.begin[0].item()/width))
+                color.set('y1', str(lg.begin[1].item()/height))
+                color.set('x2', str(lg.end[0].item()/width))
+                color.set('y2', str(lg.end[1].item()/height))
+                offsets = lg.offsets.data.cpu().numpy()
+                stop_colors = lg.stop_colors.data.cpu().numpy()
+                for j in range(offsets.shape[0]):
+                    stop = etree.SubElement(color, 'stop')
+                    stop.set('offset', str(offsets[j]))
+                    c = lg.stop_colors[j, :]
+                    stop.set('stop-color', 'rgb({}, {}, {})'.format(\
+                        int(255 * c[0]), int(255 * c[1]), int(255 * c[2])))
+                    stop.set('stop-opacity', '{}'.format(c[3]))
+            if isinstance(shape_color, pydiffvg.RadialGradient):
+                lg = shape_color
+                color = etree.SubElement(defs, 'radialGradient')
+                color.set('id', name)
+                color.set('cx', str(lg.center[0].item()/width))
+                color.set('cy', str(lg.center[1].item()/height))
+                # this only support width=height
+                color.set('r', str(lg.radius[0].item()/width))
+                offsets = lg.offsets.data.cpu().numpy()
+                stop_colors = lg.stop_colors.data.cpu().numpy()
+                for j in range(offsets.shape[0]):
+                    stop = etree.SubElement(color, 'stop')
+                    stop.set('offset', str(offsets[j]))
+                    c = lg.stop_colors[j, :]
+                    stop.set('stop-color', 'rgb({}, {}, {})'.format(\
+                        int(255 * c[0]), int(255 * c[1]), int(255 * c[2])))
+                    stop.set('stop-opacity', '{}'.format(c[3]))
+        if shape_group.fill_color is not None:
+            add_color(shape_group.fill_color, 'shape_{}_fill'.format(i))
+        if shape_group.stroke_color is not None:
+            add_color(shape_group.stroke_color, 'shape_{}_stroke'.format(i))
+    for i, shape_group in enumerate(shape_groups):
+        # shape = shapes[shape_group.shape_ids[0]]
+        for j,id in enumerate(shape_group.shape_ids):
+            shape = shapes[id]
+            if isinstance(shape, pydiffvg.Path):
+                if j == 0:
+                    shape_node = etree.SubElement(g, 'path')
+                    path_str = ''
+                # shape_node = etree.SubElement(g, 'path')
+                num_segments = shape.num_control_points.shape[0]
+                num_control_points = shape.num_control_points.data.cpu().numpy()
+                points = shape.points.data.cpu().numpy()
+                num_points = shape.points.shape[0]
+                path_str += 'M {} {}'.format(points[0, 0], points[0, 1])
+                point_id = 1
+                for j in range(0, num_segments):
+                    if num_control_points[j] == 0:
+                        p = point_id % num_points
+                        path_str += ' L {} {}'.format(\
+                                points[p, 0], points[p, 1])
+                        point_id += 1
+                    elif num_control_points[j] == 1:
+                        p1 = (point_id + 1) % num_points
+                        path_str += ' Q {} {} {} {}'.format(\
+                                points[point_id, 0], points[point_id, 1],
+                                points[p1, 0], points[p1, 1])
+                        point_id += 2
+                    elif num_control_points[j] == 2:
+                        p2 = (point_id + 2) % num_points
+                        path_str += ' C {} {} {} {} {} {}'.format(\
+                                points[point_id, 0], points[point_id, 1],
+                                points[point_id + 1, 0], points[point_id + 1, 1],
+                                points[p2, 0], points[p2, 1])
+                        point_id += 3
+            else:
+                assert(False)
+            # shape_node.set('stroke-width', str(2 * shape.stroke_width.data.cpu().item()))
+            shape_node.set('stroke-width', str(0)) # no strokes
+            if shape_group.fill_color is not None:
+                if isinstance(shape_group.fill_color, pydiffvg.LinearGradient):
+                    shape_node.set('fill', 'url(#shape_{}_fill)'.format(i))
+                elif isinstance(shape_group.fill_color, pydiffvg.RadialGradient):
+                    shape_node.set('fill', 'url(#shape_{}_fill)'.format(i))
+                else:
+                    c = shape_group.fill_color.data.cpu().numpy()
+                    shape_node.set('fill', 'rgb({}, {}, {})'.format(\
+                        int(255 * c[0]), int(255 * c[1]), int(255 * c[2])))
+                    shape_node.set('opacity', str(c[3]))
+            else:
+                shape_node.set('fill', 'none')
+            if shape_group.stroke_color is not None:
+                if isinstance(shape_group.stroke_color, pydiffvg.LinearGradient):
+                    shape_node.set('stroke', 'url(#shape_{}_stroke)'.format(i))
+                elif isinstance(shape_group.stroke_color, pydiffvg.LinearGradient):
+                    shape_node.set('stroke', 'url(#shape_{}_stroke)'.format(i))
+                else:
+                    c = shape_group.stroke_color.data.cpu().numpy()
+                    shape_node.set('stroke', 'rgb({}, {}, {})'.format(\
+                        int(255 * c[0]), int(255 * c[1]), int(255 * c[2])))
+                    shape_node.set('stroke-opacity', str(c[3]))
+                shape_node.set('stroke-linecap', 'round')
+                shape_node.set('stroke-linejoin', 'round')
+
+        shape_node.set('d', path_str)
+
+    with open(filename, "w") as f:
+        f.write(prettify(root))

code/ttf.py

@@ -0,0 +1,409 @@
+from importlib import reload
+import os
+import numpy as np
+import bezier
+import freetype as ft
+import pydiffvg
+import torch
+import save_svg
+import vharfbuzz as hb
+from svgpathtools import svgstr2paths
+import xml.etree.ElementTree as ET
+
+
+device = torch.device("cuda" if (
+        torch.cuda.is_available() and torch.cuda.device_count() > 0) else "cpu")
+
+reload(bezier)
+
+def fix_single_svg(svg_path, all_word=False):
+    target_h_letter = 360
+    target_canvas_width, target_canvas_height = 600, 600
+
+    canvas_width, canvas_height, shapes, shape_groups = pydiffvg.svg_to_scene(svg_path)
+
+    letter_h = canvas_height
+    letter_w = canvas_width
+
+    if all_word:
+        if letter_w > letter_h:
+            scale_canvas_w = target_h_letter / letter_w
+            hsize = int(letter_h * scale_canvas_w)
+            scale_canvas_h = hsize / letter_h
+        else:
+            scale_canvas_h = target_h_letter / letter_h
+            wsize = int(letter_w * scale_canvas_h)
+            scale_canvas_w = wsize / letter_w
+    else:
+        scale_canvas_h = target_h_letter / letter_h
+        wsize = int(letter_w * scale_canvas_h)
+        scale_canvas_w = wsize / letter_w
+
+    for num, p in enumerate(shapes):
+        p.points[:, 0] = p.points[:, 0] * scale_canvas_w
+        p.points[:, 1] = p.points[:, 1] * scale_canvas_h + target_h_letter
+        p.points[:, 1] = -p.points[:, 1]
+        # p.points[:, 0] = -p.points[:, 0]
+
+    w_min, w_max = min([torch.min(p.points[:, 0]) for p in shapes]), max([torch.max(p.points[:, 0]) for p in shapes])
+    h_min, h_max = min([torch.min(p.points[:, 1]) for p in shapes]), max([torch.max(p.points[:, 1]) for p in shapes])
+
+    for num, p in enumerate(shapes):
+        p.points[:, 0] = p.points[:, 0] + target_canvas_width/2 - int(w_min + (w_max - w_min) / 2)
+        p.points[:, 1] = p.points[:, 1] + target_canvas_height/2 - int(h_min + (h_max - h_min) / 2)
+
+    output_path = f"{svg_path[:-4]}_scaled.svg"
+    save_svg.save_svg(output_path, target_canvas_width, target_canvas_height, shapes, shape_groups)
+
+def normalize_letter_size(dest_path, font, txt, chars):
+    fontname = os.path.splitext(os.path.basename(font))[0]
+    # for i, c in enumerate(chars):
+    #     fname = f"{dest_path}/{fontname}_{c}.svg"
+    #     fname = fname.replace(" ", "_")
+    #     fix_single_svg(fname)
+
+    fname = f"{dest_path}/{fontname}_{txt}.svg"
+    fname = fname.replace(" ", "_")
+    fix_single_svg(fname, all_word=True)
+
+
+def glyph_to_cubics(face, x=0, y=0):
+    ''' Convert current font face glyph to cubic beziers'''
+
+    def linear_to_cubic(Q):
+        a, b = Q
+        return [a + (b - a) * t for t in np.linspace(0, 1, 4)]
+
+    def quadratic_to_cubic(Q):
+        return [Q[0],
+                Q[0] + (2 / 3) * (Q[1] - Q[0]),
+                Q[2] + (2 / 3) * (Q[1] - Q[2]),
+                Q[2]]
+
+    beziers = []
+    pt = lambda p: np.array([x + p.x, - p.y - y])  # Flipping here since freetype has y-up
+    last = lambda: beziers[-1][-1]
+
+    def move_to(a, beziers):
+        beziers.append([pt(a)])
+
+    def line_to(a, beziers):
+        Q = linear_to_cubic([last(), pt(a)])
+        beziers[-1] += Q[1:]
+
+    def conic_to(a, b, beziers):
+        Q = quadratic_to_cubic([last(), pt(a), pt(b)])
+        beziers[-1] += Q[1:]
+
+    def cubic_to(a, b, c, beziers):
+        beziers[-1] += [pt(a), pt(b), pt(c)]
+
+    face.glyph.outline.decompose(beziers, move_to=move_to, line_to=line_to, conic_to=conic_to, cubic_to=cubic_to)
+    beziers = [np.array(C).astype(float) for C in beziers]
+    return beziers
+
+# def handle_ligature(glyph_infos, glyph_positions):
+#     combined_advance = sum(pos.x_advance for pos in glyph_positions)
+#     first_x_offset = glyph_positions[0].x_offset
+
+#     combined_advance = x_adv_1 + x_adv_2
+
+
+
+
+#     # Adjust the x_offset values based on the difference between the first glyph's x_offset and the combined_advance
+#     for pos in glyph_positions:
+#         pos.x_offset += combined_advance - pos.x_advance - first_x_offset
+
+#     # Render the ligature using the adjusted glyph positions
+#     render_glyphs(glyph_infos, glyph_positions)
+
+
+def font_string_to_beziers(font, txt, size=30, spacing=1.0, merge=True, target_control=None):
+    ''' Load a font and convert the outlines for a given string to cubic bezier curves,
+        if merge is True, simply return a list of all bezier curves,
+        otherwise return a list of lists with the bezier curves for each glyph'''
+    print(font)
+    
+    vhb = hb.Vharfbuzz(font)
+    buf = vhb.shape(txt, {"features": {"kern": True, "liga": True}})
+
+    buf.guess_segment_properties()
+
+    glyph_infos = buf.glyph_infos
+    glyph_positions = buf.glyph_positions
+    glyph_count = {glyph_infos[i].cluster: 0 for i in range(len(glyph_infos))}
+    
+    svg = vhb.buf_to_svg(buf)
+    paths, attributes = svgstr2paths(svg)
+    
+    face = ft.Face(font)
+    face.set_char_size(64 * size)
+    pindex = -1
+
+    x, y = 0, 0
+    beziers, chars = [], []
+
+    for path_idx, path in enumerate(paths):
+        segment_vals = []
+        print("="*20 + str(path_idx) + "="*20)
+        for segment in path:
+            segment_type = segment.__class__.__name__
+            t_values = np.linspace(0, 1, 10)
+            points = [segment.point(t) for t in t_values]
+            for pt in points:
+                segment_vals += [[pt.real, -pt.imag]]
+
+            # points = [bezier.point(t) for t in t_values]
+
+            if segment_type == 'Line':
+                # Line segment
+                start = segment.start
+                end = segment.end
+                print(f"Line: ({start.real}, {start.imag}) to ({end.real}, {end.imag})")
+            
+            elif segment_type == 'QuadraticBezier':
+                # Quadratic Bézier segment
+                start = segment.start
+                control = segment.control
+                end = segment.end
+                print(f"Quadratic Bézier: ({start.real}, {start.imag}) to ({end.real}, {end.imag}) with control point ({control.real}, {control.imag})")
+            
+            elif segment_type == 'CubicBezier':
+                # Cubic Bézier segment
+                start = segment.start
+                control1 = segment.control1
+                control2 = segment.control2
+                end = segment.end
+                print(f"Cubic Bézier: ({start.real}, {start.imag}) to ({end.real}, {end.imag}) with control points ({control1.real}, {control1.imag}) and ({control2.real}, {control2.imag})")
+            
+            else:
+                # Other segment types (Arc, Close)
+                print(f"Segment type: {segment_type}")
+
+        beziers += [[np.array(segment_vals)]]
+
+    beziers_2 = []
+    glyph_infos = glyph_infos[::-1]
+    glyph_positions = glyph_positions[::-1]
+    for i, (info, pos) in enumerate(zip(glyph_infos, glyph_positions)):
+        index = info.cluster
+        c = f"{txt[index]}_{glyph_count[index]}"
+        chars += [c]
+        glyph_count[index] += 1
+        glyph_index = info.codepoint
+        face.load_glyph(glyph_index, flags=ft.FT_LOAD_DEFAULT | ft.FT_LOAD_NO_BITMAP)
+        # face.load_char(c, ft.FT_LOAD_DEFAULT | ft.FT_LOAD_NO_BITMAP)
+
+        findex = -1 
+        if i+1 < len(glyph_infos):
+            findex = glyph_infos[i+1].cluster
+            foffset = (glyph_positions[i+1].x_offset, glyph_positions[i+1].y_offset)
+            fadvance = (glyph_positions[i+1].x_advance, glyph_positions[i+1].y_advance)
+        
+        # bez = glyph_to_cubics(face, x+pos.x_offset+pos.x_advance, y+pos.y_offset+pos.y_advance)
+        # if findex != index:
+        #     x += pos.x_offset
+        #     y += pos.y_offset
+        # else:
+        #     x += pos.x_offset
+        #     y += pos.y_offset
+
+
+        bez = glyph_to_cubics(face, x, y)
+            
+
+        # Check number of control points if desired
+        if target_control is not None:
+            if c in target_control.keys():
+                nctrl = np.sum([len(C) for C in bez])
+                while nctrl < target_control[c]:
+                    longest = np.max(
+                        sum([[bezier.approx_arc_length(b) for b in bezier.chain_to_beziers(C)] for C in bez], []))
+                    thresh = longest * 0.5
+                    bez = [bezier.subdivide_bezier_chain(C, thresh) for C in bez]
+                    nctrl = np.sum([len(C) for C in bez])
+                    print(nctrl)
+
+        if merge:
+            beziers_2 += bez
+        else:
+            beziers_2.append(bez)
+
+        # kerning = face.get_kerning(index, findex)
+        # x += (slot.advance.x + kerning.x) * spacing
+        # previous = txt[index]
+    
+        # print(f"C: {txt[index]}/{index} | X: {x+pos.x_offset}| Y: {y+pos.y_offset}")
+        print(f"C: {txt[index]}/{index} | X: {x}: {pos.x_advance}/{pos.x_offset} | Y: {y}: {pos.y_advance}/{pos.y_offset}")
+        
+        # if findex != index:
+        x -= pos.x_advance
+        # y += pos.y_advance + pos.y_offset
+
+        pindex = index
+            
+    return beziers_2, chars
+
+
+def bezier_chain_to_commands(C, closed=True):
+    curves = bezier.chain_to_beziers(C)
+    cmds = 'M %f %f ' % (C[0][0], C[0][1])
+    n = len(curves)
+    for i, bez in enumerate(curves):
+        if i == n - 1 and closed:
+            cmds += 'C %f %f %f %f %f %fz ' % (*bez[1], *bez[2], *bez[3])
+        else:
+            cmds += 'C %f %f %f %f %f %f ' % (*bez[1], *bez[2], *bez[3])
+    return cmds
+
+
+def count_cp(file_name, font_name):
+    canvas_width, canvas_height, shapes, shape_groups = pydiffvg.svg_to_scene(file_name)
+    p_counter = 0
+    for path in shapes:
+        p_counter += path.points.shape[0]
+    print(f"TOTAL CP:   [{p_counter}]")
+    return p_counter
+
+
+def write_letter_svg(c, header, fontname, beziers, subdivision_thresh, dest_path):
+    cmds = ''
+    svg = header
+
+    path = '<g><path d="'
+    for C in beziers:
+        if subdivision_thresh is not None:
+            print('subd')
+            C = bezier.subdivide_bezier_chain(C, subdivision_thresh)
+        cmds += bezier_chain_to_commands(C, True)
+    path += cmds + '"/>\n'
+    svg += path + '</g></svg>\n'
+
+    fname = f"{dest_path}/{fontname}_{c}.svg"
+    fname = fname.replace(" ", "_")
+    f = open(fname, 'w')
+    f.write(svg)
+    f.close()
+    return fname, path
+
+def write_letter_svg_hb(vhb, c, dest_path, fontname):
+    buf = vhb.shape(c, {"features": {"kern": True, "liga": True}})    
+    svg = vhb.buf_to_svg(buf)
+
+    fname = f"{dest_path}/{fontname}_{c}.svg"
+    fname = fname.replace(" ", "_")
+    f = open(fname, 'w')
+    f.write(svg)
+    f.close()
+    return fname
+
+def font_string_to_svgs(dest_path, font, txt, size=30, spacing=1.0, target_control=None, subdivision_thresh=None):
+
+    fontname = os.path.splitext(os.path.basename(font))[0]
+    glyph_beziers, chars = font_string_to_beziers(font, txt, size, spacing, merge=False, target_control=target_control)
+    if not os.path.isdir(dest_path):
+        os.mkdir(dest_path)
+    # Compute boundig box
+    points = np.vstack(sum(glyph_beziers, []))
+    lt = np.min(points, axis=0)
+    rb = np.max(points, axis=0)
+    size = rb - lt
+
+    sizestr = 'width="%.1f" height="%.1f"' % (size[0], size[1])
+    boxstr = ' viewBox="%.1f %.1f %.1f %.1f"' % (lt[0], lt[1], size[0], size[1])
+    header = '''<?xml version="1.0" encoding="utf-8"?>
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:ev="http://www.w3.org/2001/xml-events" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" baseProfile="full" '''
+    header += sizestr
+    header += boxstr
+    header += '>\n<defs/>\n'
+
+    svg_all = header
+
+    print(f"Len Glyph Bezier: {len(glyph_beziers)} | Chars: {len(chars)}")
+    for i, (c, beziers) in enumerate(zip(chars, glyph_beziers)):
+        print(f"==== {c} ====")
+        fname, path = write_letter_svg(c, header, fontname, beziers, subdivision_thresh, dest_path)
+
+        num_cp = count_cp(fname, fontname)
+        print(num_cp)
+        print(font, c)
+        # Add to global svg
+        svg_all += path + '</g>\n'
+
+    vhb = hb.Vharfbuzz(font)
+    buf = vhb.shape(txt, {"features": {"kern": True, "liga": True}})    
+    svg = vhb.buf_to_svg(buf)
+
+    # Save global svg
+    svg_all += '</svg>\n'
+    fname = f"{dest_path}/{fontname}_{txt}.svg"
+    fname = fname.replace(" ", "_")
+    f = open(fname, 'w')
+    f.write(svg)
+    f.close()
+    return chars
+
+def font_string_to_svgs_hb(dest_path, font, txt, size=30, spacing=1.0, target_control=None, subdivision_thresh=None):
+
+    fontname = os.path.splitext(os.path.basename(font))[0]
+    if not os.path.isdir(dest_path):
+        os.mkdir(dest_path)
+
+    vhb = hb.Vharfbuzz(font)
+    buf = vhb.shape(txt, {"features": {"kern": True, "liga": True}})
+    buf.guess_segment_properties()
+
+    buf = vhb.shape(txt, {"features": {"kern": True, "liga": True}})    
+    svg = vhb.buf_to_svg(buf)
+
+    # Save global svg
+    fname = f"{dest_path}/{fontname}_{txt}.svg"
+    fname = fname.replace(" ", "_")
+    f = open(fname, 'w')
+    f.write(svg)
+    f.close()
+    return None
+
+if __name__ == '__main__':
+
+    fonts = ["KaushanScript-Regular"]
+    level_of_cc = 1
+
+    if level_of_cc == 0:
+        target_cp = None
+
+    else:
+        target_cp = {"A": 120, "B": 120, "C": 100, "D": 100,
+                     "E": 120, "F": 120, "G": 120, "H": 120,
+                     "I": 35, "J": 80, "K": 100, "L": 80,
+                     "M": 100, "N": 100, "O": 100, "P": 120,
+                     "Q": 120, "R": 130, "S": 110, "T": 90,
+                     "U": 100, "V": 100, "W": 100, "X": 130,
+                     "Y": 120, "Z": 120,
+                     "a": 120, "b": 120, "c": 100, "d": 100,
+                     "e": 120, "f": 120, "g": 120, "h": 120,
+                     "i": 35, "j": 80, "k": 100, "l": 80,
+                     "m": 100, "n": 100, "o": 100, "p": 120,
+                     "q": 120, "r": 130, "s": 110, "t": 90,
+                     "u": 100, "v": 100, "w": 100, "x": 130,
+                     "y": 120, "z": 120
+                     }
+
+        target_cp = {k: v * level_of_cc for k, v in target_cp.items()}
+
+    for f in fonts:
+        print(f"======= {f} =======")
+        font_path = f"data/fonts/{f}.ttf"
+        output_path = f"data/init"
+        txt = "BUNNY"
+        subdivision_thresh = None
+        font_string_to_svgs(output_path, font_path, txt, target_control=target_cp,
+                            subdivision_thresh=subdivision_thresh)
+        normalize_letter_size(output_path, font_path, txt)
+
+        print("DONE")
+
+
+
+

code/utils.py

@@ -0,0 +1,225 @@
+import collections.abc
+import os
+import os.path as osp
+from torch import nn
+import kornia.augmentation as K
+import pydiffvg
+import save_svg
+import cv2
+from ttf import font_string_to_svgs, font_string_to_svgs_hb, normalize_letter_size
+import torch
+import numpy as np
+
+
+def edict_2_dict(x):
+    if isinstance(x, dict):
+        xnew = {}
+        for k in x:
+            xnew[k] = edict_2_dict(x[k])
+        return xnew
+    elif isinstance(x, list):
+        xnew = []
+        for i in range(len(x)):
+            xnew.append( edict_2_dict(x[i]))
+        return xnew
+    else:
+        return x
+
+
+def check_and_create_dir(path):
+    pathdir = osp.split(path)[0]
+    if osp.isdir(pathdir):
+        pass
+    else:
+        os.makedirs(pathdir)
+
+
+def update(d, u):
+    """https://stackoverflow.com/questions/3232943/update-value-of-a-nested-dictionary-of-varying-depth"""
+    for k, v in u.items():
+        if isinstance(v, collections.abc.Mapping):
+            d[k] = update(d.get(k, {}), v)
+        else:
+            d[k] = v
+    return d
+
+
+def preprocess(font, word, letter, level_of_cc=1):
+
+    if level_of_cc == 0:
+        target_cp = None
+    else:
+        target_cp = {"A": 120, "B": 120, "C": 100, "D": 100,
+                     "E": 120, "F": 120, "G": 120, "H": 120,
+                     "I": 35, "J": 80, "K": 100, "L": 80,
+                     "M": 100, "N": 100, "O": 100, "P": 120,
+                     "Q": 120, "R": 130, "S": 110, "T": 90,
+                     "U": 100, "V": 100, "W": 100, "X": 130,
+                     "Y": 120, "Z": 120,
+                     "a": 120, "b": 120, "c": 100, "d": 100,
+                     "e": 120, "f": 120, "g": 120, "h": 120,
+                     "i": 35, "j": 80, "k": 100, "l": 80,
+                     "m": 100, "n": 100, "o": 100, "p": 120,
+                     "q": 120, "r": 130, "s": 110, "t": 90,
+                     "u": 100, "v": 100, "w": 100, "x": 130,
+                     "y": 120, "z": 120
+                     }
+        target_cp = {k: v * level_of_cc for k, v in target_cp.items()}
+
+    print(f"======= {font} =======")
+    if font[0] in ['0', '1', '2']:
+        font_path = f"code/data/arabic-fonts/{font}.ttf"
+    else:
+        font_path = f"code/data/fonts/{font}.ttf"
+
+    init_path = f"code/data/init"
+    subdivision_thresh = None
+    chars = font_string_to_svgs_hb(init_path, font_path, word, target_control=target_cp,
+                        subdivision_thresh=subdivision_thresh)
+    normalize_letter_size(init_path, font_path, word, chars)
+
+    # optimaize two adjacent letters
+    if len(letter) > 1:
+        subdivision_thresh = None
+        font_string_to_svgs_hb(init_path, font_path, letter, target_control=target_cp,
+                            subdivision_thresh=subdivision_thresh)
+        normalize_letter_size(init_path, font_path, letter, chars)
+
+    print("Done preprocess")
+
+def get_data_augs(cut_size):
+    augmentations = []
+    augmentations.append(K.RandomPerspective(distortion_scale=0.5, p=0.7))
+    augmentations.append(K.RandomCrop(size=(cut_size, cut_size), pad_if_needed=True, padding_mode='reflect', p=1.0))
+    return nn.Sequential(*augmentations)
+
+
+'''pytorch adaptation of https://github.com/google/mipnerf'''
+def learning_rate_decay(step,
+                        lr_init,
+                        lr_final,
+                        max_steps,
+                        lr_delay_steps=0,
+                        lr_delay_mult=1):
+  """Continuous learning rate decay function.
+  The returned rate is lr_init when step=0 and lr_final when step=max_steps, and
+  is log-linearly interpolated elsewhere (equivalent to exponential decay).
+  If lr_delay_steps>0 then the learning rate will be scaled by some smooth
+  function of lr_delay_mult, such that the initial learning rate is
+  lr_init*lr_delay_mult at the beginning of optimization but will be eased back
+  to the normal learning rate when steps>lr_delay_steps.
+  Args:
+    step: int, the current optimization step.
+    lr_init: float, the initial learning rate.
+    lr_final: float, the final learning rate.
+    max_steps: int, the number of steps during optimization.
+    lr_delay_steps: int, the number of steps to delay the full learning rate.
+    lr_delay_mult: float, the multiplier on the rate when delaying it.
+  Returns:
+    lr: the learning for current step 'step'.
+  """
+  if lr_delay_steps > 0:
+    # A kind of reverse cosine decay.
+    delay_rate = lr_delay_mult + (1 - lr_delay_mult) * np.sin(
+        0.5 * np.pi * np.clip(step / lr_delay_steps, 0, 1))
+  else:
+    delay_rate = 1.
+  t = np.clip(step / max_steps, 0, 1)
+  log_lerp = np.exp(np.log(lr_init) * (1 - t) + np.log(lr_final) * t)
+  return delay_rate * log_lerp
+
+
+
+def save_image(img, filename, gamma=1):
+    check_and_create_dir(filename)
+    imshow = img.detach().cpu()
+    pydiffvg.imwrite(imshow, filename, gamma=gamma)
+
+
+def get_letter_ids(letter, word, shape_groups):
+    for group, l in zip(shape_groups, word):
+        if l == letter:
+            return group.shape_ids
+
+
+def combine_word(word, letter, font, experiment_dir):
+    word_svg_scaled = f"./code/data/init/{font}_{word}_scaled.svg"
+    canvas_width_word, canvas_height_word, shapes_word, shape_groups_word = pydiffvg.svg_to_scene(word_svg_scaled)
+
+    letter_ids = []
+    for l in letter:
+        letter_ids += get_letter_ids(l, word, shape_groups_word)
+
+    w_min, w_max = min([torch.min(shapes_word[ids].points[:, 0]) for ids in letter_ids]), max(
+        [torch.max(shapes_word[ids].points[:, 0]) for ids in letter_ids])
+    h_min, h_max = min([torch.min(shapes_word[ids].points[:, 1]) for ids in letter_ids]), max(
+        [torch.max(shapes_word[ids].points[:, 1]) for ids in letter_ids])
+
+    c_w = (-w_min + w_max) / 2
+    c_h = (-h_min + h_max) / 2
+
+    svg_result = os.path.join(experiment_dir, "output-svg", "output.svg")
+    canvas_width, canvas_height, shapes, shape_groups = pydiffvg.svg_to_scene(svg_result)
+    
+    out_w_min, out_w_max = min([torch.min(p.points[:, 0]) for p in shapes]), max(
+        [torch.max(p.points[:, 0]) for p in shapes])
+    out_h_min, out_h_max = min([torch.min(p.points[:, 1]) for p in shapes]), max(
+        [torch.max(p.points[:, 1]) for p in shapes])
+
+    out_c_w = (-out_w_min + out_w_max) / 2
+    out_c_h = (-out_h_min + out_h_max) / 2
+
+    scale_canvas_w = (w_max - w_min) / (out_w_max - out_w_min)
+    scale_canvas_h = (h_max - h_min) / (out_h_max - out_h_min)
+
+    if scale_canvas_h > scale_canvas_w:
+        wsize = int((out_w_max - out_w_min) * scale_canvas_h)
+        scale_canvas_w = wsize / (out_w_max - out_w_min)
+        shift_w = -out_c_w * scale_canvas_w + c_w
+    else:
+        hsize = int((out_h_max - out_h_min) * scale_canvas_w)
+        scale_canvas_h = hsize / (out_h_max - out_h_min)
+        shift_h = -out_c_h * scale_canvas_h + c_h
+
+    for num, p in enumerate(shapes):
+        p.points[:, 0] = p.points[:, 0] * scale_canvas_w
+        p.points[:, 1] = p.points[:, 1] * scale_canvas_h
+        if scale_canvas_h > scale_canvas_w:
+            p.points[:, 0] = p.points[:, 0] - out_w_min * scale_canvas_w + w_min + shift_w
+            p.points[:, 1] = p.points[:, 1] - out_h_min * scale_canvas_h + h_min
+        else:
+            p.points[:, 0] = p.points[:, 0] - out_w_min * scale_canvas_w + w_min
+            p.points[:, 1] = p.points[:, 1] - out_h_min * scale_canvas_h + h_min + shift_h
+
+    for j, s in enumerate(letter_ids):
+        shapes_word[s] = shapes[j]
+
+    save_svg.save_svg(
+        f"{experiment_dir}/{font}_{word}_{letter}.svg", canvas_width, canvas_height, shapes_word,
+        shape_groups_word)
+
+    render = pydiffvg.RenderFunction.apply
+    scene_args = pydiffvg.RenderFunction.serialize_scene(canvas_width, canvas_height, shapes_word, shape_groups_word)
+    img = render(canvas_width, canvas_height, 2, 2, 0, None, *scene_args)
+    img = img[:, :, 3:4] * img[:, :, :3] + \
+               torch.ones(img.shape[0], img.shape[1], 3, device="cuda:0") * (1 - img[:, :, 3:4])
+    img = img[:, :, :3]
+    save_image(img, f"{experiment_dir}/{font}_{word}_{letter}.png")
+
+
+def create_video(num_iter, experiment_dir, video_frame_freq):
+    img_array = []
+    for ii in range(0, num_iter):
+        if ii % video_frame_freq == 0 or ii == num_iter - 1:
+            filename = os.path.join(
+                experiment_dir, "video-png", f"iter{ii:04d}.png")
+            img = cv2.imread(filename)
+            img_array.append(img)
+
+    video_name = os.path.join(
+        experiment_dir, "video.mp4")
+    check_and_create_dir(video_name)
+    out = cv2.VideoWriter(video_name, cv2.VideoWriter_fourcc(*'mp4v'), 30.0, (600, 600))
+    for iii in range(len(img_array)):
+        out.write(img_array[iii])
+    out.release()

diffvg

@@ -0,0 +1 @@
+Subproject commit adb04d04ee63e82d6569a2fa178ba0dd49115561

packages.txt

@@ -0,0 +1 @@
+python3-dev

requirements.txt

@@ -0,0 +1,28 @@
+--extra-index-url https://download.pytorch.org/whl/cu113
+torch==1.12.1+cu113
+torchvision==0.13.1+cu113
+
+cmake
+numpy
+scikit-image
+ffmpeg
+svgwrite
+svgpathtools
+cssutils
+numba
+torch-tools
+scikit-fmm
+easydict
+visdom
+opencv-python==4.5.4.60  
+
+diffusers==0.8
+transformers
+scipy
+ftfy
+accelerate
+
+vharfbuzz
+freetype-py
+shapely
+kornia==0.6.8