# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the Chameleon License found in the
# LICENSE file in the root directory of this source tree.
from functools import cached_property
import torch
class VocabInfo:
def __init__(self, vocab_map: dict[str, int]):
self.name2val = vocab_map
self.bos_id = vocab_map.get("")
self.eos_id = vocab_map.get("")
self.boi_id = vocab_map.get("")
self.eoi_id = vocab_map.get("")
self.pad_id = vocab_map.get("")
self.eot_id = vocab_map.get("")
@property
def begin_sequence(self) -> int:
return self.bos_id
@property
def end_sequence(self) -> int:
return self.eos_id
@property
def begin_image(self) -> int:
return self.boi_id
@property
def end_image(self) -> int:
return self.eoi_id
@property
def padding(self) -> int:
return self.pad_id
@property
def end_turn(self) -> int:
return self.eot_id
@cached_property
def val2name(self) -> dict[int, str]:
return {v: k for k, v in self.name2val.items()}
@cached_property
def all_tokens(self) -> list[int]:
return sorted(self.name2val.values())
@cached_property
def image_tokens(self) -> list[int]:
return sorted(
[val for name, val in self.name2val.items() if name.startswith("IMGIMG")]
)
@cached_property
def special_tokens(self) -> list[int]:
return sorted(
[
val
for name, val in self.name2val.items()
if name.startswith("<") and name != "<"
]
)
@cached_property
def text_tokens(self) -> list[int]:
return sorted(
set(self.all_tokens) - set(self.image_tokens) - set(self.special_tokens)
)
class VocabTranslation:
def __init__(self, vocab_info: VocabInfo, device: str | None = None):
self._vocab = vocab_info
self._device = device
@cached_property
def bpe2img(self) -> dict[int, int]: # vocab id => codebook id, i.e. [4:8195] => [0:8191]
img_tkn_chr_mapping = {chr(ord("A") + i): str(i) for i in range(10)} # A-J: 0-9
def remap(old_name: str) -> str:
return "".join(
img_tkn_chr_mapping.get(c, c) for c in old_name[len("IMGIMG") : -1] # last chr is 'Z'
)
# e.g.: IMGIMGFDZ => FD => 53,
return {
tok: int(remap(self._vocab.val2name[tok]))
for tok in self._vocab.image_tokens # the token starts with 'IMGIMG', value: [4: 8195]
}
@cached_property
def img2bpe(self) -> dict[int, int]:
return {v: k for k, v in self.bpe2img.items()} # codebook id => vocab id, i.e. [0:8191] => [4:8191]
@cached_property
def bpe2img_search_tensors(self) -> tuple[torch.Tensor, torch.Tensor]:
sorted_bpe = torch.tensor(sorted(self.bpe2img.keys()), device=self._device)
sorted_img = torch.tensor(sorted(self.bpe2img.values()), device=self._device)
return sorted_bpe, sorted_img
@cached_property
def img2bpe_mapping_tensor(self) -> torch.LongTensor:
mapping = torch.zeros(
max(self.img2bpe.keys()) + 1,
dtype=torch.int,
device=self._device,
)
for k, v in self.img2bpe.items():
mapping[k] = v
return mapping
def convert_bpe2img(self, bpe_batch: torch.Tensor) -> torch.Tensor:
bpe_tok, img_tok = self.bpe2img_search_tensors
return img_tok[torch.searchsorted(bpe_tok, bpe_batch)]
def convert_img2bp2(self, img_batch: torch.Tensor) -> torch.Tensor:
return self.img2bpe_mapping_tensor[img_batch]