Upload 131 files

Data/BangDream/config.json

@@ -0,0 +1,184 @@
+{
+  "train": {
+    "log_interval": 200,
+    "eval_interval": 20000,
+    "seed": 42,
+    "epochs": 1000,
+    "learning_rate": 0.0001,
+    "betas": [
+      0.8,
+      0.99
+    ],
+    "eps": 1e-09,
+    "batch_size": 8,
+    "bf16_run": false,
+    "fp16_run": false,
+    "lr_decay": 0.99996,
+    "segment_size": 16384,
+    "init_lr_ratio": 1,
+    "warmup_epochs": 0,
+    "c_mel": 45,
+    "c_kl": 1.0,
+    "c_commit": 100,
+    "skip_optimizer": true,
+    "freeze_ZH_bert": false,
+    "freeze_JP_bert": false,
+    "freeze_EN_bert": false,
+    "freeze_emo": false
+  },
+  "data": {
+    "training_files": "Data/BangDream/filelists/train.list",
+    "validation_files": "Data/BangDream/filelists/val.list",
+    "max_wav_value": 32768.0,
+    "sampling_rate": 44100,
+    "filter_length": 2048,
+    "hop_length": 512,
+    "win_length": 2048,
+    "n_mel_channels": 128,
+    "mel_fmin": 0.0,
+    "mel_fmax": null,
+    "add_blank": true,
+    "n_speakers": 75,
+    "cleaned_text": true,
+    "spk2id": {
+      "紗夜": 0,
+      "有咲": 1,
+      "たえ": 2,
+      "りみ": 3,
+      "香澄": 4,
+      "ロック": 5,
+      "パレオ": 6,
+      "レイヤ": 7,
+      "千聖": 8,
+      "イヴ": 9,
+      "蘭": 10,
+      "巴": 11,
+      "ひまり": 12,
+      "つぐみ": 13,
+      "華戀": 14,
+      "晶": 15,
+      "光": 16,
+      "未知留": 17,
+      "香子": 18,
+      "雙葉": 19,
+      "真晝": 20,
+      "艾露": 21,
+      "珠緒": 22,
+      "艾露露": 23,
+      "純那": 24,
+      "克洛迪娜": 25,
+      "真矢": 26,
+      "奈奈": 27,
+      "壘": 28,
+      "文": 29,
+      "一愛": 30,
+      "菈樂菲": 31,
+      "司": 32,
+      "美空": 33,
+      "靜羽": 34,
+      "悠悠子": 35,
+      "八千代": 36,
+      "栞": 37,
+      "美帆": 38,
+      "安德露": 39,
+      "瑪莉亞貝菈": 40,
+      "克拉迪亞": 41,
+      "桃樂西": 42,
+      "瑪麗安": 43,
+      "花音": 44,
+      "はぐみ": 45,
+      "こころ": 46,
+      "美咲": 47,
+      "沙綾": 48,
+      "つくし": 49,
+      "瑠唯": 50,
+      "透子": 51,
+      "七深": 52,
+      "ましろ": 53,
+      "友希那": 54,
+      "リサ": 55,
+      "あこ": 56,
+      "チュチュ": 57,
+      "薫": 58,
+      "麻弥": 59,
+      "彩": 60,
+      "日菜": 61,
+      "愛音": 62,
+      "楽奈": 63,
+      "そよ": 64,
+      "立希": 65,
+      "燐子": 66,
+      "モカ": 67,
+      "燈": 68,
+      "ますき": 69,
+      "祥子": 70,
+      "睦": 71,
+      "海鈴": 72,
+      "にゃむ": 73,
+      "初華": 74
+    }
+  },
+  "model": {
+    "use_spk_conditioned_encoder": true,
+    "use_noise_scaled_mas": true,
+    "use_mel_posterior_encoder": false,
+    "use_duration_discriminator": false,
+    "use_wavlm_discriminator": true,
+    "inter_channels": 192,
+    "hidden_channels": 192,
+    "filter_channels": 768,
+    "n_heads": 2,
+    "n_layers": 6,
+    "kernel_size": 3,
+    "p_dropout": 0.1,
+    "resblock": "1",
+    "resblock_kernel_sizes": [
+      3,
+      7,
+      11
+    ],
+    "resblock_dilation_sizes": [
+      [
+        1,
+        3,
+        5
+      ],
+      [
+        1,
+        3,
+        5
+      ],
+      [
+        1,
+        3,
+        5
+      ]
+    ],
+    "upsample_rates": [
+      8,
+      8,
+      2,
+      2,
+      2
+    ],
+    "upsample_initial_channel": 512,
+    "upsample_kernel_sizes": [
+      16,
+      16,
+      8,
+      2,
+      2
+    ],
+    "n_layers_q": 3,
+    "use_spectral_norm": false,
+    "gin_channels": 512,
+    "slm": {
+      "model": "./slm/wavlm-base-plus",
+      "sr": 16000,
+      "hidden": 768,
+      "nlayers": 13,
+      "initial_channel": 64
+    }
+  },
+  "version": "2.4"
+}

Data/BangDream/models/G_060000.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4fd175e07cf8e52eeae57b4130030a580c0b31edb314b1f3e8e60f1c6439db71
+size 914308886

app.py

@@ -0,0 +1,279 @@
+import argparse
+import os
+from pathlib import Path
+
+import logging
+import re_matching
+
+logging.getLogger("numba").setLevel(logging.WARNING)
+logging.getLogger("markdown_it").setLevel(logging.WARNING)
+logging.getLogger("urllib3").setLevel(logging.WARNING)
+logging.getLogger("matplotlib").setLevel(logging.WARNING)
+
+logging.basicConfig(
+    level=logging.INFO, format="| %(name)s | %(levelname)s | %(message)s"
+)
+
+logger = logging.getLogger(__name__)
+
+import librosa
+import numpy as np
+import torch
+import torch.nn as nn
+from torch.utils.data import Dataset
+from torch.utils.data import DataLoader, Dataset
+from tqdm import tqdm
+from clap_wrapper import get_clap_audio_feature, get_clap_text_feature
+
+
+import gradio as gr
+
+import utils
+from config import config
+
+import torch
+import commons
+from text import cleaned_text_to_sequence, get_bert
+from text.cleaner import clean_text
+import utils
+
+from models import SynthesizerTrn
+from text.symbols import symbols
+import sys
+
+net_g = None
+'''
+device = (
+        "cuda:0"
+        if torch.cuda.is_available()
+        else (
+            "mps"
+            if sys.platform == "darwin" and torch.backends.mps.is_available()
+            else "cpu"
+        )
+    )
+'''
+device = "cpu"
+BandList = {
+        "PoppinParty":["香澄","有咲","たえ","りみ","沙綾"],
+        "Afterglow":["蘭","モカ","ひまり","巴","つぐみ"],
+        "HelloHappyWorld":["こころ","美咲","薫","花音","はぐみ"],
+        "PastelPalettes":["彩","日菜","千聖","イヴ","麻弥"],
+        "Roselia":["友希那","紗夜","リサ","燐子","あこ"],
+        "RaiseASuilen":["レイヤ","ロック","ますき","チュチュ","パレオ"],
+        "Morfonica":["ましろ","瑠唯","つくし","七深","透子"],
+        "MyGo":["燈","愛音","そよ","立希","楽奈"],
+        "AveMujica":["祥子","睦","海鈴","にゃむ","初華"],
+        "圣翔音乐学园":["華戀","光","香子","雙葉","真晝","純那","克洛迪娜","真矢","奈奈"],
+        "凛明馆女子学校":["珠緒","壘","文","悠悠子","一愛"],
+        "弗隆提亚艺术学校":["艾露","艾露露","菈樂菲","司","靜羽"],
+        "西克菲尔特音乐学院":["晶","未知留","八千代","栞","美帆"]
+}
+
+def get_net_g(model_path: str, device: str, hps):
+    # 当前版本模型 net_g
+    net_g = SynthesizerTrn(
+        len(symbols),
+        hps.data.filter_length // 2 + 1,
+        hps.train.segment_size // hps.data.hop_length,
+        n_speakers=hps.data.n_speakers,
+        **hps.model,
+    ).to(device)
+    _ = net_g.eval()
+    _ = utils.load_checkpoint(model_path, net_g, None, skip_optimizer=True)
+    return net_g
+
+
+def get_text(text, language_str, hps, device, style_text=None, style_weight=0.7):
+    style_text = None if style_text == "" else style_text
+    # 在此处实现当前版本的get_text
+    norm_text, phone, tone, word2ph = clean_text(text, language_str)
+    phone, tone, language = cleaned_text_to_sequence(phone, tone, language_str)
+    if hps.data.add_blank:
+        phone = commons.intersperse(phone, 0)
+        tone = commons.intersperse(tone, 0)
+        language = commons.intersperse(language, 0)
+        for i in range(len(word2ph)):
+            word2ph[i] = word2ph[i] * 2
+        word2ph[0] += 1
+    bert = get_bert(norm_text, word2ph, language_str, device, style_text, style_weight)
+    del word2ph
+
+    assert bert.shape[-1] == len(
+        phone
+    ), f"Bert seq len {bert.shape[-1]} != {len(phone)}"
+
+    phone = torch.LongTensor(phone)
+    tone = torch.LongTensor(tone)
+    language = torch.LongTensor(language)
+    return bert, phone, tone, language
+
+def infer(
+    text,
+    sdp_ratio,
+    noise_scale,
+    noise_scale_w,
+    length_scale,
+    sid,
+    emotion,
+    reference_audio=None,
+    skip_start=False,
+    skip_end=False,
+    style_text=None,
+    style_weight=0.7,
+):
+    language = "JP"
+    if isinstance(reference_audio, np.ndarray):
+        emo = get_clap_audio_feature(reference_audio, device)
+    else:
+        emo = get_clap_text_feature(emotion, device)
+    emo = torch.squeeze(emo, dim=1)
+
+    bert, phones, tones, lang_ids = get_text(
+        text,
+        language,
+        hps,
+        device,
+        style_text=style_text,
+        style_weight=style_weight,
+    )
+    if skip_start:
+        phones = phones[3:]
+        tones = tones[3:]
+        lang_ids = lang_ids[3:]
+        bert = bert[:, 3:]
+    if skip_end:
+        phones = phones[:-2]
+        tones = tones[:-2]
+        lang_ids = lang_ids[:-2]
+        bert = bert[:, :-2]
+    with torch.no_grad():
+        x_tst = phones.to(device).unsqueeze(0)
+        tones = tones.to(device).unsqueeze(0)
+        lang_ids = lang_ids.to(device).unsqueeze(0)
+        bert = bert.to(device).unsqueeze(0)
+        x_tst_lengths = torch.LongTensor([phones.size(0)]).to(device)
+        emo = emo.to(device).unsqueeze(0)
+        del phones
+        speakers = torch.LongTensor([hps.data.spk2id[sid]]).to(device)
+        print(text)
+        audio = (
+            net_g.infer(
+                x_tst,
+                x_tst_lengths,
+                speakers,
+                tones,
+                lang_ids,
+                bert,
+                emo,
+                sdp_ratio=sdp_ratio,
+                noise_scale=noise_scale,
+                noise_scale_w=noise_scale_w,
+                length_scale=length_scale,
+            )[0][0, 0]
+            .data.cpu()
+            .float()
+            .numpy()
+        )
+        del (
+            x_tst,
+            tones,
+            lang_ids,
+            bert,
+            x_tst_lengths,
+            speakers,
+            emo,
+        )  # , emo
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+        return (hps.data.sampling_rate,gr.processing_utils.convert_to_16_bit_wav(audio))
+
+def loadmodel(model):
+    _ = net_g.eval()
+    _ = utils.load_checkpoint(model, net_g, None, skip_optimizer=True)
+    return "success"
+
+if __name__ == "__main__":
+    modelPaths = []
+    for dirpath, dirnames, filenames in os.walk('Data/BangDream/models/'):
+        for filename in filenames:
+            modelPaths.append(os.path.join(dirpath, filename))
+    hps = utils.get_hparams_from_file('Data/BangDream//config.json')
+    net_g = get_net_g(
+        model_path=modelPaths[-1], device=device, hps=hps
+    )
+    speaker_ids = hps.data.spk2id
+    speakers = list(speaker_ids.keys())
+    with gr.Blocks() as app:
+        for band in BandList:
+            with gr.TabItem(band):
+                for name in BandList[band]:
+                    with gr.TabItem(name):
+                        with gr.Row():
+                            with gr.Column():
+                                with gr.Row():
+                                    gr.Markdown(
+                                        '<div align="center">'
+                                        f'<img style="width:auto;height:400px;" src="https://mahiruoshi-bangdream-bert-vits2.hf.space/file/image/{name}.png">' 
+                                        '</div>'
+                                    )
+                                length_scale = gr.Slider(
+                                        minimum=0.1, maximum=2, value=1, step=0.01, label="语速调节"
+                                    )
+                                emotion = gr.Textbox(
+                                        label="情感标注文本t",
+                                        value = 'なんではるひかげやったの?!!'
+                                    )
+                                style_weight = gr.Slider(
+                                        minimum=0.1, maximum=2, value=1, step=0.01, label="感情比重"
+                                    )
+                                with gr.Accordion(label="参数设定", open=False):
+                                    sdp_ratio = gr.Slider(
+                                    minimum=0, maximum=1, value=0.2, step=0.01, label="SDP/DP混合比"
+                                    )
+                                    noise_scale = gr.Slider(
+                                        minimum=0.1, maximum=2, value=0.6, step=0.01, label="感情调节"
+                                    )
+                                    noise_scale_w = gr.Slider(
+                                        minimum=0.1, maximum=2, value=0.8, step=0.01, label="音素长度"
+                                    )
+                                    speaker = gr.Dropdown(
+                                        choices=speakers, value=name, label="说话人"
+                                    )
+                                    skip_start = gr.Checkbox(label="跳过开头")
+                                    skip_end = gr.Checkbox(label="跳过结尾")
+                                with gr.Accordion(label="切换模型", open=False):
+                                    modelstrs = gr.Dropdown(label = "模型", choices = modelPaths, value = modelPaths[0], type = "value")
+                                    btnMod = gr.Button("载入模型")
+                                    statusa = gr.TextArea()
+                                    btnMod.click(loadmodel, inputs=[modelstrs], outputs = [statusa])
+                            with gr.Column():
+                                text = gr.TextArea(
+                                    label="输入纯日语",
+                                    placeholder="输入纯日语",
+                                    value="なんではるひかげやったの?!!",
+                                )
+                                reference_audio = gr.Audio(label="情感参考音频）", type="filepath")
+                                btn = gr.Button("点击生成", variant="primary")
+                                audio_output = gr.Audio(label="Output Audio")
+                    btn.click(
+                        infer,
+                        inputs=[
+                            text,
+                            sdp_ratio,
+                            noise_scale,
+                            noise_scale_w,
+                            length_scale,
+                            speaker,
+                            emotion,
+                            reference_audio,
+                            skip_start,
+                            skip_end,
+                            emotion,
+                            style_weight,
+                        ],
+                        outputs=[audio_output],
+                    )
+
+    print("推理页面已开启!")
+    app.launch(share=True)

attentions.py

@@ -0,0 +1,464 @@
+import math
+import torch
+from torch import nn
+from torch.nn import functional as F
+
+import commons
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class LayerNorm(nn.Module):
+    def __init__(self, channels, eps=1e-5):
+        super().__init__()
+        self.channels = channels
+        self.eps = eps
+
+        self.gamma = nn.Parameter(torch.ones(channels))
+        self.beta = nn.Parameter(torch.zeros(channels))
+
+    def forward(self, x):
+        x = x.transpose(1, -1)
+        x = F.layer_norm(x, (self.channels,), self.gamma, self.beta, self.eps)
+        return x.transpose(1, -1)
+
+
+@torch.jit.script
+def fused_add_tanh_sigmoid_multiply(input_a, input_b, n_channels):
+    n_channels_int = n_channels[0]
+    in_act = input_a + input_b
+    t_act = torch.tanh(in_act[:, :n_channels_int, :])
+    s_act = torch.sigmoid(in_act[:, n_channels_int:, :])
+    acts = t_act * s_act
+    return acts
+
+
+class Encoder(nn.Module):
+    def __init__(
+        self,
+        hidden_channels,
+        filter_channels,
+        n_heads,
+        n_layers,
+        kernel_size=1,
+        p_dropout=0.0,
+        window_size=4,
+        isflow=True,
+        **kwargs
+    ):
+        super().__init__()
+        self.hidden_channels = hidden_channels
+        self.filter_channels = filter_channels
+        self.n_heads = n_heads
+        self.n_layers = n_layers
+        self.kernel_size = kernel_size
+        self.p_dropout = p_dropout
+        self.window_size = window_size
+        # if isflow:
+        #  cond_layer = torch.nn.Conv1d(256, 2*hidden_channels*n_layers, 1)
+        #  self.cond_pre = torch.nn.Conv1d(hidden_channels, 2*hidden_channels, 1)
+        #  self.cond_layer = weight_norm(cond_layer, name='weight')
+        #  self.gin_channels = 256
+        self.cond_layer_idx = self.n_layers
+        if "gin_channels" in kwargs:
+            self.gin_channels = kwargs["gin_channels"]
+            if self.gin_channels != 0:
+                self.spk_emb_linear = nn.Linear(self.gin_channels, self.hidden_channels)
+                # vits2 says 3rd block, so idx is 2 by default
+                self.cond_layer_idx = (
+                    kwargs["cond_layer_idx"] if "cond_layer_idx" in kwargs else 2
+                )
+                logging.debug(self.gin_channels, self.cond_layer_idx)
+                assert (
+                    self.cond_layer_idx < self.n_layers
+                ), "cond_layer_idx should be less than n_layers"
+        self.drop = nn.Dropout(p_dropout)
+        self.attn_layers = nn.ModuleList()
+        self.norm_layers_1 = nn.ModuleList()
+        self.ffn_layers = nn.ModuleList()
+        self.norm_layers_2 = nn.ModuleList()
+        for i in range(self.n_layers):
+            self.attn_layers.append(
+                MultiHeadAttention(
+                    hidden_channels,
+                    hidden_channels,
+                    n_heads,
+                    p_dropout=p_dropout,
+                    window_size=window_size,
+                )
+            )
+            self.norm_layers_1.append(LayerNorm(hidden_channels))
+            self.ffn_layers.append(
+                FFN(
+                    hidden_channels,
+                    hidden_channels,
+                    filter_channels,
+                    kernel_size,
+                    p_dropout=p_dropout,
+                )
+            )
+            self.norm_layers_2.append(LayerNorm(hidden_channels))
+
+    def forward(self, x, x_mask, g=None):
+        attn_mask = x_mask.unsqueeze(2) * x_mask.unsqueeze(-1)
+        x = x * x_mask
+        for i in range(self.n_layers):
+            if i == self.cond_layer_idx and g is not None:
+                g = self.spk_emb_linear(g.transpose(1, 2))
+                g = g.transpose(1, 2)
+                x = x + g
+                x = x * x_mask
+            y = self.attn_layers[i](x, x, attn_mask)
+            y = self.drop(y)
+            x = self.norm_layers_1[i](x + y)
+
+            y = self.ffn_layers[i](x, x_mask)
+            y = self.drop(y)
+            x = self.norm_layers_2[i](x + y)
+        x = x * x_mask
+        return x
+
+
+class Decoder(nn.Module):
+    def __init__(
+        self,
+        hidden_channels,
+        filter_channels,
+        n_heads,
+        n_layers,
+        kernel_size=1,
+        p_dropout=0.0,
+        proximal_bias=False,
+        proximal_init=True,
+        **kwargs
+    ):
+        super().__init__()
+        self.hidden_channels = hidden_channels
+        self.filter_channels = filter_channels
+        self.n_heads = n_heads
+        self.n_layers = n_layers
+        self.kernel_size = kernel_size
+        self.p_dropout = p_dropout
+        self.proximal_bias = proximal_bias
+        self.proximal_init = proximal_init
+
+        self.drop = nn.Dropout(p_dropout)
+        self.self_attn_layers = nn.ModuleList()
+        self.norm_layers_0 = nn.ModuleList()
+        self.encdec_attn_layers = nn.ModuleList()
+        self.norm_layers_1 = nn.ModuleList()
+        self.ffn_layers = nn.ModuleList()
+        self.norm_layers_2 = nn.ModuleList()
+        for i in range(self.n_layers):
+            self.self_attn_layers.append(
+                MultiHeadAttention(
+                    hidden_channels,
+                    hidden_channels,
+                    n_heads,
+                    p_dropout=p_dropout,
+                    proximal_bias=proximal_bias,
+                    proximal_init=proximal_init,
+                )
+            )
+            self.norm_layers_0.append(LayerNorm(hidden_channels))
+            self.encdec_attn_layers.append(
+                MultiHeadAttention(
+                    hidden_channels, hidden_channels, n_heads, p_dropout=p_dropout
+                )
+            )
+            self.norm_layers_1.append(LayerNorm(hidden_channels))
+            self.ffn_layers.append(
+                FFN(
+                    hidden_channels,
+                    hidden_channels,
+                    filter_channels,
+                    kernel_size,
+                    p_dropout=p_dropout,
+                    causal=True,
+                )
+            )
+            self.norm_layers_2.append(LayerNorm(hidden_channels))
+
+    def forward(self, x, x_mask, h, h_mask):
+        """
+        x: decoder input
+        h: encoder output
+        """
+        self_attn_mask = commons.subsequent_mask(x_mask.size(2)).to(
+            device=x.device, dtype=x.dtype
+        )
+        encdec_attn_mask = h_mask.unsqueeze(2) * x_mask.unsqueeze(-1)
+        x = x * x_mask
+        for i in range(self.n_layers):
+            y = self.self_attn_layers[i](x, x, self_attn_mask)
+            y = self.drop(y)
+            x = self.norm_layers_0[i](x + y)
+
+            y = self.encdec_attn_layers[i](x, h, encdec_attn_mask)
+            y = self.drop(y)
+            x = self.norm_layers_1[i](x + y)
+
+            y = self.ffn_layers[i](x, x_mask)
+            y = self.drop(y)
+            x = self.norm_layers_2[i](x + y)
+        x = x * x_mask
+        return x
+
+
+class MultiHeadAttention(nn.Module):
+    def __init__(
+        self,
+        channels,
+        out_channels,
+        n_heads,
+        p_dropout=0.0,
+        window_size=None,
+        heads_share=True,
+        block_length=None,
+        proximal_bias=False,
+        proximal_init=False,
+    ):
+        super().__init__()
+        assert channels % n_heads == 0
+
+        self.channels = channels
+        self.out_channels = out_channels
+        self.n_heads = n_heads
+        self.p_dropout = p_dropout
+        self.window_size = window_size
+        self.heads_share = heads_share
+        self.block_length = block_length
+        self.proximal_bias = proximal_bias
+        self.proximal_init = proximal_init
+        self.attn = None
+
+        self.k_channels = channels // n_heads
+        self.conv_q = nn.Conv1d(channels, channels, 1)
+        self.conv_k = nn.Conv1d(channels, channels, 1)
+        self.conv_v = nn.Conv1d(channels, channels, 1)
+        self.conv_o = nn.Conv1d(channels, out_channels, 1)
+        self.drop = nn.Dropout(p_dropout)
+
+        if window_size is not None:
+            n_heads_rel = 1 if heads_share else n_heads
+            rel_stddev = self.k_channels**-0.5
+            self.emb_rel_k = nn.Parameter(
+                torch.randn(n_heads_rel, window_size * 2 + 1, self.k_channels)
+                * rel_stddev
+            )
+            self.emb_rel_v = nn.Parameter(
+                torch.randn(n_heads_rel, window_size * 2 + 1, self.k_channels)
+                * rel_stddev
+            )
+
+        nn.init.xavier_uniform_(self.conv_q.weight)
+        nn.init.xavier_uniform_(self.conv_k.weight)
+        nn.init.xavier_uniform_(self.conv_v.weight)
+        if proximal_init:
+            with torch.no_grad():
+                self.conv_k.weight.copy_(self.conv_q.weight)
+                self.conv_k.bias.copy_(self.conv_q.bias)
+
+    def forward(self, x, c, attn_mask=None):
+        q = self.conv_q(x)
+        k = self.conv_k(c)
+        v = self.conv_v(c)
+
+        x, self.attn = self.attention(q, k, v, mask=attn_mask)
+
+        x = self.conv_o(x)
+        return x
+
+    def attention(self, query, key, value, mask=None):
+        # reshape [b, d, t] -> [b, n_h, t, d_k]
+        b, d, t_s, t_t = (*key.size(), query.size(2))
+        query = query.view(b, self.n_heads, self.k_channels, t_t).transpose(2, 3)
+        key = key.view(b, self.n_heads, self.k_channels, t_s).transpose(2, 3)
+        value = value.view(b, self.n_heads, self.k_channels, t_s).transpose(2, 3)
+
+        scores = torch.matmul(query / math.sqrt(self.k_channels), key.transpose(-2, -1))
+        if self.window_size is not None:
+            assert (
+                t_s == t_t
+            ), "Relative attention is only available for self-attention."
+            key_relative_embeddings = self._get_relative_embeddings(self.emb_rel_k, t_s)
+            rel_logits = self._matmul_with_relative_keys(
+                query / math.sqrt(self.k_channels), key_relative_embeddings
+            )
+            scores_local = self._relative_position_to_absolute_position(rel_logits)
+            scores = scores + scores_local
+        if self.proximal_bias:
+            assert t_s == t_t, "Proximal bias is only available for self-attention."
+            scores = scores + self._attention_bias_proximal(t_s).to(
+                device=scores.device, dtype=scores.dtype
+            )
+        if mask is not None:
+            scores = scores.masked_fill(mask == 0, -1e4)
+            if self.block_length is not None:
+                assert (
+                    t_s == t_t
+                ), "Local attention is only available for self-attention."
+                block_mask = (
+                    torch.ones_like(scores)
+                    .triu(-self.block_length)
+                    .tril(self.block_length)
+                )
+                scores = scores.masked_fill(block_mask == 0, -1e4)
+        p_attn = F.softmax(scores, dim=-1)  # [b, n_h, t_t, t_s]
+        p_attn = self.drop(p_attn)
+        output = torch.matmul(p_attn, value)
+        if self.window_size is not None:
+            relative_weights = self._absolute_position_to_relative_position(p_attn)
+            value_relative_embeddings = self._get_relative_embeddings(
+                self.emb_rel_v, t_s
+            )
+            output = output + self._matmul_with_relative_values(
+                relative_weights, value_relative_embeddings
+            )
+        output = (
+            output.transpose(2, 3).contiguous().view(b, d, t_t)
+        )  # [b, n_h, t_t, d_k] -> [b, d, t_t]
+        return output, p_attn
+
+    def _matmul_with_relative_values(self, x, y):
+        """
+        x: [b, h, l, m]
+        y: [h or 1, m, d]
+        ret: [b, h, l, d]
+        """
+        ret = torch.matmul(x, y.unsqueeze(0))
+        return ret
+
+    def _matmul_with_relative_keys(self, x, y):
+        """
+        x: [b, h, l, d]
+        y: [h or 1, m, d]
+        ret: [b, h, l, m]
+        """
+        ret = torch.matmul(x, y.unsqueeze(0).transpose(-2, -1))
+        return ret
+
+    def _get_relative_embeddings(self, relative_embeddings, length):
+        2 * self.window_size + 1
+        # Pad first before slice to avoid using cond ops.
+        pad_length = max(length - (self.window_size + 1), 0)
+        slice_start_position = max((self.window_size + 1) - length, 0)
+        slice_end_position = slice_start_position + 2 * length - 1
+        if pad_length > 0:
+            padded_relative_embeddings = F.pad(
+                relative_embeddings,
+                commons.convert_pad_shape([[0, 0], [pad_length, pad_length], [0, 0]]),
+            )
+        else:
+            padded_relative_embeddings = relative_embeddings
+        used_relative_embeddings = padded_relative_embeddings[
+            :, slice_start_position:slice_end_position
+        ]
+        return used_relative_embeddings
+
+    def _relative_position_to_absolute_position(self, x):
+        """
+        x: [b, h, l, 2*l-1]
+        ret: [b, h, l, l]
+        """
+        batch, heads, length, _ = x.size()
+        # Concat columns of pad to shift from relative to absolute indexing.
+        x = F.pad(x, commons.convert_pad_shape([[0, 0], [0, 0], [0, 0], [0, 1]]))
+
+        # Concat extra elements so to add up to shape (len+1, 2*len-1).
+        x_flat = x.view([batch, heads, length * 2 * length])
+        x_flat = F.pad(
+            x_flat, commons.convert_pad_shape([[0, 0], [0, 0], [0, length - 1]])
+        )
+
+        # Reshape and slice out the padded elements.
+        x_final = x_flat.view([batch, heads, length + 1, 2 * length - 1])[
+            :, :, :length, length - 1 :
+        ]
+        return x_final
+
+    def _absolute_position_to_relative_position(self, x):
+        """
+        x: [b, h, l, l]
+        ret: [b, h, l, 2*l-1]
+        """
+        batch, heads, length, _ = x.size()
+        # pad along column
+        x = F.pad(
+            x, commons.convert_pad_shape([[0, 0], [0, 0], [0, 0], [0, length - 1]])
+        )
+        x_flat = x.view([batch, heads, length**2 + length * (length - 1)])
+        # add 0's in the beginning that will skew the elements after reshape
+        x_flat = F.pad(x_flat, commons.convert_pad_shape([[0, 0], [0, 0], [length, 0]]))
+        x_final = x_flat.view([batch, heads, length, 2 * length])[:, :, :, 1:]
+        return x_final
+
+    def _attention_bias_proximal(self, length):
+        """Bias for self-attention to encourage attention to close positions.
+        Args:
+          length: an integer scalar.
+        Returns:
+          a Tensor with shape [1, 1, length, length]
+        """
+        r = torch.arange(length, dtype=torch.float32)
+        diff = torch.unsqueeze(r, 0) - torch.unsqueeze(r, 1)
+        return torch.unsqueeze(torch.unsqueeze(-torch.log1p(torch.abs(diff)), 0), 0)
+
+
+class FFN(nn.Module):
+    def __init__(
+        self,
+        in_channels,
+        out_channels,
+        filter_channels,
+        kernel_size,
+        p_dropout=0.0,
+        activation=None,
+        causal=False,
+    ):
+        super().__init__()
+        self.in_channels = in_channels
+        self.out_channels = out_channels
+        self.filter_channels = filter_channels
+        self.kernel_size = kernel_size
+        self.p_dropout = p_dropout
+        self.activation = activation
+        self.causal = causal
+
+        if causal:
+            self.padding = self._causal_padding
+        else:
+            self.padding = self._same_padding
+
+        self.conv_1 = nn.Conv1d(in_channels, filter_channels, kernel_size)
+        self.conv_2 = nn.Conv1d(filter_channels, out_channels, kernel_size)
+        self.drop = nn.Dropout(p_dropout)
+
+    def forward(self, x, x_mask):
+        x = self.conv_1(self.padding(x * x_mask))
+        if self.activation == "gelu":
+            x = x * torch.sigmoid(1.702 * x)
+        else:
+            x = torch.relu(x)
+        x = self.drop(x)
+        x = self.conv_2(self.padding(x * x_mask))
+        return x * x_mask
+
+    def _causal_padding(self, x):
+        if self.kernel_size == 1:
+            return x
+        pad_l = self.kernel_size - 1
+        pad_r = 0
+        padding = [[0, 0], [0, 0], [pad_l, pad_r]]
+        x = F.pad(x, commons.convert_pad_shape(padding))
+        return x
+
+    def _same_padding(self, x):
+        if self.kernel_size == 1:
+            return x
+        pad_l = (self.kernel_size - 1) // 2
+        pad_r = self.kernel_size // 2
+        padding = [[0, 0], [0, 0], [pad_l, pad_r]]
+        x = F.pad(x, commons.convert_pad_shape(padding))
+        return x

bert/Erlangshen-DeBERTa-v2-710M-Chinese/config.json

@@ -0,0 +1,35 @@
+{
+    "model_type": "deberta-v2",
+    "architectures": [
+        "DebertaV2ForMaskedLM"
+    ],
+    "attention_probs_dropout_prob": 0.1,
+    "attention_head_size": 64,
+    "hidden_act": "gelu",
+    "hidden_dropout_prob": 0.1,
+    "hidden_size": 1536,
+    "initializer_range": 0.02,
+    "intermediate_size": 6144,
+    "max_position_embeddings": 512,
+    "relative_attention": true,
+    "position_buckets": 256,
+    "norm_rel_ebd": "layer_norm",
+    "share_att_key": true,
+    "pos_att_type": [
+        "p2c",
+        "c2p"
+    ],
+    "conv_kernel_size": 3,
+    "pooler_dropout": 0,
+    "pooler_hidden_act": "gelu",
+    "pooler_hidden_size": 1536,
+    "conv_act": "gelu",
+    "layer_norm_eps": 1e-7,
+    "max_relative_positions": -1,
+    "position_biased_input": false,
+    "num_attention_heads": 24,
+    "num_hidden_layers": 24,
+    "type_vocab_size": 0,
+    "num_labels": 119,
+    "vocab_size": 12800
+}

bert/Erlangshen-DeBERTa-v2-710M-Chinese/special_tokens_map.json

@@ -0,0 +1 @@
+{"unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]"}

bert/Erlangshen-DeBERTa-v2-710M-Chinese/tokenizer_config.json

@@ -0,0 +1,15 @@
+{
+    "do_lower_case": true,
+    "do_basic_tokenize": true,
+    "never_split": null,
+    "unk_token": "[UNK]",
+    "sep_token": "[SEP]",
+    "pad_token": "[PAD]",
+    "cls_token": "[CLS]",
+    "mask_token": "[MASK]",
+    "tokenize_chinese_chars": true,
+    "strip_accents": null,
+    "special_tokens_map_file": null,
+    "name_or_path": "/cognitive_comp/gaoxinyu/pretrained_model/bert-1.3B",
+    "tokenizer_class": "BertTokenizer"
+}

bert/Erlangshen-DeBERTa-v2-710M-Chinese/vocab.txt

@@ -0,0 +1,12800 @@
+[PAD]
+[CLS]
+[SEP]
+[UNK]
+[MASK]
+[unused1]
+[unused2]
+[unused3]
+[unused4]
+[unused5]
+[unused6]
+[unused7]
+[unused8]
+[unused9]
+[unused10]
+[unused11]
+[unused12]
+[unused13]
+[unused14]
+[unused15]
+[unused16]
+[unused17]
+[unused18]
+[unused19]
+[unused20]
+[unused21]
+[unused22]
+[unused23]
+[unused24]
+[unused25]
+[unused26]
+[unused27]
+[unused28]
+[unused29]
+[unused30]
+[unused31]
+[unused32]
+[unused33]
+[unused34]
+[unused35]
+[unused36]
+[unused37]
+[unused38]
+[unused39]
+[unused40]
+[unused41]
+[unused42]
+[unused43]
+[unused44]
+[unused45]
+[unused46]
+[unused47]
+[unused48]
+[unused49]
+[unused50]
+[unused51]
+[unused52]
+[unused53]
+[unused54]
+[unused55]
+[unused56]
+[unused57]
+[unused58]
+[unused59]
+[unused60]
+[unused61]
+[unused62]
+[unused63]
+[unused64]
+[unused65]
+[unused66]
+[unused67]
+[unused68]
+[unused69]
+[unused70]
+[unused71]
+[unused72]
+[unused73]
+[unused74]
+[unused75]
+[unused76]
+[unused77]
+[unused78]
+[unused79]
+[unused80]
+[unused81]
+[unused82]
+[unused83]
+[unused84]
+[unused85]
+[unused86]
+[unused87]
+[unused88]
+[unused89]
+[unused90]
+[unused91]
+[unused92]
+[unused93]
+[unused94]
+[unused95]
+[unused96]
+[unused97]
+[unused98]
+[unused99]
+!
+"
+#
+$
+%
+&
+'
+(
+)
+*
++
+,
+-
+.
+/
+:
+;
+<
+=
+>
+?
+@
+[
+\
+]
+^
+_
+`
+{
+|
+}
+~
+·
+–
+—
+‘
+’
+‛
+“
+”
+„
+‟
+…
+‧
+、
+。
+〃
+〈
+〉
+《
+》
+「
+」
+『
+』
+【
+】
+〔
+〕
+〖
+〗
+〜
+〝
+〞
+〰
+﹏
+﹑
+﹔
+！
+＂
+＃
+＄
+％
+＆
+＇
+（
+）
+＊
+＋
+，
+－
+：
+；
+＜
+＝
+＞
+？
+＠
+［
+＼
+］
+＾
+＿
+｀
+｛
+｜
+｝
+～
+｡
+｢
+｣
+､
+0
+##0
+1
+##1
+2
+##2
+3
+##3
+4
+##4
+5
+##5
+6
+##6
+7
+##7
+8
+##8
+9
+##9
+一
+丁
+七
+丄
+丅
+丆
+万
+丈
+三
+上
+下
+丌
+不
+与
+丏
+丐
+丑
+专
+且
+丕
+世
+丘
+丙
+业
+丛
+东
+丝
+丞
+両
+丢
+两
+严
+丧
+丨
+个
+丫
+中
+丰
+串
+临
+丶
+丸
+丹
+为
+主
+丼
+丽
+举
+丿
+乂
+乃
+乄
+久
+乇
+么
+义
+之
+乌
+乍
+乎
+乏
+乐
+乒
+乓
+乔
+乖
+乗
+乘
+乙
+乛
+乜
+九
+乞
+也
+习
+乡
+书
+乩
+买
+乱
+乳
+乸
+乾
+亀
+了
+亇
+予
+争
+亊
+事
+二
+亍
+于
+亏
+云
+互
+亓
+五
+井
+亘
+亚
+些
+亜
+亟
+亡
+亢
+交
+亥
+亦
+产
+亨
+亩
+享
+京
+亭
+亮
+亲
+亳
+亵
+亶
+亸
+亹
+人
+亻
+亽
+亾
+亿
+什
+仁
+仂
+仃
+仄
+仅
+仆
+仇
+仉
+今
+介
+仍
+从
+仏
+仑
+仓
+仔
+仕
+他
+仗
+付
+仙
+仝
+仞
+仟
+仡
+代
+令
+以
+仨
+仩
+仪
+仫
+们
+仮
+仰
+仲
+仳
+仵
+件
+价
+任
+仼
+份
+仿
+企
+伉
+伊
+伋
+伍
+伎
+伏
+伐
+休
+众
+优
+伙
+会
+伛
+伝
+伞
+伟
+传
+伢
+伤
+伥
+伦
+伧
+伪
+伫
+伯
+估
+伱
+伲
+伴
+伶
+伷
+伸
+伺
+似
+伽
+伾
+佀
+佃
+但
+佉
+位
+低
+住
+佐
+佑
+体
+何
+佗
+佘
+余
+佚
+佛
+作
+佝
+佞
+佟
+你
+佢
+佣
+佤
+佥
+佧
+佩
+佬
+佯
+佰
+佳
+佴
+佶
+佷
+佺
+佻
+佼
+佾
+使
+侁
+侂
+侃
+侄
+來
+侈
+侉
+例
+侍
+侏
+侑
+侔
+侗
+侘
+供
+依
+侠
+価
+侣
+侥
+侦
+侧
+侨
+侩
+侪
+侬
+侮
+侯
+侵
+便
+促
+俄
+俅
+俊
+俎
+俏
+俐
+俑
+俗
+俘
+俚
+俛
+俜
+保
+俞
+俟
+信
+俢
+俣
+俤
+俦
+俨
+俩
+俪
+俬
+俭
+修
+俯
+俱
+俳
+俶
+俸
+俺
+俾
+倅
+倌
+倍
+倏
+倒
+倓
+倔
+倕
+倘
+候
+倚
+倜
+倞
+借
+倡
+値
+倥
+倦
+倧
+倨
+倩
+倪
+倬
+倭
+倮
+倶
+债
+倻
+值
+倾
+偁
+偃
+假
+偈
+偌
+偍
+偎
+偏
+偓
+偕
+做
+停
+偠
+健
+偬
+偰
+偲
+偶
+偷
+偻
+偾
+偿
+傀
+傅
+傈
+傉
+傍
+傒
+傕
+傣
+傥
+傧
+储
+傩
+催
+傲
+傺
+傻
+働
+像
+僖
+僚
+僜
+僣
+僦
+僧
+僬
+僭
+僮
+僰
+僳
+僵
+僻
+僾
+儁
+儆
+儇
+儋
+儒
+儞
+儡
+儿
+兀
+允
+元
+兄
+充
+兆
+先
+光
+克
+免
+兎
+児
+兑
+兔
+兕
+兖
+党
+兜
+兢
+入
+全
+八
+公
+六
+兮
+兰
+共
+兲
+关
+兴
+兵
+其
+具
+典
+兹
+养
+兼
+兽
+冀
+内
+円
+冇
+冈
+冉
+册
+再
+冏
+冒
+冕
+冗
+写
+冚
+军
+农
+冠
+冢
+冤
+冥
+冧
+冨
+冬
+冮
+冯
+冰
+冲
+决
+冴
+况
+冶
+冷
+冻
+冼
+冽
+净
+凃
+凄
+准
+凇
+凉
+凊
+凋
+凌
+减
+凑
+凖
+凛
+凝
+几
+凡
+凤
+処
+凪
+凫
+凭
+凯
+凰
+凳
+凶
+凸
+凹
+出
+击
+凼
+函
+凿
+刀
+刁
+刂
+刃
+分
+切
+刈
+刊
+刋
+刍
+刎
+刑
+划
+刖
+列
+刘
+则
+刚
+创
+初
+删
+判
+刨
+利
+别
+刬
+刭
+刮
+到
+刳
+制
+刷
+券
+刹
+刺
+刻
+刽
+刿
+剀
+剁
+剂
+剃
+剅
+削
+剌
+前
+剐
+剑
+剔
+剖
+剜
+剞
+剡
+剣
+剥
+剧
+剩
+剪
+副
+割
+剽
+剿
+劂
+劄
+劈
+劏
+劓
+力
+劝
+办
+功
+加
+务
+劢
+劣
+动
+助
+努
+劫
+劬
+劭
+励
+劲
+劳
+劵
+効
+劻
+劼
+劾
+势
+勃
+勅
+勇
+勉
+勋
+勍
+勐
+勑
+勒
+勔
+勖
+勘
+募
+勠
+勤
+勰
+勲
+勳
+勷
+勺
+勾
+勿
+匀
+匂
+匄
+包
+匆
+匈
+匋
+匍
+匏
+匐
+匕
+化
+北
+匙
+匚
+匜
+匝
+匠
+匡
+匣
+匦
+匪
+匮
+匹
+区
+医
+匾
+匿
+區
+十
+千
+卅
+升
+午
+卉
+半
+卌
+卍
+华
+协
+卐
+卑
+卒
+卓
+单
+卖
+南
+単
+博
+卜
+卝
+卞
+卟
+占
+卡
+卢
+卣
+卤
+卦
+卧
+卨
+卫
+卬
+卮
+卯
+印
+危
+卲
+即
+却
+卵
+卷
+卸
+卺
+卽
+卿
+厂
+厄
+厅
+历
+厉
+压
+厌
+厍
+厓
+厔
+厕
+厘
+厚
+厝
+原
+厢
+厣
+厥
+厦
+厨
+厩
+厮
+厶
+去
+县
+叁
+参
+叆
+又
+叉
+及
+友
+双
+反
+収
+发
+叒
+叔
+叕
+取
+受
+变
+叙
+叛
+叟
+叠
+叡
+口
+古
+句
+另
+叨
+叩
+只
+叫
+召
+叭
+叮
+可
+台
+叱
+史
+右
+叵
+叶
+号
+司
+叹
+叻
+叼
+叽
+吁
+吃
+各
+吅
+吆
+吇
+合
+吉
+吊
+吋
+同
+名
+后
+吏
+吐
+向
+吒
+吓
+吔
+吕
+吖
+吗
+吙
+吚
+君
+吝
+吞
+吟
+吠
+吡
+吥
+否
+吧
+吨
+吩
+含
+听
+吭
+吮
+启
+吱
+吲
+吴
+吵
+吸
+吹
+吻
+吼
+吽
+吾
+吿
+呀
+呃
+呆
+呈
+呉
+告
+呋
+呎
+呐
+呑
+呒
+呓
+呔
+呕
+呖
+呗
+员
+呙
+呛
+呜
+呢
+呣
+呤
+呦
+周
+呪
+呬
+呯
+呱
+呲
+味
+呴
+呵
+呶
+呷
+呸
+呻
+呼
+命
+呾
+咀
+咁
+咂
+咄
+咆
+咋
+和
+咎
+咏
+咐
+咒
+咔
+咕
+咖
+咗
+咘
+咙
+咚
+咛
+咝
+咢
+咣
+咤
+咥
+咦
+咧
+咨
+咩
+咪
+咫
+咬
+咭
+咯
+咱
+咲
+咳
+咴
+咸
+咻
+咽
+咾
+咿
+哀
+品
+哂
+哄
+哆
+哇
+哈
+哉
+哋
+哌
+响
+哎
+哏
+哐
+哑
+哒
+哓
+哔
+哕
+哗
+哙
+哚
+哝
+哞
+哟
+哥
+哦
+哧
+哨
+哩
+哪
+哭
+哮
+哲
+哺
+哼
+哽
+哿
+唁
+唃
+唆
+唇
+唉
+唎
+唏
+唐
+唑
+唔
+唛
+唠
+唢
+唤
+唦
+唧
+唬
+售
+唯
+唰
+唱
+唳
+唵
+唷
+唻
+唼
+唾
+唿
+啁
+啃
+啄
+啅
+商
+啉
+啊
+啐
+啓
+啕
+啖
+啜
+啝
+啡
+啤
+啥
+啦
+啧
+啪
+啫
+啬
+啭
+啮
+啯
+啰
+啱
+啲
+啵
+啶
+啷
+啸
+啻
+啼
+啾
+喀
+喁
+喂
+喃
+善
+喆
+喇
+喈
+喉
+喊
+喋
+喎
+喏
+喑
+喔
+喘
+喙
+喜
+喝
+喟
+喦
+喧
+喰
+喱
+喳
+喵
+営
+喷
+喹
+喺
+喻
+喽
+喾
+嗄
+嗅
+嗉
+嗌
+嗍
+嗑
+嗒
+嗓
+嗔
+嗖
+嗜
+嗝
+嗞
+嗟
+嗡
+嗣
+嗤
+嗥
+嗦
+嗨
+嗪
+嗫
+嗬
+嗮
+嗯
+嗰
+嗲
+嗳
+嗵
+嗷
+嗽
+嗾
+嘀
+嘁
+嘅
+嘈
+嘉
+嘌
+嘎
+嘏
+嘘
+嘚
+嘛
+嘞
+嘟
+嘢
+嘣
+嘤
+嘦
+嘧
+嘬
+嘭
+嘱
+嘲
+嘴
+嘶
+嘹
+嘻
+嘿
+噁
+噉
+噌
+噎
+噐
+噔
+噗
+噘
+噙
+噜
+噢
+噤
+器
+噩
+噪
+噫
+噬
+噱
+噶
+噻
+噼
+嚅
+嚈
+嚎
+嚏
+嚐
+嚒
+嚓
+嚟
+嚣
+嚧
+嚩
+嚭
+嚯
+嚷
+嚼
+囊
+囍
+囔
+囖
+囗
+囘
+囚
+四
+囝
+回
+囟
+因
+囡
+团
+団
+囤
+囧
+囫
+囬
+园
+囯
+困
+囱
+囲
+図
+围
+囵
+囷
+囹
+固
+国
+图
+囿
+圃
+圄
+圆
+圈
+圉
+圊
+國
+圌
+圏
+圜
+圝
+圞
+土
+圣
+圧
+在
+圩
+圪
+圬
+圭
+圮
+圯
+地
+圳
+圹
+场
+圻
+圾
+址
+坂
+均
+坊
+坌
+坍
+坎
+坏
+坐
+坑
+块
+坚
+坛
+坜
+坝
+坞
+坟
+坠
+坡
+坤
+坦
+坨
+坩
+坪
+坫
+坬
+坭
+坯
+坳
+坷
+坻
+坼
+垂
+垃
+垄
+垅
+垆
+型
+垌
+垍
+垒
+垓
+垕
+垚
+垛
+垞
+垟
+垠
+垡
+垢
+垣
+垤
+垦
+垧
+垩
+垫
+垭
+垮
+垱
+垲
+垴
+垵
+垸
+埂
+埃
+埇
+埈
+埋
+埌
+城
+埏
+埒
+埔
+埕
+埗
+埘
+埙
+埚
+埜
+埝
+域
+埠
+埤
+埭
+埯
+埴
+埵
+埸
+培
+基
+埼
+埽
+堀
+堂
+堃
+堆
+堇
+堈
+堉
+堋
+堌
+堍
+堑
+堕
+堙
+堞
+堠
+堡
+堤
+堨
+堪
+堰
+堵
+堺
+堽
+塁
+塄
+塅
+塆
+塌
+塍
+塑
+塔
+塘
+塚
+塝
+塞
+塩
+填
+塬
+塭
+塱
+塽
+塾
+墀
+墁
+境
+墅
+墉
+墒
+墓
+墕
+増
+墘
+墙
+增
+墟
+墨
+墩
+墫
+壁
+壅
+壆
+壊
+壑
+壕
+壤
+士
+壬
+壮
+声
+壱
+売
+壳
+壶
+壸
+壹
+处
+备
+変
+复
+夏
+夑
+夔
+夕
+外
+夙
+多
+夜
+够
+夤
+夥
+大
+夨
+天
+太
+夫
+夬
+夭
+央
+夯
+失
+夲
+头
+夶
+夷
+夸
+夹
+夺
+夼
+奀
+奁
+奂
+奄
+奇
+奈
+奉
+奋
+奌
+奎
+奏
+契
+奔
+奕
+奖
+套
+奘
+奚
+奠
+奢
+奥
+奨
+奭
+女
+奴
+奶
+奸
+她
+好
+妁
+如
+妃
+妄
+妆
+妇
+妈
+妊
+妍
+妒
+妓
+妖
+妗
+妘
+妙
+妞
+妠
+妣
+妤
+妥
+妨
+妩
+妪
+妫
+妬
+妮
+妯
+妲
+妹
+妺
+妻
+妼
+妾
+姁
+姆
+姈
+姉
+姊
+始
+姐
+姑
+姒
+姓
+委
+姗
+姘
+姚
+姜
+姝
+姞
+姣
+姤
+姥
+姨
+姫
+姬
+姮
+姱
+姵
+姹
+姻
+姽
+姿
+威
+娃
+娄
+娅
+娆
+娇
+娈
+娉
+娌
+娑
+娓
+娘
+娜
+娟
+娠
+娡
+娣
+娥
+娩
+娭
+娱
+娲
+娴
+娶
+娼
+婀
+婆
+婉
+婊
+婕
+婚
+婠
+婢
+婧
+婪
+婬
+婳
+婴
+婵
+婶
+婷
+婺
+婻
+婼
+婿
+媄
+媒
+媗
+媚
+媛
+媜
+媞
+媪
+媲
+媳
+媵
+媸
+媺
+媾
+嫁
+嫂
+嫄
+嫉
+嫌
+嫒
+嫔
+嫖
+嫘
+嫚
+嫠
+嫡
+嫣
+嫦
+嫩
+嫪
+嫫
+嫰
+嫱
+嫲
+嫽
+嬅
+嬉
+嬖
+嬗
+嬛
+嬜
+嬢
+嬲
+嬴
+嬷
+嬾
+嬿
+孀
+子
+孑
+孒
+孓
+孔
+孕
+孖
+字
+存
+孙
+孚
+孛
+孜
+孝
+孟
+孢
+季
+孤
+孥
+学
+孩
+孪
+孬
+孰
+孱
+孳
+孵
+孺
+孽
+宀
+宁
+它
+宄
+宅
+宇
+守
+安
+宋
+完
+宍
+宏
+宓
+宕
+宗
+官
+宙
+定
+宛
+宜
+宝
+实
+実
+宠
+审
+客
+宣
+室
+宥
+宦
+宪
+宫
+宬
+宰
+害
+宴
+宵
+家
+宸
+容
+宽
+宾
+宿
+寀
+寂
+寃
+寄
+寅
+密
+寇
+富
+寐
+寒
+寓
+寔
+寘
+寛
+寝
+寞
+察
+寡
+寤
+寥
+寨
+寮
+寯
+寰
+寳
+寸
+对
+寺
+寻
+导
+対
+寿
+封
+専
+射
+尅
+将
+尉
+尊
+對
+小
+尐
+少
+尒
+尓
+尔
+尕
+尖
+尘
+尙
+尚
+尛
+尜
+尝
+尢
+尤
+尧
+尨
+尪
+尬
+就
+尴
+尸
+尹
+尺
+尻
+尼
+尽
+尾
+尿
+局
+屁
+层
+屃
+屄
+居
+屈
+屉
+届
+屋
+屌
+屍
+屎
+屏
+屐
+屑
+展
+屙
+属
+屠
+屡
+屣
+履
+屦
+屮
+屯
+山
+屹
+屺
+屾
+屿
+岀
+岁
+岂
+岈
+岌
+岐
+岑
+岔
+岕
+岖
+岗
+岘
+岙
+岚
+岛
+岜
+岞
+岢
+岣
+岩
+岫
+岬
+岭
+岱
+岳
+岵
+岷
+岸
+岺
+岽
+岿
+峁
+峄
+峇
+峋
+峒
+峕
+峙
+峠
+峡
+峣
+峤
+峥
+峦
+峨
+峩
+峪
+峭
+峯
+峰
+峻
+崀
+崁
+崂
+崃
+崄
+崆
+崇
+崎
+崐
+崑
+崔
+崖
+崚
+崛
+崞
+崟
+崤
+崦
+崧
+崩
+崭
+崮
+崴
+崽
+崾
+嵇
+嵊
+嵋
+嵌
+嵎
+嵖
+嵗
+嵘
+嵚
+嵛
+嵝
+嵩
+嵬
+嵯
+嵴
+嶂
+嶋
+嶙
+嶝
+嶲
+嶷
+巂
+巅
+巇
+巉
+巍
+巎
+巘
+巜
+川
+州
+巡
+巢
+巣
+工
+左
+巧
+巨
+巩
+巫
+差
+巯
+己
+已
+巳
+巴
+巷
+巻
+巽
+巾
+巿
+币
+市
+布
+帅
+帆
+师
+希
+帏
+帐
+帑
+帔
+帕
+帖
+帘
+帙
+帚
+帛
+帜
+帝
+带
+帧
+席
+帮
+帯
+帰
+帷
+常
+帻
+帼
+帽
+幂
+幄
+幅
+幌
+幔
+幕
+幛
+幞
+幡
+幢
+干
+平
+年
+幵
+并
+幷
+幸
+幺
+幻
+幼
+幽
+广
+庀
+庁
+広
+庄
+庆
+庇
+床
+庋
+序
+庐
+庑
+库
+应
+底
+庖
+店
+庙
+庚
+府
+庞
+废
+庠
+庤
+庥
+度
+座
+庭
+庵
+庶
+康
+庸
+庹
+庾
+廆
+廉
+廊
+廋
+廌
+廑
+廒
+廓
+廕
+廖
+廙
+廛
+廞
+廨
+廪
+廯
+延
+廷
+建
+廻
+廼
+廾
+廿
+开
+弁
+异
+弃
+弄
+弇
+弈
+弊
+弋
+式
+弐
+弑
+弓
+引
+弗
+弘
+弛
+弟
+张
+弢
+弥
+弦
+弧
+弩
+弭
+弯
+弱
+弹
+强
+弼
+弾
+彀
+归
+当
+录
+彖
+彗
+彘
+彝
+彟
+彡
+形
+彤
+彦
+彧
+彩
+彪
+彬
+彭
+彰
+影
+彳
+彵
+彷
+役
+彻
+彼
+往
+征
+徂
+径
+待
+徇
+很
+徉
+徊
+律
+徐
+徒
+従
+徕
+得
+徘
+徙
+徜
+御
+徧
+徨
+循
+徭
+微
+徳
+徴
+徵
+德
+徼
+徽
+心
+忄
+必
+忆
+忉
+忌
+忍
+忏
+忐
+忑
+忒
+忖
+志
+忘
+忙
+応
+忝
+忞
+忠
+忡
+忤
+忧
+忪
+快
+忭
+忱
+念
+忸
+忻
+忽
+忾
+忿
+怀
+态
+怂
+怃
+怄
+怅
+怆
+怍
+怎
+怏
+怒
+怔
+怕
+怖
+怙
+怛
+怜
+思
+怠
+怡
+急
+怦
+性
+怨
+怩
+怪
+怫
+怯
+怱
+怳
+怵
+怹
+总
+怼
+怿
+恁
+恂
+恃
+恋
+恍
+恏
+恐
+恒
+恕
+恙
+恚
+恠
+恢
+恣
+恤
+恨
+恩
+恪
+恫
+恬
+恭
+息
+恰
+恳
+恵
+恶
+恸
+恹
+恺
+恻
+恼
+恽
+恿
+悃
+悄
+悉
+悌
+悍
+悒
+悔
+悖
+悚
+悛
+悝
+悟
+悠
+患
+悦
+您
+悩
+悪
+悫
+悬
+悭
+悯
+悰
+悱
+悲
+悳
+悴
+悸
+悻
+悼
+情
+惆
+惇
+惊
+惋
+惑
+惔
+惕
+惘
+惚
+惛
+惜
+惝
+惟
+惠
+惢
+惣
+惦
+惧
+惨
+惩
+惪
+惫
+惬
+惭
+惮
+惯
+惰
+想
+惴
+惶
+惹
+惺
+愀
+愁
+愆
+愈
+愉
+愍
+愎
+意
+愔
+愕
+愚
+感
+愠
+愣
+愤
+愦
+愧
+愫
+愬
+愰
+愽
+愿
+慆
+慈
+慊
+慌
+慎
+慑
+慕
+慜
+慝
+慢
+慥
+慧
+慨
+慰
+慵
+慷
+慾
+憋
+憍
+憎
+憔
+憙
+憧
+憨
+憩
+憬
+憷
+憺
+憾
+懂
+懈
+懊
+懋
+懐
+懑
+懒
+懔
+懦
+懮
+懵
+懽
+懿
+戆
+戈
+戊
+戋
+戌
+戍
+戎
+戏
+成
+我
+戒
+戓
+戕
+或
+戗
+战
+戚
+戛
+戟
+戡
+戢
+戥
+戦
+截
+戬
+戮
+戯
+戳
+戴
+户
+戻
+戽
+戾
+房
+所
+扁
+扃
+扆
+扇
+扈
+扉
+手
+扌
+才
+扎
+扑
+扒
+打
+扔
+托
+扛
+扞
+扣
+扥
+扦
+执
+扩
+扪
+扫
+扬
+扭
+扮
+扯
+扰
+扳
+扶
+批
+扼
+扽
+找
+承
+技
+抃
+抄
+抉
+把
+抑
+抒
+抓
+抔
+投
+抖
+抗
+折
+抚
+抛
+抜
+抟
+抠
+抡
+抢
+护
+报
+抧
+抨
+披
+抬
+抱
+抳
+抵
+抹
+抺
+抻
+押
+抽
+抿
+拂
+拄
+担
+拆
+拇
+拈
+拉
+拊
+拌
+拍
+拎
+拏
+拐
+拒
+拓
+拔
+拖
+拗
+拘
+拙
+招
+拜
+拟
+拢
+拣
+拥
+拦
+拧
+拨
+择
+括
+拭
+拮
+拯
+拱
+拳
+拴
+拶
+拷
+拼
+拽
+拾
+拿
+挀
+持
+挂
+指
+挈
+按
+挎
+挑
+挒
+挖
+挚
+挛
+挝
+挞
+挟
+挠
+挡
+挢
+挣
+挤
+挥
+挨
+挪
+挫
+振
+挲
+挹
+挺
+挻
+挼
+挽
+捂
+捃
+捅
+捆
+捉
+捋
+捌
+捍
+捎
+捏
+捐
+捕
+捜
+捞
+损
+捡
+换
+捣
+捧
+捩
+捭
+据
+捯
+捱
+捶
+捷
+捺
+捻
+捽
+掀
+掂
+掇
+授
+掉
+掊
+掌
+掎
+掏
+掐
+排
+掖
+掘
+掞
+掠
+探
+掣
+掤
+接
+控
+推
+掩
+措
+掬
+掭
+掮
+掰
+掲
+掳
+掴
+掷
+掸
+掺
+掼
+掾
+揄
+揆
+揉
+揍
+描
+提
+插
+揖
+揠
+握
+揣
+揩
+揪
+揭
+揲
+援
+揵
+揶
+揸
+揺
+揽
+揾
+揿
+搀
+搁
+搂
+搅
+搋
+搏
+搐
+搓
+搔
+搜
+搞
+搠
+搡
+搢
+搥
+搦
+搧
+搨
+搪
+搬
+搭
+搴
+搵
+携
+搽
+搿
+摁
+摄
+摅
+摆
+摇
+摈
+摊
+摒
+摔
+摘
+摛
+摞
+摧
+摩
+摭
+摸
+摹
+摺
+摽
+撂
+撃
+撄
+撅
+撇
+撑
+撒
+撕
+撘
+撙
+撝
+撞
+撤
+撩
+撬
+播
+撮
+撰
+撵
+撷
+撸
+撺
+撼
+擀
+擂
+擅
+操
+擎
+擒
+擗
+擘
+擞
+擢
+擤
+擦
+擫
+擿
+攀
+攒
+攘
+攞
+攥
+攫
+支
+攲
+攴
+攵
+收
+攸
+改
+攻
+攽
+放
+政
+故
+效
+敉
+敌
+敎
+敏
+救
+敔
+敕
+敖
+教
+敚
+敛
+敝
+敞
+敢
+散
+敦
+敫
+敬
+数
+敲
+整
+敷
+敻
+文
+斉
+斋
+斌
+斎
+斐
+斑
+斓
+斗
+料
+斛
+斜
+斝
+斟
+斡
+斤
+斥
+斧
+斩
+斫
+断
+斯
+新
+斲
+斶
+方
+於
+施
+旁
+旃
+旄
+旅
+旆
+旋
+旌
+旎
+族
+旒
+旖
+旗
+旛
+无
+既
+旣
+日
+旦
+旧
+旨
+早
+旬
+旭
+旮
+旯
+旰
+旱
+旳
+旴
+时
+旷
+旸
+旺
+旻
+旼
+昀
+昂
+昃
+昆
+昇
+昉
+昊
+昌
+明
+昏
+易
+昔
+昕
+昙
+昚
+昝
+昞
+星
+映
+春
+昧
+昨
+昪
+昫
+昭
+是
+昰
+昱
+昳
+昴
+昵
+昶
+昺
+昼
+显
+晁
+時
+晃
+晄
+晋
+晌
+晏
+晒
+晓
+晔
+晕
+晖
+晗
+晙
+晚
+晞
+晟
+晡
+晢
+晤
+晦
+晧
+晨
+晩
+晬
+普
+景
+晰
+晳
+晴
+晶
+晷
+晸
+智
+晻
+晾
+暁
+暂
+暄
+暇
+暌
+暍
+暎
+暐
+暑
+暕
+暖
+暗
+暝
+暠
+暧
+暨
+暮
+暴
+暸
+暹
+暻
+暾
+曈
+曌
+曕
+曙
+曛
+曜
+曝
+曦
+曩
+曰
+曱
+曲
+曳
+更
+曷
+曹
+曺
+曼
+曽
+曾
+替
+最
+會
+朅
+月
+有
+朊
+朋
+服
+朏
+朐
+朓
+朔
+朕
+朗
+望
+朝
+期
+朥
+朦
+木
+未
+末
+本
+札
+术
+朱
+朲
+朴
+朵
+机
+朽
+朿
+杀
+杂
+权
+杆
+杈
+杉
+杌
+李
+杏
+材
+村
+杓
+杖
+杜
+杞
+束
+杠
+条
+来
+杧
+杨
+杪
+杭
+杮
+杯
+杰
+杲
+杳
+杵
+杷
+杼
+松
+板
+极
+构
+枇
+枉
+枋
+枏
+析
+枓
+枕
+林
+枘
+枚
+果
+枝
+枞
+枟
+枠
+枢
+枣
+枥
+枧
+枨
+枪
+枫
+枭
+枯
+枰
+枱
+枳
+枵
+架
+枷
+枸
+枹
+柁
+柃
+柄
+柊
+柏
+某
+柑
+柒
+染
+柔
+柘
+柙
+柚
+柜
+柝
+柞
+柟
+柠
+柢
+查
+柩
+柬
+柯
+柰
+柱
+柳
+柴
+柷
+査
+柽
+柾
+柿
+栀
+栃
+栄
+栅
+标
+栈
+栉
+栊
+栋
+栌
+栎
+栏
+树
+栒
+栓
+栖
+栗
+栝
+栞
+栟
+校
+栢
+栩
+株
+栱
+栲
+栳
+栴
+样
+核
+根
+栻
+格
+栽
+栾
+栿
+桀
+桁
+桂
+桃
+桄
+桅
+框
+案
+桉
+桌
+桎
+桐
+桑
+桓
+桔
+桕
+桖
+桜
+桠
+桡
+桢
+档
+桤
+桥
+桦
+桧
+桨
+桩
+桫
+桴
+桶
+桷
+梁
+梃
+梅
+梆
+梏
+梓
+梗
+梢
+梣
+梦
+梧
+梨
+梭
+梯
+械
+梳
+梵
+梶
+梼
+梿
+检
+棂
+棉
+棋
+棍
+棐
+棒
+棕
+棘
+棚
+棠
+棣
+棨
+棪
+棫
+森
+棰
+棱
+棵
+棹
+棺
+棻
+棼
+椀
+椁
+椅
+椇
+椋
+植
+椎
+椐
+椒
+椛
+検
+椟
+椠
+椤
+椪
+椭
+椰
+椴
+椹
+椽
+椿
+楂
+楔
+楗
+楙
+楚
+楛
+楝
+楞
+楠
+楢
+楣
+楤
+楦
+楩
+楪
+楫
+業
+楮
+楯
+楶
+楷
+楸
+楹
+楼
+楽
+榀
+概
+榃
+榄
+榆
+榇
+榈
+榉
+榊
+榎
+榔
+榕
+榖
+榘
+榛
+榜
+榧
+榨
+榫
+榭
+榱
+榴
+榷
+榻
+榼
+槁
+槃
+槅
+槊
+槌
+槎
+槐
+槑
+槔
+様
+槙
+槚
+槛
+槜
+槟
+槠
+槭
+槱
+槲
+槵
+槻
+槽
+槿
+樊
+樋
+樗
+樘
+樛
+樟
+模
+樨
+権
+横
+樫
+樯
+樱
+樵
+樽
+樾
+橄
+橇
+橐
+橘
+橙
+橚
+橛
+橡
+橥
+橦
+橱
+橹
+橼
+檀
+檄
+檇
+檎
+檐
+檗
+檞
+檠
+檩
+檫
+檬
+檵
+櫂
+櫆
+欠
+次
+欢
+欣
+欤
+欧
+欲
+欷
+欸
+欹
+欺
+欻
+款
+歃
+歆
+歇
+歉
+歌
+歔
+歘
+歙
+止
+正
+此
+步
+武
+歧
+歩
+歪
+歳
+歴
+歹
+歺
+死
+歼
+殁
+殂
+殃
+殄
+殆
+殇
+殉
+殊
+残
+殍
+殑
+殒
+殓
+殖
+殚
+殛
+殡
+殢
+殪
+殳
+殴
+段
+殷
+殿
+毁
+毂
+毅
+毋
+毌
+母
+毎
+每
+毐
+毑
+毒
+毓
+比
+毕
+毖
+毗
+毘
+毙
+毛
+毡
+毫
+毯
+毳
+毵
+毽
+氀
+氂
+氅
+氆
+氇
+氍
+氏
+氐
+民
+氓
+气
+氖
+気
+氘
+氙
+氚
+氛
+氟
+氡
+氢
+氤
+氦
+氧
+氨
+氩
+氪
+氮
+氯
+氰
+氲
+水
+氵
+氷
+永
+氹
+氺
+氽
+氿
+汀
+汁
+求
+汆
+汇
+汉
+汊
+汐
+汕
+汖
+汗
+汛
+汜
+汝
+汞
+江
+池
+污
+汤
+汧
+汨
+汩
+汪
+汭
+汰
+汲
+汴
+汶
+汸
+汹
+汽
+汾
+沁
+沂
+沃
+沄
+沅
+沆
+沇
+沈
+沉
+沌
+沏
+沐
+沒
+沓
+沔
+沕
+沙
+沚
+沛
+沟
+没
+沢
+沣
+沤
+沥
+沦
+沧
+沨
+沩
+沪
+沫
+沬
+沭
+沮
+沱
+河
+沴
+沵
+沸
+油
+治
+沼
+沽
+沾
+沿
+泃
+泄
+泅
+泇
+泉
+泊
+泌
+泐
+泓
+泔
+法
+泖
+泗
+泚
+泛
+泞
+泠
+泡
+波
+泣
+泥
+注
+泩
+泪
+泫
+泮
+泯
+泰
+泱
+泳
+泵
+泷
+泸
+泺
+泻
+泼
+泽
+泾
+洁
+洄
+洇
+洈
+洊
+洋
+洌
+洎
+洑
+洒
+洗
+洙
+洛
+洞
+洢
+洣
+津
+洧
+洨
+洪
+洫
+洮
+洱
+洲
+洳
+洵
+洸
+洹
+洺
+活
+洼
+洽
+派
+流
+浃
+浄
+浅
+浆
+浇
+浈
+浉
+浊
+测
+浍
+济
+浏
+浐
+浑
+浒
+浓
+浔
+浙
+浚
+浛
+浜
+浞
+浠
+浡
+浣
+浥
+浦
+浩
+浪
+浬
+浮
+浯
+浴
+海
+浸
+浼
+涂
+涅
+消
+涉
+涌
+涎
+涐
+涑
+涓
+涔
+涕
+涘
+涙
+涛
+涝
+涞
+涟
+涠
+涡
+涢
+涣
+涤
+润
+涧
+涨
+涩
+涪
+涫
+涮
+涯
+液
+涴
+涵
+涸
+涿
+淀
+淄
+淅
+淆
+淇
+淋
+淌
+淏
+淑
+淖
+淘
+淙
+淛
+淝
+淞
+淠
+淡
+淤
+淦
+淩
+淫
+淬
+淮
+淯
+深
+淳
+混
+淸
+淹
+添
+淼
+渀
+渃
+清
+済
+渉
+渊
+渋
+渌
+渍
+渎
+渐
+渑
+渔
+渕
+渖
+渗
+渚
+渝
+渟
+渠
+渡
+渣
+渤
+渥
+温
+渫
+渭
+港
+渲
+渴
+游
+渺
+渼
+湃
+湄
+湉
+湋
+湍
+湎
+湑
+湓
+湔
+湖
+湘
+湛
+湜
+湟
+湣
+湫
+湮
+湲
+湳
+湴
+湾
+湿
+満
+溁
+溃
+溅
+溆
+溇
+溉
+溍
+溏
+源
+溘
+溜
+溞
+溟
+溢
+溥
+溦
+溧
+溪
+溯
+溱
+溲
+溴
+溶
+溷
+溺
+溽
+滁
+滂
+滃
+滆
+滇
+滈
+滉
+滋
+滍
+滏
+滑
+滓
+滔
+滕
+滗
+滘
+滙
+滚
+滝
+滞
+滟
+滠
+满
+滢
+滤
+滥
+滦
+滨
+滩
+滴
+滹
+漀
+漂
+漆
+漈
+漉
+漏
+漓
+演
+漕
+漠
+漩
+漪
+漫
+漭
+漯
+漱
+漳
+漶
+漷
+漾
+潆
+潇
+潋
+潍
+潏
+潘
+潜
+潞
+潟
+潢
+潦
+潭
+潮
+潲
+潴
+潸
+潺
+潼
+潽
+潾
+澂
+澄
+澈
+澉
+澌
+澍
+澎
+澐
+澔
+澜
+澡
+澥
+澧
+澪
+澳
+澶
+澹
+激
+濂
+濆
+濉
+濊
+濑
+濒
+濙
+濛
+濞
+濠
+濡
+濩
+濬
+濮
+濯
+瀀
+瀍
+瀑
+瀚
+瀛
+瀞
+瀣
+瀬
+瀹
+瀼
+灌
+灏
+灞
+火
+灬
+灭
+灯
+灰
+灵
+灶
+灸
+灼
+灾
+灿
+炀
+炁
+炅
+炆
+炉
+炊
+炎
+炒
+炔
+炕
+炖
+炘
+炙
+炜
+炝
+炟
+炤
+炩
+炫
+炬
+炭
+炮
+炯
+炱
+炳
+炷
+炸
+点
+為
+炻
+炼
+炽
+烀
+烁
+烂
+烃
+烈
+烊
+烎
+烔
+烘
+烙
+烛
+烜
+烝
+烟
+烤
+烦
+烧
+烨
+烩
+烫
+烬
+热
+烯
+烷
+烹
+烺
+烽
+焉
+焊
+焌
+焐
+焓
+焕
+焖
+焗
+焘
+焙
+焚
+焜
+焞
+焦
+焮
+焯
+焰
+焱
+然
+焼
+煅
+煇
+煊
+煌
+煎
+煐
+煕
+煖
+煚
+煜
+煞
+煤
+煦
+照
+煨
+煮
+煲
+煳
+煴
+煸
+煺
+煽
+熄
+熇
+熊
+熏
+熔
+熘
+熙
+熜
+熟
+熠
+熥
+熨
+熬
+熳
+熵
+熹
+熺
+燃
+燊
+燋
+燎
+燏
+燔
+燕
+燚
+燠
+燥
+燧
+燮
+燹
+燻
+燿
+爀
+爆
+爇
+爨
+爪
+爬
+爰
+爱
+爲
+爵
+父
+爷
+爸
+爹
+爻
+爽
+爿
+牀
+牁
+牂
+片
+版
+牋
+牌
+牍
+牐
+牒
+牖
+牙
+牛
+牝
+牟
+牠
+牡
+牢
+牤
+牦
+牧
+物
+牯
+牲
+牵
+特
+牺
+牻
+牾
+犀
+犁
+犄
+犇
+犊
+犍
+犏
+犒
+犟
+犨
+犬
+犭
+犯
+犰
+犴
+状
+犷
+犸
+犹
+犼
+犽
+狁
+狂
+狃
+狄
+狈
+狌
+狍
+狎
+狐
+狒
+狗
+狙
+狛
+狝
+狞
+狠
+狡
+狨
+狩
+独
+狭
+狮
+狯
+狰
+狱
+狲
+狳
+狴
+狷
+狸
+狻
+狼
+猀
+猁
+猃
+猄
+猇
+猊
+猋
+猎
+猓
+猕
+猖
+猗
+猛
+猜
+猝
+猞
+猟
+猡
+猢
+猥
+猩
+猪
+猫
+猬
+献
+猰
+猱
+猴
+猷
+猹
+猾
+猿
+獍
+獐
+獒
+獗
+獠
+獣
+獬
+獭
+獴
+獾
+玁
+玄
+率
+玉
+王
+玎
+玏
+玑
+玕
+玖
+玗
+玘
+玙
+玚
+玛
+玟
+玠
+玡
+玢
+玥
+玦
+玧
+玩
+玫
+玭
+玮
+环
+现
+玲
+玳
+玷
+玹
+玺
+玻
+珀
+珂
+珅
+珈
+珉
+珊
+珍
+珏
+珐
+珑
+珖
+珙
+珝
+珞
+珠
+珣
+珥
+珦
+珧
+珩
+珪
+班
+珰
+珲
+珵
+珹
+珺
+珽
+琀
+球
+琅
+理
+琇
+琉
+琊
+琋
+琍
+琎
+琏
+琐
+琚
+琛
+琢
+琤
+琥
+琦
+琨
+琪
+琬
+琮
+琯
+琰
+琲
+琳
+琴
+琵
+琶
+琹
+琼
+瑀
+瑁
+瑄
+瑆
+瑊
+瑒
+瑕
+瑗
+瑙
+瑚
+瑛
+瑜
+瑞
+瑟
+瑠
+瑢
+瑧
+瑨
+瑭
+瑰
+瑱
+瑶
+瑷
+瑸
+瑺
+瑾
+璀
+璁
+璂
+璃
+璆
+璇
+璈
+璋
+璎
+璐
+璘
+璜
+璞
+璟
+璠
+璧
+璨
+璩
+璪
+璮
+璲
+璺
+璿
+瓌
+瓒
+瓘
+瓛
+瓜
+瓞
+瓟
+瓠
+瓢
+瓣
+瓤
+瓦
+瓮
+瓯
+瓴
+瓶
+瓷
+瓿
+甃
+甄
+甍
+甏
+甑
+甓
+甗
+甘
+甙
+甚
+甜
+生
+甡
+產
+甥
+用
+甩
+甪
+甫
+甬
+甭
+甯
+田
+由
+甲
+申
+电
+男
+甸
+町
+画
+甾
+畀
+畅
+畈
+畊
+畋
+界
+畎
+畏
+畑
+畔
+留
+畚
+畛
+畜
+畠
+畤
+略
+畦
+番
+畯
+畲
+畴
+畸
+畹
+畿
+疃
+疆
+疋
+疍
+疎
+疏
+疑
+疒
+疔
+疖
+疗
+疙
+疚
+疝
+疟
+疠
+疡
+疣
+疤
+疥
+疫
+疬
+疮
+疯
+疰
+疱
+疲
+疳
+疴
+疵
+疸
+疹
+疼
+疽
+疾
+痂
+痄
+病
+症
+痈
+痉
+痊
+痍
+痒
+痔
+痕
+痖
+痘
+痛
+痞
+痢
+痣
+痤
+痦
+痧
+痨
+痩
+痪
+痫
+痰
+痱
+痴
+痹
+痼
+痿
+瘀
+瘁
+瘅
+瘆
+瘊
+瘌
+瘐
+瘕
+瘖
+瘗
+瘘
+瘙
+瘛
+瘟
+瘠
+瘢
+瘤
+瘥
+瘦
+瘩
+瘪
+瘫
+瘰
+瘳
+瘴
+瘵
+瘸
+瘼
+瘾
+瘿
+癀
+癃
+癌
+癍
+癎
+癒
+癔
+癖
+癜
+癞
+癣
+癫
+癯
+癸
+発
+登
+發
+白
+百
+癿
+皂
+的
+皆
+皇
+皈
+皋
+皎
+皐
+皑
+皒
+皓
+皕
+皖
+皙
+皛
+皝
+皞
+皤
+皦
+皮
+皱
+皲
+皴
+皿
+盂
+盃
+盅
+盆
+盈
+盉
+益
+盌
+盍
+盎
+盏
+盐
+监
+盒
+盔
+盖
+盗
+盘
+盛
+盝
+盟
+盥
+盦
+盨
+盩
+目
+盯
+盱
+盲
+直
+相
+盹
+盼
+盾
+眀
+省
+眄
+眇
+眈
+眉
+看
+県
+眙
+眚
+眛
+眞
+真
+眠
+眦
+眨
+眩
+眬
+眭
+眯
+眵
+眶
+眷
+眸
+眺
+眼
+着
+睁
+睇
+睐
+睑
+睒
+睚
+睛
+睡
+睢
+督
+睥
+睦
+睨
+睪
+睫
+睬
+睱
+睹
+睺
+睽
+睾
+睿
+瞀
+瞄
+瞅
+瞋
+瞌
+瞎
+瞑
+瞒
+瞟
+瞠
+瞢
+瞥
+瞧
+瞩
+瞪
+瞬
+瞭
+瞰
+瞳
+瞻
+瞽
+瞿
+矍
+矗
+矛
+矜
+矞
+矢
+矣
+知
+矧
+矩
+矫
+矬
+短
+矮
+石
+矶
+矸
+矽
+矾
+矿
+砀
+码
+砂
+砉
+砌
+砍
+砑
+砒
+研
+砕
+砖
+砗
+砚
+砜
+砝
+砟
+砢
+砣
+砥
+砦
+砧
+砩
+砬
+砭
+砰
+砳
+破
+砵
+砷
+砸
+砹
+砺
+砻
+砼
+砾
+础
+硅
+硇
+硌
+硎
+硏
+硐
+硒
+硔
+硕
+硖
+硗
+硙
+硚
+硝
+硪
+硫
+硬
+确
+硷
+硼
+碁
+碇
+碉
+碌
+碍
+碎
+碏
+碑
+碓
+碗
+碘
+碚
+碛
+碜
+碟
+碡
+碣
+碥
+碧
+碰
+碱
+碲
+碳
+碴
+碶
+碹
+碾
+磁
+磅
+磉
+磊
+磋
+磐
+磔
+磕
+磙
+磜
+磡
+磦
+磨
+磬
+磲
+磴
+磷
+磺
+磻
+磾
+礁
+礅
+礐
+礓
+礞
+礤
+礴
+示
+礻
+礼
+礽
+社
+祀
+祁
+祂
+祆
+祇
+祈
+祉
+祊
+祎
+祏
+祐
+祓
+祔
+祖
+祗
+祘
+祚
+祛
+祜
+祝
+神
+祟
+祠
+祢
+祥
+祧
+票
+祭
+祯
+祲
+祷
+祸
+祹
+祺
+祼
+祾
+禀
+禁
+禄
+禅
+禇
+禊
+禋
+福
+禑
+禔
+禖
+禘
+禚
+禛
+禟
+禤
+禧
+禩
+禳
+禵
+禹
+禺
+离
+禽
+禾
+秀
+私
+秂
+秃
+秆
+秉
+秋
+种
+科
+秒
+秕
+秘
+租
+秣
+秤
+秦
+秧
+秩
+秫
+秬
+秭
+积
+称
+秸
+移
+秽
+秾
+稀
+稃
+程
+稍
+税
+稔
+稗
+稙
+稚
+稞
+稠
+稣
+稲
+稳
+稷
+稹
+稻
+稼
+稽
+稿
+穂
+穆
+穉
+穏
+穑
+穗
+穣
+穰
+穴
+究
+穷
+穹
+空
+穿
+突
+窃
+窄
+窅
+窆
+窈
+窊
+窋
+窍
+窑
+窒
+窓
+窕
+窖
+窗
+窘
+窜
+窝
+窟
+窠
+窣
+窥
+窦
+窨
+窭
+窰
+窳
+窸
+窿
+立
+竑
+竖
+站
+竜
+竝
+竞
+竟
+章
+竣
+童
+竦
+竭
+端
+竹
+竺
+竽
+竿
+笃
+笄
+笆
+笈
+笊
+笋
+笏
+笑
+笔
+笕
+笙
+笛
+笞
+笠
+笤
+笥
+符
+笨
+笪
+笫
+第
+笮
+笱
+笳
+笸
+笹
+笺
+笼
+笾
+筅
+筇
+等
+筊
+筋
+筌
+筏
+筐
+筑
+筒
+答
+策
+筘
+筚
+筛
+筜
+筝
+筠
+筭
+筮
+筯
+筱
+筲
+筴
+筵
+筷
+筹
+筼
+签
+简
+箅
+箌
+箍
+箐
+箓
+箔
+箕
+算
+箜
+箝
+管
+箦
+箧
+箨
+箩
+箪
+箫
+箬
+箭
+箱
+箴
+箸
+篁
+篆
+篇
+篌
+篑
+篓
+篙
+篚
+篝
+篡
+篥
+篦
+篪
+篮
+篯
+篱
+篷
+篼
+篾
+簃
+簇
+簋
+簌
+簏
+簕
+簖
+簟
+簠
+簦
+簧
+簪
+簰
+簸
+簿
+籀
+籁
+籍
+籓
+籙
+米
+籴
+籺
+类
+籼
+籽
+粄
+粉
+粑
+粒
+粕
+粗
+粘
+粜
+粝
+粞
+粟
+粢
+粤
+粥
+粦
+粧
+粪
+粬
+粮
+粱
+粲
+粳
+粹
+粼
+粽
+精
+粿
+糁
+糅
+糊
+糌
+糍
+糕
+糖
+糗
+糙
+糜
+糟
+糠
+糨
+糬
+糯
+糸
+系
+紊
+紘
+素
+索
+紧
+紫
+紬
+紮
+累
+経
+絜
+絪
+絮
+絵
+絶
+絷
+絺
+綎
+綖
+継
+続
+綝
+綦
+綫
+綮
+総
+緑
+緾
+縁
+縂
+縠
+縢
+縻
+繁
+繇
+繋
+繸
+繻
+纁
+纂
+纚
+纛
+纟
+纠
+纡
+红
+纣
+纤
+纥
+约
+级
+纨
+纩
+纪
+纫
+纬
+纭
+纮
+纯
+纰
+纱
+纲
+纳
+纵
+纶
+纷
+纸
+纹
+纺
+纻
+纽
+纾
+线
+绀
+绁
+绂
+练
+组
+绅
+细
+织
+终
+绉
+绊
+绌
+绍
+绎
+经
+绐
+绑
+绒
+结
+绔
+绕
+绗
+绘
+给
+绚
+绛
+络
+绝
+绞
+统
+绠
+绡
+绢
+绣
+绥
+绦
+继
+绨
+绩
+绪
+绫
+续
+绮
+绯
+绰
+绱
+绲
+绳
+维
+绵
+绶
+绷
+绸
+绹
+绺
+绻
+综
+绽
+绾
+绿
+缀
+缁
+缂
+缃
+缄
+缅
+缆
+缇
+缈
+缉
+缊
+缋
+缌
+缍
+缎
+缐
+缑
+缒
+缓
+缔
+缕
+编
+缗
+缘
+缙
+缚
+缛
+缜
+缝
+缟
+缠
+缡
+缢
+缣
+缤
+缥
+缦
+缧
+缨
+缩
+缪
+缫
+缬
+缭
+缮
+缯
+缰
+缱
+缲
+缳
+缴
+缵
+缶
+缷
+缸
+缺
+罂
+罃
+罄
+罅
+罍
+罐
+网
+罔
+罕
+罗
+罘
+罚
+罝
+罟
+罠
+罡
+罢
+罣
+罥
+罨
+罩
+罪
+置
+罱
+署
+罴
+罹
+罽
+罾
+羁
+羊
+羌
+美
+羑
+羔
+羕
+羚
+羝
+羞
+羟
+羡
+羣
+群
+羧
+羮
+羯
+羰
+羲
+羸
+羹
+羼
+羽
+羿
+翀
+翁
+翃
+翅
+翈
+翊
+翌
+翎
+翔
+翕
+翘
+翙
+翚
+翛
+翟
+翠
+翡
+翥
+翦
+翩
+翫
+翮
+翰
+翱
+翳
+翻
+翼
+翾
+耀
+老
+考
+耄
+者
+耆
+耋
+而
+耍
+耐
+耒
+耔
+耕
+耗
+耘
+耙
+耜
+耦
+耧
+耨
+耩
+耪
+耳
+耵
+耶
+耷
+耸
+耻
+耽
+耿
+聂
+聃
+聆
+聊
+聋
+职
+聍
+聒
+联
+聕
+聘
+聚
+聡
+聩
+聪
+聴
+聼
+聿
+肃
+肄
+肆
+肇
+肉
+肋
+肌
+肓
+肖
+肘
+肚
+肛
+肜
+肝
+肟
+肠
+股
+肢
+肤
+肥
+肩
+肪
+肫
+肮
+肯
+肱
+育
+肴
+肸
+肺
+肼
+肽
+肾
+肿
+胀
+胁
+胂
+胃
+胄
+胆
+背
+胍
+胎
+胖
+胗
+胙
+胚
+胛
+胜
+胝
+胞
+胠
+胡
+胤
+胥
+胧
+胨
+胪
+胫
+胬
+胭
+胯
+胰
+胱
+胳
+胴
+胶
+胸
+胺
+胼
+能
+脁
+脂
+脆
+脇
+脉
+脊
+脍
+脏
+脐
+脑
+脒
+脓
+脔
+脖
+脘
+脚
+脞
+脢
+脩
+脬
+脯
+脱
+脲
+脳
+脷
+脸
+脾
+脿
+腆
+腈
+腊
+腋
+腌
+腐
+腑
+腓
+腔
+腕
+腘
+腙
+腚
+腠
+腥
+腧
+腩
+腭
+腮
+腰
+腱
+腴
+腹
+腺
+腻
+腼
+腾
+腿
+膀
+膂
+膈
+膊
+膏
+膑
+膘
+膛
+膜
+膝
+膦
+膨
+膳
+膺
+膻
+臀
+臁
+臂
+臃
+臆
+臊
+臌
+臑
+臓
+臜
+臞
+臣
+臧
+自
+臬
+臭
+至
+致
+臵
+臻
+臼
+臾
+舀
+舁
+舂
+舄
+舅
+舆
+舌
+舍
+舎
+舐
+舒
+舔
+舖
+舘
+舛
+舜
+舞
+舟
+舡
+舢
+舣
+舨
+航
+舫
+般
+舯
+舰
+舱
+舲
+舳
+舴
+舵
+舶
+舷
+舸
+船
+舺
+舻
+舾
+艄
+艇
+艉
+艋
+艏
+艘
+艟
+艨
+艮
+良
+艰
+色
+艳
+艹
+艺
+艽
+艾
+艿
+节
+芃
+芄
+芈
+芊
+芋
+芍
+芎
+芏
+芑
+芒
+芗
+芘
+芙
+芜
+芝
+芟
+芡
+芣
+芤
+芥
+芦
+芨
+芩
+芪
+芫
+芬
+芭
+芮
+芯
+芰
+花
+芳
+芴
+芶
+芷
+芸
+芹
+芽
+芾
+苁
+苄
+苇
+苈
+苊
+苋
+苌
+苍
+苎
+苏
+苑
+苒
+苓
+苔
+苕
+苗
+苘
+苛
+苜
+苞
+苟
+苡
+苢
+苣
+苤
+若
+苦
+苫
+苭
+苯
+英
+苴
+苷
+苹
+苺
+苻
+苼
+苾
+茀
+茁
+茂
+范
+茄
+茅
+茆
+茇
+茈
+茉
+茌
+茎
+茏
+茑
+茔
+茕
+茗
+茚
+茛
+茜
+茝
+茧
+茨
+茫
+茬
+茭
+茯
+茱
+茳
+茴
+茵
+茶
+茸
+茹
+茺
+茼
+荀
+荃
+荄
+荅
+荆
+荇
+荈
+草
+荏
+荐
+荑
+荒
+荔
+荖
+荘
+荚
+荛
+荜
+荞
+荟
+荠
+荡
+荣
+荤
+荥
+荦
+荧
+荨
+荩
+荪
+荫
+荬
+荭
+荮
+药
+荳
+荷
+荸
+荻
+荼
+荽
+莃
+莅
+莆
+莉
+莎
+莒
+莓
+莘
+莙
+莛
+莜
+莞
+莠
+莨
+莩
+莪
+莫
+莱
+莲
+莳
+莴
+莶
+获
+莸
+莹
+莺
+莼
+莽
+菀
+菁
+菂
+菅
+菇
+菈
+菉
+菊
+菌
+菏
+菑
+菓
+菔
+菖
+菘
+菜
+菝
+菟
+菠
+菡
+菩
+菪
+菫
+菰
+菱
+菲
+菸
+菹
+菽
+菿
+萁
+萃
+萄
+萆
+萋
+萌
+萍
+萎
+萏
+萑
+萘
+萜
+萝
+萢
+萤
+营
+萦
+萧
+萨
+萩
+萱
+萸
+萼
+落
+葆
+葎
+葑
+葖
+著
+葙
+葚
+葛
+葜
+葡
+葢
+董
+葩
+葫
+葬
+葭
+葱
+葳
+葵
+葶
+葸
+葺
+蒂
+蒇
+蒉
+蒋
+蒌
+蒍
+蒎
+蒐
+蒗
+蒙
+蒜
+蒟
+蒡
+蒨
+蒯
+蒲
+蒴
+蒸
+蒹
+蒺
+蒻
+蒽
+蒾
+蒿
+蓁
+蓂
+蓄
+蓇
+蓉
+蓊
+蓍
+蓐
+蓑
+蓓
+蓖
+蓝
+蓟
+蓠
+蓢
+蓣
+蓥
+蓦
+蓬
+蓼
+蓿
+蔀
+蔌
+蔑
+蔓
+蔗
+蔘
+蔚
+蔟
+蔡
+蔫
+蔬
+蔴
+蔵
+蔷
+蔸
+蔹
+蔺
+蔻
+蔼
+蔽
+蕃
+蕅
+蕈
+蕉
+蕊
+蕖
+蕗
+蕙
+蕞
+蕡
+蕤
+蕨
+蕰
+蕲
+蕴
+蕹
+蕺
+蕻
+蕾
+薄
+薅
+薆
+薇
+薏
+薙
+薛
+薜
+薢
+薤
+薨
+薪
+薫
+薬
+薮
+薯
+薰
+薷
+薹
+藁
+藉
+藏
+藐
+藓
+藕
+藜
+藟
+藠
+藤
+藦
+藨
+藩
+藻
+藿
+蘅
+蘑
+蘖
+蘧
+蘩
+蘸
+蘼
+虎
+虏
+虐
+虑
+虒
+虓
+虔
+虚
+虞
+虢
+虫
+虬
+虮
+虱
+虹
+虺
+虻
+虽
+虾
+虿
+蚀
+蚁
+蚂
+蚊
+蚋
+蚌
+蚍
+蚓
+蚕
+蚖
+蚜
+蚝
+蚡
+蚣
+蚤
+蚧
+蚨
+蚩
+蚪
+蚬
+蚯
+蚰
+蚱
+蚴
+蚵
+蚶
+蚺
+蛀
+蛄
+蛆
+蛇
+蛉
+蛊
+蛋
+蛎
+蛏
+蛐
+蛔
+蛙
+蛛
+蛞
+蛟
+蛤
+蛩
+蛭
+蛮
+蛰
+蛱
+蛲
+蛳
+蛴
+蛸
+蛹
+蛾
+蜀
+蜂
+蜃
+蜇
+蜈
+蜉
+蜊
+蜍
+蜑
+蜒
+蜓
+蜕
+蜗
+蜘
+蜚
+蜜
+蜞
+蜡
+蜢
+蜣
+蜥
+蜩
+蜮
+蜱
+蜴
+蜷
+蜻
+蜾
+蜿
+蝇
+蝈
+蝉
+蝌
+蝎
+蝓
+蝗
+蝙
+蝠
+蝣
+蝤
+蝥
+蝮
+蝰
+蝲
+蝴
+蝶
+蝻
+蝼
+蝽
+蝾
+螂
+螃
+螅
+螈
+螋
+融
+螓
+螟
+螣
+螨
+螫
+螬
+螭
+螯
+螳
+螵
+螺
+螽
+蟀
+蟆
+蟊
+蟋
+蟌
+蟑
+蟒
+蟛
+蟜
+蟠
+蟥
+蟪
+蟮
+蟳
+蟹
+蟾
+蠃
+蠊
+蠋
+蠍
+蠓
+蠕
+蠖
+蠡
+蠢
+蠲
+蠹
+蠼
+血
+衄
+衅
+衆
+行
+衍
+衎
+衔
+街
+衙
+衞
+衡
+衢
+衣
+补
+表
+衩
+衪
+衫
+衬
+衮
+衰
+衲
+衷
+衽
+衾
+衿
+袁
+袂
+袄
+袅
+袆
+袈
+袋
+袍
+袒
+袓
+袖
+袛
+袜
+袢
+袤
+袪
+被
+袭
+袮
+袱
+袴
+袷
+袼
+裁
+裂
+装
+裆
+裈
+裉
+裒
+裔
+裕
+裘
+裙
+裛
+裟
+裡
+裢
+裤
+裥
+裨
+裪
+裱
+裳
+裴
+裸
+裹
+裼
+裾
+褀
+褂
+褆
+褊
+褐
+褒
+褓
+褔
+褙
+褚
+褛
+褡
+褥
+褪
+褫
+褰
+褴
+褶
+襁
+襄
+襌
+襕
+襜
+襞
+襟
+襦
+襻
+西
+要
+覃
+覆
+覇
+覚
+覧
+覩
+観
+见
+观
+规
+觅
+视
+觇
+览
+觉
+觊
+觋
+觌
+觎
+觏
+觐
+觑
+角
+觚
+觜
+觞
+解
+觥
+触
+觯
+觳
+觽
+觿
+言
+訇
+訏
+訚
+訫
+訳
+訾
+詈
+詜
+詝
+詧
+詹
+誉
+誊
+誐
+誓
+説
+読
+諡
+諲
+諴
+謇
+謦
+譞
+警
+譬
+譲
+讌
+讠
+计
+订
+讣
+认
+讥
+讦
+讧
+讨
+让
+讪
+讫
+讬
+训
+议
+讯
+记
+讲
+讳
+讴
+讵
+讶
+讷
+许
+讹
+论
+讼
+讽
+设
+访
+诀
+证
+诂
+诃
+评
+诅
+识
+诈
+诉
+诊
+诋
+诌
+词
+诎
+诏
+译
+诒
+诓
+诔
+试
+诖
+诗
+诘
+诙
+诚
+诛
+诜
+话
+诞
+诟
+诠
+诡
+询
+诣
+诤
+该
+详
+诧
+诨
+诩
+诫
+诬
+语
+诮
+误
+诰
+诱
+诲
+诳
+说
+诵
+诶
+请
+诸
+诹
+诺
+读
+诼
+诽
+课
+诿
+谀
+谁
+谂
+调
+谄
+谅
+谆
+谇
+谈
+谊
+谋
+谌
+谍
+谎
+谏
+谐
+谑
+谒
+谓
+谔
+谕
+谖
+谗
+谘
+谙
+谚
+谛
+谜
+谝
+谞
+谟
+谠
+谡
+谢
+谣
+谤
+谥
+谦
+谧
+谨
+谩
+谪
+谫
+谬
+谭
+谮
+谯
+谰
+谱
+谲
+谳
+谴
+谵
+谶
+谷
+谿
+豁
+豆
+豇
+豉
+豊
+豌
+豕
+豚
+象
+豢
+豨
+豪
+豫
+豳
+豸
+豹
+豺
+貂
+貅
+貉
+貊
+貌
+貐
+貔
+貘
+賨
+賸
+贇
+贝
+贞
+负
+贠
+贡
+财
+责
+贤
+败
+账
+货
+质
+贩
+贪
+贫
+贬
+购
+贮
+贯
+贰
+贱
+贲
+贳
+贴
+贵
+贶
+贷
+贸
+费
+贺
+贻
+贼
+贽
+贾
+贿
+赀
+赁
+赂
+赃
+资
+赅
+赈
+赉
+赊
+赋
+赌
+赍
+赎
+赏
+赐
+赑
+赓
+赔
+赕
+赖
+赘
+赙
+赚
+赛
+赜
+赝
+赞
+赟
+赠
+赡
+赢
+赣
+赤
+赦
+赧
+赪
+赫
+赭
+赮
+走
+赳
+赴
+赵
+赶
+起
+趁
+趄
+超
+越
+趋
+趔
+趟
+趣
+趯
+趱
+足
+趴
+趵
+趸
+趺
+趼
+趾
+趿
+跂
+跃
+跄
+跆
+跋
+跌
+跎
+跏
+跑
+跖
+跗
+跚
+跛
+距
+跞
+跟
+跢
+跣
+跤
+跨
+跩
+跪
+跫
+跬
+路
+跳
+践
+跶
+跷
+跸
+跹
+跺
+跻
+跽
+踅
+踉
+踊
+踌
+踏
+踔
+踝
+踞
+踟
+踢
+踣
+踩
+踪
+踬
+踮
+踯
+踰
+踱
+踵
+踹
+踽
+蹀
+蹁
+蹂
+蹄
+蹇
+蹈
+蹉
+蹊
+蹋
+蹑
+蹒
+蹓
+蹙
+蹚
+蹟
+蹦
+蹩
+蹬
+蹭
+蹰
+蹲
+蹴
+蹶
+蹻
+蹼
+蹿
+躁
+躄
+躅
+躇
+躏
+躐
+躔
+躜
+躞
+身
+躬
+躯
+躲
+躺
+転
+軽
+輋
+轘
+车
+轧
+轨
+轩
+轫
+转
+轭
+轮
+软
+轰
+轱
+轲
+轳
+轴
+轵
+轶
+轸
+轹
+轺
+轻
+轼
+载
+轾
+轿
+辂
+较
+辄
+辅
+辆
+辇
+辈
+辉
+辊
+辋
+辍
+辎
+辏
+辐
+辑
+输
+辔
+辕
+辖
+辗
+辘
+辙
+辚
+辛
+辜
+辞
+辟
+辣
+辨
+辩
+辫
+辰
+辱
+辶
+边
+辺
+辻
+込
+辽
+达
+辿
+迁
+迂
+迄
+迅
+过
+迈
+迍
+迎
+运
+近
+迓
+返
+迕
+还
+这
+进
+远
+违
+连
+迟
+迢
+迤
+迥
+迦
+迨
+迩
+迪
+迫
+迭
+迮
+述
+迳
+迷
+迸
+迹
+追
+退
+送
+适
+逃
+逄
+逅
+逆
+选
+逊
+逋
+逍
+透
+逐
+逑
+递
+途
+逖
+逗
+這
+通
+逛
+逝
+逞
+速
+造
+逡
+逢
+逦
+逭
+逮
+逯
+進
+逵
+逶
+逸
+逹
+逺
+逻
+逼
+逾
+遁
+遂
+遄
+遇
+遍
+遏
+遐
+遑
+遒
+道
+遗
+遘
+遛
+遢
+遣
+遥
+遨
+遭
+遮
+遯
+遴
+遵
+遶
+遹
+遽
+避
+邀
+邂
+邃
+邅
+邈
+邉
+邋
+邑
+邓
+邕
+邗
+邙
+邛
+邝
+邠
+邡
+邢
+那
+邦
+邨
+邪
+邬
+邮
+邯
+邰
+邱
+邳
+邴
+邵
+邶
+邸
+邹
+邺
+邻
+邽
+邾
+郁
+郄
+郅
+郇
+郈
+郊
+郎
+郏
+郐
+郑
+郓
+郕
+郗
+郚
+郛
+郜
+郝
+郞
+郡
+郢
+郤
+郦
+郧
+部
+郪
+郫
+郭
+郯
+郴
+郷
+郸
+都
+郾
+郿
+鄀
+鄂
+鄄
+鄗
+鄘
+鄙
+鄚
+鄜
+鄞
+鄠
+鄢
+鄣
+鄩
+鄫
+鄮
+鄯
+鄱
+鄹
+酂
+酃
+酆
+酉
+酊
+酋
+酌
+配
+酎
+酐
+酒
+酗
+酚
+酝
+酞
+酡
+酢
+酣
+酤
+酥
+酩
+酪
+酬
+酮
+酯
+酰
+酱
+酲
+酴
+酵
+酶
+酷
+酸
+酹
+酺
+酽
+酾
+酿
+醂
+醅
+醇
+醉
+醋
+醌
+醍
+醐
+醑
+醒
+醚
+醛
+醢
+醣
+醪
+醭
+醮
+醯
+醴
+醵
+醺
+醿
+釆
+采
+釉
+释
+里
+重
+野
+量
+金
+釜
+釭
+釿
+鈇
+鈈
+鈊
+鈎
+鈡
+鉄
+鉏
+鉨
+鉴
+鉷
+銎
+銙
+銭
+銮
+鋆
+鋈
+鋐
+鋗
+鋬
+鋹
+錞
+錡
+錤
+録
+錾
+鍀
+鍪
+鎌
+鎏
+鎚
+鏊
+鏐
+鏖
+鐏
+鑨
+鑫
+钅
+钆
+钇
+针
+钉
+钊
+钋
+钌
+钍
+钎
+钏
+钐
+钒
+钓
+钔
+钕
+钖
+钗
+钙
+钚
+钛
+钜
+钝
+钞
+钟
+钠
+钡
+钢
+钣
+钤
+钥
+钦
+钧
+钨
+钩
+钪
+钫
+钬
+钭
+钮
+钯
+钰
+钱
+钲
+钳
+钴
+钵
+钶
+钷
+钹
+钺
+钻
+钼
+钽
+钾
+钿
+铀
+铁
+铂
+铃
+铄
+铅
+铆
+铈
+铉
+铊
+铋
+铌
+铍
+铎
+铐
+铑
+铒
+铓
+铕
+铖
+铗
+铙
+铚
+铛
+铜
+铝
+铟
+铠
+铡
+铢
+铣
+铤
+铥
+铦
+铧
+铨
+铩
+铪
+铫
+铬
+铭
+铮
+铯
+铰
+铱
+铲
+铳
+铵
+银
+铷
+铸
+铺
+铼
+铽
+链
+铿
+销
+锁
+锂
+锃
+锄
+锅
+锆
+锇
+锈
+锉
+锋
+锌
+锍
+锎
+锏
+锐
+锑
+锒
+锔
+锕
+锖
+锗
+锘
+错
+锚
+锛
+锜
+锝
+锞
+锟
+锠
+锡
+锢
+锣
+锤
+锥
+锦
+锨
+锩
+锪
+锫
+锬
+锭
+键
+锯
+锰
+锱
+锲
+锳
+锴
+锵
+锶
+锷
+锸
+锹
+锺
+锻
+锼
+锽
+镀
+镁
+镂
+镄
+镅
+镆
+镇
+镈
+镉
+镊
+镋
+镌
+镍
+镎
+镏
+镐
+镑
+镒
+镓
+镔
+镕
+镖
+镗
+镘
+镙
+镚
+镛
+镜
+镝
+镞
+镟
+镠
+镡
+镢
+镣
+镥
+镦
+镧
+镨
+镪
+镫
+镬
+镭
+镯
+镰
+镱
+镲
+镳
+镶
+长
+開
+閟
+関
+閦
+闇
+闍
+闘
+门
+闩
+闪
+闫
+闭
+问
+闯
+闰
+闱
+闲
+闳
+间
+闵
+闶
+闷
+闸
+闹
+闺
+闻
+闼
+闽
+闾
+闿
+阀
+阁
+阂
+阃
+阄
+阅
+阆
+阇
+阈
+阉
+阊
+阋
+阌
+阍
+阎
+阏
+阐
+阑
+阒
+阓
+阔
+阕
+阖
+阗
+阙
+阚
+阛
+阜
+阝
+队
+阡
+阪
+阮
+阱
+防
+阳
+阴
+阵
+阶
+阻
+阼
+阿
+陀
+陂
+附
+际
+陆
+陇
+陈
+陉
+陋
+陌
+降
+限
+陔
+陕
+陛
+陞
+陟
+陡
+院
+除
+陨
+险
+陪
+陬
+陲
+陵
+陶
+陷
+険
+隂
+隅
+隆
+隈
+隋
+隍
+随
+隐
+隔
+隗
+隘
+隙
+障
+隠
+隣
+隧
+隰
+隳
+隶
+隹
+隼
+隽
+难
+雀
+雁
+雄
+雅
+集
+雇
+雉
+雊
+雌
+雍
+雎
+雏
+雑
+雒
+雕
+雠
+雨
+雩
+雪
+雫
+雯
+雱
+雳
+零
+雷
+雹
+雾
+需
+霁
+霂
+霄
+霅
+霆
+震
+霈
+霉
+霊
+霍
+霎
+霏
+霓
+霖
+霙
+霜
+霞
+霪
+霭
+霰
+露
+霸
+霹
+霾
+靑
+青
+靓
+靖
+静
+靛
+非
+靠
+靡
+面
+靥
+革
+靬
+靳
+靴
+靶
+靺
+靼
+鞅
+鞋
+鞍
+鞑
+鞔
+鞘
+鞞
+鞠
+鞣
+鞥
+鞨
+鞫
+鞬
+鞭
+鞮
+鞯
+鞴
+韘
+韡
+韦
+韧
+韩
+韪
+韫
+韬
+韭
+音
+韵
+韶
+頔
+頞
+頠
+頫
+頵
+頼
+顒
+顔
+顕
+顗
+页
+顶
+顷
+顸
+项
+顺
+须
+顼
+顽
+顾
+顿
+颀
+颁
+颂
+颃
+预
+颅
+领
+颇
+颈
+颉
+颊
+颋
+颌
+颍
+颎
+颏
+颐
+频
+颓
+颔
+颖
+颗
+题
+颙
+颚
+颛
+颜
+额
+颞
+颟
+颠
+颡
+颢
+颤
+颦
+颧
+风
+飏
+飐
+飑
+飒
+飓
+飕
+飖
+飘
+飙
+飚
+飞
+食
+飧
+飨
+餍
+餐
+餗
+餮
+饔
+饕
+饣
+饥
+饦
+饧
+饨
+饩
+饪
+饫
+饬
+饭
+饮
+饯
+饰
+饱
+饲
+饴
+饵
+饶
+饷
+饸
+饹
+饺
+饼
+饽
+饿
+馀
+馁
+馃
+馄
+馅
+馆
+馇
+馈
+馊
+馋
+馍
+馏
+馐
+馑
+馒
+馓
+馔
+馕
+首
+馗
+馘
+香
+馥
+馨
+駄
+駅
+駆
+騄
+騑
+騒
+験
+驎
+驒
+驩
+马
+驭
+驮
+驯
+驰
+驱
+驳
+驴
+驶
+驷
+驸
+驹
+驺
+驻
+驼
+驽
+驾
+驿
+骀
+骁
+骂
+骃
+骄
+骅
+骆
+骇
+骈
+骉
+骊
+骋
+验
+骍
+骎
+骏
+骐
+骑
+骓
+骕
+骖
+骗
+骘
+骙
+骚
+骛
+骜
+骝
+骞
+骟
+骠
+骡
+骢
+骤
+骥
+骧
+骨
+骰
+骱
+骶
+骷
+骸
+骺
+骼
+髀
+髁
+髂
+髃
+髅
+髈
+髋
+髌
+髎
+髑
+髓
+高
+髙
+髡
+髦
+髪
+髫
+髭
+髯
+髹
+髻
+鬃
+鬈
+鬐
+鬓
+鬘
+鬟
+鬣
+鬯
+鬲
+鬶
+鬻
+鬼
+魁
+魂
+魃
+魄
+魅
+魆
+魇
+魈
+魉
+魋
+魍
+魏
+魑
+魔
+魟
+鮀
+鮈
+鮋
+鮟
+鮠
+鮨
+鮰
+鰕
+鰤
+鱀
+鱇
+鱓
+鱬
+鱲
+鱻
+鱼
+鱿
+鲀
+鲁
+鲂
+鲃
+鲅
+鲆
+鲇
+鲈
+鲉
+鲊
+鲋
+鲌
+鲍
+鲎
+鲏
+鲐
+鲑
+鲔
+鲕
+鲗
+鲘
+鲙
+鲚
+鲛
+鲜
+鲞
+鲟
+鲠
+鲡
+鲢
+鲣
+鲤
+鲥
+鲦
+鲧
+鲨
+鲩
+鲫
+鲭
+鲮
+鲱
+鲲
+鲳
+鲴
+鲵
+鲶
+鲷
+鲸
+鲹
+鲺
+鲻
+鲼
+鲽
+鲿
+鳀
+鳃
+鳄
+鳅
+鳇
+鳉
+鳊
+鳌
+鳍
+鳎
+鳏
+鳐
+鳑
+鳓
+鳔
+鳕
+鳖
+鳗
+鳙
+鳚
+鳜
+鳝
+鳞
+鳟
+鳡
+鳢
+鳣
+鳯
+鳽
+鳾
+鴂
+鴞
+鴷
+鵖
+鵙
+鵟
+鵺
+鶒
+鶲
+鷇
+鷉
+鷟
+鸂
+鸊
+鸑
+鸟
+鸠
+鸡
+鸢
+鸣
+鸥
+鸦
+鸨
+鸩
+鸪
+鸫
+鸬
+鸭
+鸮
+鸯
+鸰
+鸱
+鸲
+鸳
+鸵
+鸶
+鸷
+鸸
+鸹
+鸺
+鸻
+鸽
+鸾
+鸿
+鹀
+鹁
+鹂
+鹃
+鹄
+鹅
+鹆
+鹇
+鹈
+鹉
+鹊
+鹋
+鹌
+鹍
+鹎
+鹏
+鹑
+鹓
+鹕
+鹖
+鹗
+鹘
+鹚
+鹛
+鹜
+鹞
+鹟
+鹠
+鹡
+鹢
+鹣
+鹤
+鹦
+鹧
+鹨
+鹩
+鹪
+鹫
+鹬
+鹭
+鹮
+鹰
+鹱
+鹳
+鹾
+鹿
+麂
+麇
+麈
+麋
+麐
+麑
+麒
+麓
+麝
+麟
+麤
+麦
+麴
+麸
+麹
+麻
+麽
+麾
+麿
+黄
+黉
+黍
+黎
+黏
+黐
+黑
+黒
+黔
+默
+黙
+黛
+黜
+黝
+黟
+黠
+黡
+黢
+黥
+黧
+黩
+黯
+黻
+黼
+黾
+鼋
+鼍
+鼎
+鼐
+鼓
+鼗
+鼙
+鼠
+鼢
+鼩
+鼬
+鼯
+鼱
+鼷
+鼹
+鼻
+鼽
+鼾
+齁
+齐
+齑
+齢
+齮
+齿
+龀
+龁
+龃
+龄
+龅
+龆
+龇
+龈
+龉
+龊
+龋
+龌
+龑
+龘
+龙
+龚
+龛
+龟
+龠
+龢
+A
+B
+C
+D
+E
+F
+G
+H
+I
+J
+K
+L
+M
+N
+O
+P
+Q
+R
+S
+T
+U
+V
+W
+X
+Y
+Z
+a
+b
+c
+d
+e
+f
+g
+h
+i
+j
+k
+l
+m
+n
+o
+p
+q
+r
+s
+t
+u
+v
+w
+x
+y
+z
+AA
+AB
+AC
+AD
+AE
+AF
+AG
+AH
+AI
+AJ
+AK
+AL
+AM
+AN
+AP
+AQ
+AR
+AS
+AT
+AU
+AV
+AW
+AX
+AZ
+Al
+An
+Au
+Aw
+BA
+BB
+BC
+BD
+BE
+BF
+BG
+BH
+BI
+BJ
+BK
+BL
+BM
+BN
+BO
+BP
+BQ
+BR
+BS
+BT
+BU
+BV
+BW
+BY
+Bo
+Br
+Bu
+CA
+CB
+CC
+CD
+CE
+CF
+CG
+CH
+CI
+CJ
+CK
+CL
+CM
+CN
+CO
+CP
+CQ
+CR
+CS
+CT
+CU
+CV
+CW
+CX
+CY
+CZ
+Ca
+Ch
+Cl
+Co
+Cu
+DA
+DB
+DC
+DD
+DE
+DF
+DG
+DH
+DI
+DJ
+DK
+DL
+DM
+DN
+DO
+DQ
+DR
+DS
+DT
+DV
+DW
+DX
+DY
+DZ
+Da
+De
+Di
+Do
+Dr
+Du
+EA
+EB
+EC
+ED
+EE
+EF
+EG
+EH
+EI
+EK
+EL
+EM
+EN
+EP
+EQ
+ER
+ES
+ET
+EU
+EV
+EW
+EX
+EZ
+Ed
+En
+Ev
+Ex
+FA
+FB
+FC
+FD
+FE
+FF
+FG
+FH
+FI
+FJ
+FL
+FM
+FN
+FO
+FP
+FR
+FS
+FT
+FU
+FW
+FX
+FY
+FZ
+Fa
+Fi
+Fl
+Fo
+Fr
+Fu
+GA
+GB
+GC
+GD
+GE
+GF
+GG
+GH
+GI
+GJ
+GK
+GL
+GM
+GN
+GO
+GP
+GQ
+GR
+GS
+GT
+GU
+GW
+GX
+GY
+GZ
+Ga
+Go
+Gr
+Gu
+HA
+HB
+HC
+HD
+HE
+HF
+HG
+HH
+HI
+HJ
+HK
+HL
+HO
+HP
+HQ
+HR
+HS
+HT
+HU
+HV
+HW
+HX
+HY
+HZ
+Ha
+He
+Hi
+Ho
+Hu
+Hz
+IB
+IC
+ID
+IE
+IF
+IG
+IH
+II
+IK
+IL
+IM
+IN
+IO
+IP
+IQ
+IR
+IS
+IT
+IU
+IV
+IX
+If
+In
+JA
+JB
+JC
+JD
+JF
+JG
+JH
+JI
+JJ
+JK
+JL
+JM
+JO
+JP
+JQ
+JR
+JS
+JT
+JU
+JW
+JX
+JY
+JZ
+Ja
+Ji
+Jo
+Ju
+KA
+KB
+KC
+KD
+KE
+KF
+KG
+KH
+KI
+KJ
+KK
+KL
+KM
+KN
+KO
+KP
+KR
+KS
+KT
+KV
+KW
+KX
+KY
+KZ
+LA
+LB
+LC
+LD
+LE
+LF
+LG
+LH
+LI
+LJ
+LK
+LL
+LM
+LN
+LO
+LP
+LQ
+LR
+LS
+LT
+LU
+LV
+LW
+LX
+LY
+LZ
+La
+Le
+Li
+Lo
+Lu
+MA
+MB
+MC
+MD
+ME
+MF
+MG
+MH
+MI
+MJ
+MK
+ML
+MM
+MN
+MO
+MP
+MQ
+MR
+MS
+MT
+MU
+MV
+MW
+MX
+MY
+Ma
+Me
+Mi
+Mo
+Mu
+My
+NA
+NB
+NC
+ND
+NE
+NF
+NG
+NH
+NI
+NJ
+NK
+NL
+NN
+NO
+NP
+NR
+NS
+NT
+NU
+NV
+NW
+NX
+NY
+NZ
+Na
+Ne
+No
+Nu
+OA
+OB
+OC
+OD
+OE
+OF
+OG
+OH
+OK
+OL
+OM
+ON
+OO
+OP
+OR
+OS
+OT
+OU
+OV
+OZ
+Of
+Oh
+On
+Op
+Or
+Ou
+Ox
+PA
+PB
+PC
+PD
+PE
+PF
+PG
+PH
+PI
+PJ
+PK
+PL
+PM
+PN
+PO
+PP
+PQ
+PR
+PS
+PT
+PU
+PV
+PW
+PX
+Pa
+Ph
+Pl
+Po
+Pr
+Pu
+QA
+QB
+QC
+QE
+QF
+QG
+QJ
+QL
+QQ
+QR
+QS
+QT
+QU
+QW
+QY
+Qi
+Qu
+RA
+RB
+RC
+RE
+RF
+RG
+RH
+RI
+RJ
+RK
+RL
+RM
+RN
+RO
+RP
+RQ
+RR
+RS
+RT
+RV
+RW
+RX
+RZ
+Ra
+Re
+Ro
+Ru
+SA
+SB
+SC
+SD
+SE
+SF
+SG
+SH
+SI
+SJ
+SK
+SL
+SM
+SN
+SO
+SP
+SQ
+SR
+SS
+ST
+SU
+SV
+SW
+SX
+SY
+SZ
+Sc
+Sh
+So
+Sp
+St
+Su
+Sw
+Sy
+TA
+TB
+TC
+TD
+TE
+TF
+TG
+TH
+TI
+TJ
+TK
+TL
+TM
+TN
+TO
+TP
+TQ
+TR
+TS
+TT
+TU
+TV
+TW
+TX
+TY
+TZ
+Th
+To
+Tr
+Tw
+UA
+UC
+UD
+UE
+UF
+UG
+UH
+UI
+UK
+UL
+UM
+UN
+UP
+UR
+US
+UT
+UU
+UV
+UW
+UX
+Ub
+Un
+Up
+VA
+VB
+VC
+VE
+VF
+VG
+VH
+VI
+VJ
+VK
+VL
+VM
+VN
+VO
+VP
+VR
+VS
+VT
+VU
+VV
+VX
+Vi
+Vo
+WA
+WB
+WC
+WE
+WH
+WI
+WJ
+WL
+WM
+WN
+WO
+WQ
+WR
+WS
+WT
+WU
+WW
+WX
+WZ
+Wa
+We
+Wi
+Wo
+Wu
+XB
+XC
+XD
+XF
+XG
+XH
+XI
+XJ
+XK
+XL
+XM
+XO
+XP
+XQ
+XR
+XS
+XT
+XU
+XV
+XW
+XX
+XY
+XZ
+Xi
+Xu
+YA
+YB
+YC
+YD
+YE
+YF
+YG
+YH
+YJ
+YL
+YM
+YO
+YP
+YS
+YT
+YU
+YX
+YY
+YZ
+Ya
+Yo
+Yu
+ZA
+ZB
+ZC
+ZD
+ZE
+ZF
+ZG
+ZH
+ZI
+ZJ
+ZL
+ZM
+ZN
+ZO
+ZQ
+ZR
+ZS
+ZU
+ZW
+ZX
+ZY
+ZZ
+Zh
+ab
+aj
+an
+ap
+ar
+bb
+be
+bj
+bo
+bu
+by
+ca
+cb
+cf
+ch
+cl
+cm
+co
+cp
+cv
+dB
+da
+de
+di
+dj
+dn
+do
+dr
+dv
+ed
+em
+en
+ep
+eq
+ev
+ex
+ez
+fa
+fe
+ff
+fi
+fl
+fo
+fr
+fu
+gb
+gd
+gh
+gi
+go
+gp
+gr
+gu
+gz
+ha
+he
+hi
+ho
+hp
+hz
+iP
+iT
+ib
+ic
+id
+if
+ig
+im
+in
+io
+ip
+iq
+is
+it
+jQ
+ja
+ji
+jj
+jo
+jq
+ju
+kJ
+kN
+kW
+kg
+kn
+kz
+la
+ld
+le
+lg
+li
+ll
+lo
+lp
+lz
+ma
+mb
+me
+mi
+mm
+mo
+mp
+mq
+mu
+mv
+my
+na
+nb
+ng
+no
+nv
+ob
+of
+oh
+ok
+ol
+on
+op
+or
+ou
+ow
+oz
+pH
+pa
+pc
+ph
+pk
+pl
+po
+pp
+pr
+pu
+pv
+qf
+qq
+qu
+qz
+ra
+re
+rn
+ro
+rq
+se
+sh
+sk
+so
+sp
+sq
+st
+su
+sw
+sz
+th
+ti
+to
+tr
+tv
+tw
+ub
+uc
+uf
+ui
+uk
+un
+up
+us
+uv
+ux
+uz
+vc
+vi
+vo
+vr
+wa
+we
+wh
+wi
+wo
+wr
+ww
+xj
+xq
+xx
+ya
+ye
+yj
+yo
+yu
+yy
+yz
+zf
+zh
+zi
+zj
+zq
+zu
+zz
+AAA
+AAC
+ABA
+ABB
+ABC
+ABO
+ABS
+ABT
+ACA
+ACC
+ACD
+ACE
+ACG
+ACK
+ACL
+ACM
+ACP
+ACR
+ACS
+ACT
+ADA
+ADC
+ADD
+ADF
+ADI
+ADO
+ADP
+ADR
+ADS
+ADV
+AED
+AES
+AFC
+AFP
+AFS
+AGB
+AGC
+AGE
+AGM
+AGP
+AGV
+AIA
+AIC
+AIG
+AIM
+AIP
+AIR
+AIS
+AIX
+AKB
+AKM
+ALA
+ALL
+ALT
+AMA
+AMC
+AMD
+AMG
+AMI
+AML
+AMP
+AMR
+AMS
+AMT
+AMX
+AND
+AOC
+AOE
+AOL
+APA
+APC
+APE
+APG
+API
+APK
+APL
+APM
+APP
+APS
+APT
+APU
+ARA
+ARC
+ARE
+ARM
+ARP
+ART
+ASA
+ASC
+ASF
+ASM
+ASP
+ASR
+AST
+ATA
+ATC
+ATF
+ATI
+ATK
+ATM
+ATP
+ATS
+ATV
+ATX
+AUC
+AUG
+AUX
+AVC
+AVG
+AVI
+AVR
+AVS
+AVX
+AWM
+AWS
+All
+And
+Ang
+App
+Aqu
+BAC
+BAD
+BAE
+BAR
+BAT
+BAU
+BBA
+BBB
+BBC
+BBE
+BBQ
+BBS
+BBT
+BCD
+BEA
+BEC
+BEI
+BET
+BGA
+BGM
+BGP
+BIG
+BIM
+BIS
+BLG
+BMC
+BMD
+BMG
+BMI
+BMP
+BMW
+BMX
+BNC
+BOD
+BOM
+BOT
+BOX
+BOY
+BPM
+BPO
+BRN
+BRT
+BSA
+BSC
+BSD
+BSI
+BSM
+BSP
+BSS
+BTC
+BTR
+BTS
+BTV
+BUG
+BUN
+BUS
+BWV
+Bur
+Bus
+But
+CAA
+CAC
+CAD
+CAE
+CAI
+CAJ
+CAM
+CAN
+CAP
+CAR
+CAS
+CAT
+CBA
+CBC
+CBD
+CBN
+CBR
+CBS
+CCA
+CCC
+CCD
+CCF
+CCG
+CCI
+CCK
+CCM
+CCN
+CCP
+CCS
+CDC
+CDM
+CDN
+CDO
+CDP
+CDR
+CDS
+CEA
+CEC
+CEO
+CES
+CET
+CFA
+CFC
+CFD
+CFO
+CFR
+CGI
+CHA
+CHM
+CHO
+CIA
+CIC
+CID
+CIE
+CIF
+CIK
+CIO
+CIP
+CIS
+CLA
+CLI
+CLM
+CLS
+CMA
+CMC
+CME
+CML
+CMM
+CMO
+CMP
+CMS
+CMV
+CNC
+CNG
+CNN
+CNS
+COB
+COC
+COD
+COM
+CON
+COO
+COP
+COS
+COX
+CPA
+CPC
+CPE
+CPI
+CPL
+CPM
+CPP
+CPR
+CPS
+CPU
+CQC
+CRC
+CRM
+CRP
+CRS
+CRT
+CSA
+CSF
+CSI
+CSM
+CSP
+CSR
+CSS
+CST
+CTA
+CTC
+CTI
+CTO
+CTP
+CTS
+CUB
+CUT
+CVD
+CVN
+CVS
+CVT
+CXW
+CYP
+Car
+Cha
+Chr
+Chu
+Com
+Con
+Cou
+Cur
+DAB
+DAC
+DAO
+DAS
+DAT
+DAY
+DBA
+DBM
+DCD
+DCE
+DCF
+DCS
+DCT
+DDC
+DDD
+DDG
+DDN
+DDR
+DDS
+DDT
+DEA
+DEC
+DEM
+DES
+DFS
+DFT
+DHA
+DHL
+DIC
+DID
+DIF
+DIN
+DIP
+DIV
+DIY
+DLC
+DLL
+DLP
+DLT
+DMA
+DMC
+DMD
+DMF
+DMI
+DMO
+DMZ
+DNA
+DNF
+DNS
+DNV
+DOC
+DOI
+DOM
+DON
+DOS
+DOT
+DPI
+DPP
+DPS
+DRM
+DRX
+DSA
+DSC
+DSG
+DSL
+DSM
+DSP
+DSS
+DTC
+DTE
+DTM
+DTS
+DTU
+DVB
+DVD
+DVI
+DVR
+DWG
+DYG
+Day
+Div
+Don
+Dou
+Dow
+EAN
+EAP
+EBD
+EBS
+ECC
+ECM
+ECO
+ECT
+ECU
+ECW
+EDA
+EDG
+EDI
+EDM
+EDP
+EDR
+EEG
+EEP
+EFR
+EGF
+EHS
+EIA
+EJB
+EMA
+EMC
+EMI
+EMP
+EMS
+END
+EOS
+EPA
+EPC
+EPO
+EPR
+EPS
+ERP
+ESC
+ESD
+ESI
+ESL
+ESP
+ESR
+EST
+ETC
+ETF
+ETH
+ETL
+ETS
+EVA
+EVE
+EVO
+EXE
+EXO
+EXP
+EYE
+Eff
+Ell
+Emb
+Emp
+End
+Eng
+Equ
+Eur
+Eva
+Exc
+Exp
+FAA
+FAB
+FAG
+FAL
+FAN
+FAO
+FAQ
+FAT
+FBI
+FCA
+FCC
+FCI
+FCS
+FDA
+FDD
+FDI
+FEM
+FES
+FET
+FFT
+FGO
+FHD
+FIA
+FLV
+FLY
+FMS
+FNC
+FOB
+FOF
+FOR
+FOX
+FPC
+FPS
+FPX
+FRP
+FSA
+FSB
+FSC
+FSH
+FTA
+FTC
+FTP
+FUE
+FUN
+Fin
+Fiv
+Fly
+For
+Fou
+Fuj
+Fun
+Fut
+GAP
+GAT
+GAY
+GBA
+GBK
+GBT
+GBU
+GCC
+GCS
+GCT
+GDI
+GDP
+GEN
+GEO
+GET
+GFP
+GHz
+GIA
+GIF
+GIS
+GLA
+GLC
+GLP
+GLS
+GMA
+GMC
+GMP
+GMS
+GMT
+GMV
+GND
+GNP
+GNU
+GOD
+GOT
+GPA
+GPL
+GPS
+GPT
+GPU
+GRC
+GRE
+GRF
+GSH
+GSM
+GSP
+GTA
+GTI
+GTO
+GTP
+GTR
+GTS
+GTX
+GUI
+Giv
+Gmb
+Gua
+Gui
+Gun
+Guo
+Guy
+HAD
+HAL
+HBA
+HBO
+HBV
+HBs
+HCG
+HCI
+HCV
+HCl
+HDD
+HDL
+HDR
+HDV
+HEY
+HFC
+HGH
+HGT
+HID
+HIP
+HIS
+HIT
+HIV
+HLA
+HMG
+HMI
+HMS
+HOP
+HOT
+HOW
+HPC
+HPV
+HRC
+HRT
+HSE
+HSK
+HSV
+HTC
+HUB
+HUD
+HVG
+Haz
+Her
+Hom
+Hon
+Hou
+How
+Hua
+Hub
+Hum
+Hun
+IAI
+IAS
+IAT
+IBC
+IBF
+IBM
+ICA
+ICC
+ICD
+ICE
+ICO
+ICP
+ICQ
+ICS
+ICT
+ICU
+IDC
+IDD
+IDE
+IDF
+IDG
+IDS
+IEC
+IET
+IFA
+IFC
+IFI
+IFN
+IGF
+IGN
+IIA
+III
+IIS
+IKO
+IMA
+IMC
+IMD
+IME
+IMF
+IMG
+IMO
+IMS
+IMT
+INA
+INC
+INF
+ING
+INS
+INT
+IOS
+IPA
+IPC
+IPO
+IPS
+IPX
+IRC
+IRI
+ISA
+ISI
+ISM
+ISO
+ISP
+ITC
+ITF
+ITO
+ITS
+ITT
+ITU
+ITV
+IVR
+Imp
+InC
+Inf
+Inj
+Int
+JAR
+JBL
+JBT
+JCB
+JCR
+JDB
+JDG
+JET
+JGJ
+JIS
+JIT
+JKL
+JOE
+JPG
+JSF
+JSP
+JST
+JTA
+JVC
+JVM
+JYJ
+JYP
+Jac
+Jam
+Jan
+Jap
+Jav
+Jay
+Jin
+Joh
+Jon
+Jul
+Jun
+Jus
+KAB
+KAT
+KBS
+KDF
+KDJ
+KEY
+KFC
+KFR
+KID
+KIS
+KJm
+KOF
+KOH
+KOL
+KPI
+KPL
+KTV
+KVM
+Kin
+Kon
+Kur
+LAB
+LAN
+LBS
+LCA
+LCD
+LCK
+LCS
+LDA
+LDH
+LDL
+LDP
+LED
+LEE
+LEO
+LES
+LET
+LGA
+LGD
+LIN
+LIU
+LLC
+LME
+LMS
+LNG
+LOF
+LOL
+LOW
+LPG
+LPL
+LPR
+LRC
+LSA
+LSD
+LSI
+LSP
+LTD
+LTE
+LUC
+LUN
+LVM
+Laz
+Lib
+Lif
+Lin
+Liu
+Liz
+Lon
+Lou
+Low
+Luc
+Lum
+Luo
+Lux
+MAC
+MAD
+MAG
+MAN
+MAO
+MAP
+MAR
+MAS
+MAT
+MAX
+MAY
+MBA
+MBC
+MBO
+MBR
+MBS
+MCA
+MCC
+MCM
+MCN
+MCP
+MCS
+MCU
+MDA
+MDI
+MDL
+MDR
+MDS
+MEN
+MES
+MFA
+MFC
+MHC
+MHz
+MIB
+MIC
+MID
+MIL
+MIN
+MIS
+MIT
+MIX
+MKV
+MLC
+MLF
+MMA
+MMC
+MMI
+MMO
+MMS
+MMX
+MOD
+MOM
+MOS
+MOV
+MPA
+MPC
+MPG
+MPI
+MPS
+MPV
+MPa
+MRC
+MRI
+MRO
+MRP
+MSA
+MSC
+MSI
+MSN
+MTI
+MTK
+MTS
+MTU
+MTV
+MVC
+MVP
+Mac
+Mag
+Maj
+Man
+Mar
+Max
+May
+Mic
+Min
+Mon
+Mou
+Mur
+NAD
+NAS
+NAT
+NBA
+NBC
+NBL
+NCT
+NDS
+NEC
+NEO
+NES
+NET
+NEW
+NEX
+NFA
+NFC
+NFL
+NFS
+NGC
+NGN
+NGO
+NHK
+NHL
+NIC
+NIH
+NLP
+NME
+NMR
+NOT
+NOW
+NOX
+NOx
+NPC
+NPN
+NPR
+NSA
+NSC
+NSF
+NSK
+NTN
+NTP
+NTT
+NTV
+NVH
+NWA
+NXT
+NYT
+Nic
+Nob
+Nor
+Nov
+Now
+Nur
+OAD
+OBD
+OCG
+OCP
+OCR
+OCT
+ODM
+OEM
+OFF
+OGG
+OLE
+OMG
+ONE
+ONU
+OOO
+OPC
+OPP
+ORC
+OSD
+OSI
+OSS
+OST
+OTA
+OTC
+OTG
+OTT
+OUT
+OVA
+OVP
+Obj
+Off
+Oly
+Ope
+Oph
+Opt
+Our
+Out
+Ove
+PAC
+PAD
+PAH
+PAL
+PAM
+PAN
+PAS
+PBS
+PBT
+PCA
+PCB
+PCD
+PCI
+PCL
+PCM
+PCR
+PCS
+PCT
+PDA
+PDB
+PDC
+PDD
+PDF
+PDM
+PDP
+PDU
+PEG
+PEP
+PER
+PES
+PET
+PFA
+PFC
+PGA
+PGC
+PHP
+PHS
+PIC
+PID
+PIM
+PIN
+PKI
+PLA
+PLC
+PLD
+PLL
+PLM
+PMC
+PMI
+PMP
+PND
+PNG
+PNP
+POE
+POM
+PON
+POP
+POS
+PPA
+PPC
+PPG
+PPH
+PPI
+PPM
+PPP
+PPR
+PPS
+PPT
+PPV
+PRL
+PRO
+PSA
+PSD
+PSE
+PSG
+PSI
+PSK
+PSP
+PSS
+PSV
+PSW
+PSY
+PTA
+PTC
+PTH
+PTT
+PUB
+PVA
+PVC
+PVE
+PVP
+PWM
+Par
+Per
+Pic
+Pow
+Pro
+Pur
+QAM
+QDI
+QFP
+QGh
+QOS
+QPI
+QPS
+QRS
+QTL
+Qin
+Qua
+Que
+RAM
+RAP
+RAR
+RAS
+RAW
+RBC
+RCA
+RCS
+RDF
+RDS
+RED
+REF
+REG
+REM
+REX
+RFC
+RGB
+RIA
+RIM
+RIP
+RMB
+RMS
+RNA
+RNG
+ROC
+ROE
+ROI
+ROM
+RPC
+RPG
+RPM
+RRW
+RSA
+RSC
+RSI
+RSS
+RTA
+RTC
+RTK
+RTP
+RTS
+RTU
+RTX
+RUN
+RUS
+Ray
+Raz
+Ric
+Riv
+Rom
+Rou
+Rub
+Run
+Rus
+SAC
+SAE
+SAM
+SAN
+SAO
+SAP
+SAR
+SAS
+SAT
+SAY
+SBR
+SBS
+SCE
+SCH
+SCI
+SCM
+SCP
+SCR
+SDH
+SDI
+SDK
+SDR
+SDS
+SEA
+SEC
+SEE
+SEM
+SEO
+SER
+SET
+SFC
+SFP
+SGH
+SGI
+SGS
+SHA
+SHE
+SID
+SIG
+SIM
+SIP
+SIR
+SIS
+SKF
+SKT
+SKU
+SKY
+SLA
+SLC
+SLE
+SLG
+SLI
+SLR
+SLS
+SMA
+SMB
+SMC
+SMD
+SMG
+SMI
+SMP
+SMS
+SMT
+SNK
+SNP
+SNR
+SNS
+SOA
+SOC
+SOD
+SOI
+SOP
+SOS
+SPA
+SPC
+SPD
+SPE
+SPF
+SPI
+SPR
+SPS
+SPT
+SPV
+SQL
+SQU
+SRS
+SRT
+SSA
+SSC
+SSD
+SSE
+SSH
+SSL
+SSR
+SSS
+SST
+STC
+STD
+STK
+STL
+STM
+STN
+STP
+STR
+STS
+SUB
+SUN
+SUV
+SVC
+SVD
+SVG
+SVM
+SWF
+SXG
+SYN
+SYS
+Sch
+Ser
+She
+Siz
+Som
+Sou
+Squ
+Sub
+Sum
+Sun
+Sup
+Suz
+TAB
+TAC
+TAG
+TAO
+TBC
+TBM
+TBS
+TCG
+TCL
+TCM
+TCO
+TCP
+TCR
+TCS
+TCT
+TDD
+TDI
+TDM
+TDP
+TDS
+TEC
+TED
+TEL
+TEM
+TES
+TEU
+TEX
+TFT
+TGA
+TGF
+TGV
+THD
+THE
+TIA
+TIF
+TKO
+TLC
+TLS
+TMD
+TMP
+TMS
+TMT
+TNA
+TNF
+TNT
+TOC
+TOD
+TOE
+TOM
+TOP
+TPC
+TPE
+TPM
+TPO
+TPP
+TPR
+TPS
+TPU
+TQM
+TSC
+TSH
+TSI
+TSP
+TTL
+TTS
+TTT
+TUV
+TVB
+TVC
+TVP
+TVS
+TWO
+TXT
+Tay
+The
+Tom
+Tou
+Tow
+Tur
+UAR
+UBC
+UCC
+UCL
+UDP
+UFC
+UFO
+UGC
+UHF
+UIP
+UMD
+UML
+UNI
+UPC
+UPS
+URL
+USA
+USB
+USD
+USM
+USP
+USS
+UTC
+UTF
+UTP
+UTR
+UVA
+UVB
+UWB
+UZI
+Umb
+Uni
+Upp
+Uzi
+VAC
+VAR
+VBA
+VBR
+VBS
+VCC
+VCD
+VCR
+VDC
+VDE
+VGA
+VHF
+VHS
+VIA
+VII
+VIP
+VIS
+VMw
+VOA
+VOB
+VOC
+VOD
+VOL
+VPN
+VPS
+VRP
+VSS
+VTE
+VVT
+Ver
+Vic
+Vid
+Vis
+Viv
+WAN
+WAP
+WAV
+WAY
+WBA
+WBC
+WBO
+WBS
+WCG
+WCW
+WDM
+WDS
+WEB
+WEP
+WEY
+WGK
+WHO
+WIN
+WMA
+WMS
+WMV
+WOW
+WPA
+WPF
+WPS
+WRC
+WSA
+WTA
+WTI
+WTO
+WVG
+WWE
+WWF
+WWW
+Way
+Wha
+Whe
+Whi
+Who
+Why
+WiF
+Win
+Wiz
+Wom
+Wor
+Wou
+XGA
+XII
+XML
+XPS
+XXX
+XYZ
+YAG
+YES
+YOU
+YZB
+Yin
+You
+Yua
+Yuk
+Yun
+ZIP
+ZOL
+Zer
+Zha
+Zhu
+Zom
+Zon
+Zou
+abb
+abc
+abo
+abs
+act
+adj
+aff
+all
+and
+ang
+any
+app
+aws
+bbb
+bbc
+bbq
+bbs
+but
+cAM
+cDN
+cGM
+can
+car
+cba
+cha
+chi
+col
+com
+con
+cor
+cou
+cpi
+cpu
+dan
+day
+des
+did
+dif
+dis
+div
+diy
+doc
+don
+dow
+eAA
+eSA
+ech
+eff
+emb
+emp
+end
+eng
+eqc
+equ
+euv
+eve
+exc
+exe
+exp
+fac
+fil
+fin
+fir
+fiv
+fla
+fly
+for
+fox
+fre
+fri
+gAS
+gdp
+gen
+giv
+gmp
+gon
+goo
+got
+gps
+gra
+gre
+gro
+had
+har
+has
+hav
+haz
+her
+his
+hiv
+hol
+hom
+hou
+how
+iBT
+iOS
+iPa
+iPh
+iPo
+iSC
+ima
+imp
+inc
+inf
+inj
+int
+ipa
+iph
+ipo
+isb
+iso
+jam
+jap
+jav
+jay
+jus
+kHz
+kJm
+kdj
+kin
+lay
+laz
+lck
+lea
+led
+let
+lib
+lif
+lin
+liq
+lis
+lit
+liv
+liz
+lly
+lng
+loc
+lof
+log
+loo
+los
+low
+mRN
+mac
+mad
+maj
+man
+mar
+mat
+max
+may
+maz
+mba
+men
+mic
+min
+mmH
+mod
+mon
+mor
+mys
+nVI
+nba
+nex
+nic
+not
+nov
+now
+nxp
+obj
+off
+one
+ope
+opp
+our
+out
+ove
+par
+pay
+per
+phe
+php
+piz
+pla
+pow
+ppp
+pre
+pro
+pvc
+qHD
+qgh
+qua
+que
+qui
+rRN
+ray
+raz
+rea
+rec
+red
+ref
+reg
+rem
+rep
+req
+res
+rev
+ric
+riv
+rmb
+rng
+rom
+rou
+say
+sch
+sha
+she
+shi
+sho
+sim
+sin
+siz
+som
+sou
+spa
+spe
+sql
+squ
+sta
+ste
+sto
+str
+sty
+sub
+suv
+tRN
+tha
+the
+thi
+thr
+tim
+tip
+top
+tow
+tpp
+tra
+tur
+tuv
+two
+ubc
+uiv
+unc
+und
+uni
+unk
+ups
+usb
+uva
+uvb
+uzi
+val
+var
+ver
+vie
+vip
+vis
+viv
+wan
+was
+way
+web
+wer
+wha
+whi
+who
+why
+wif
+wit
+wom
+won
+wor
+wou
+www
+xin
+xxx
+yin
+you
+zha
+zhi
+zho
+zhu
+zon
+zzf
+zzy
+AAAA
+AACS
+ABCD
+ACCA
+ACCE
+ACCP
+ACDC
+ACGN
+ACID
+ACPI
+ACTH
+ADHD
+ADPC
+ADSL
+AIDS
+AJAX
+ALPH
+AMEX
+AMOL
+ANGE
+ANSI
+ANSY
+APEC
+APPL
+APTE
+ARDS
+ARPA
+ARPG
+ASCE
+ASCI
+ASIA
+ASIC
+ASIN
+ASME
+ASSO
+ASTM
+ASUS
+AUDI
+AUTO
+AVCH
+AWAR
+Andr
+BABY
+BACK
+BAND
+BANG
+BANK
+BASI
+BASS
+BATT
+BEAS
+BEAT
+BEST
+BEYO
+BIGB
+BIOS
+BLAC
+BLEA
+BLOG
+BLOO
+BLUE
+BOBO
+BOOK
+BOOL
+BOOM
+BOPP
+BOSS
+BOYS
+BRAV
+BREA
+BUFF
+Buck
+Buff
+Bull
+Bung
+Buzz
+CADC
+CALL
+CAPC
+CAPP
+CARD
+CASE
+CASI
+CAST
+CATI
+CATV
+CAXA
+CCFL
+CCIE
+CCNA
+CCTV
+CDMA
+CEPA
+CERN
+CHAN
+CHAP
+CHAR
+CHEN
+CHIN
+CHOR
+CIMS
+CIPA
+CISC
+CITE
+CITY
+CLAM
+CLAN
+CLAS
+CLOS
+CLUB
+CMMB
+CMMI
+CMOS
+CMYK
+CNAS
+CNBC
+CNBL
+CNKI
+CNNI
+COCO
+CODE
+COLL
+COLO
+COMB
+COME
+COMI
+COMP
+CONT
+COOL
+CORB
+CORE
+COSM
+COSP
+COST
+COUN
+COVI
+CPLD
+CREA
+CROS
+CSCD
+CSDN
+CSMA
+CSOL
+CSSC
+CSTN
+CTRL
+CUBA
+CUDA
+CURR
+CVBS
+Chin
+Chur
+DANC
+DARK
+DARP
+DASH
+DATA
+DAYS
+DCDC
+DDNS
+DDOS
+DDRI
+DELL
+DEMO
+DESI
+DEST
+DHCP
+DIGI
+DIMM
+DISC
+DIVX
+DLNA
+DOHC
+DOTA
+DOWN
+DRAG
+DRAM
+DREA
+DRIV
+DSLR
+DVDC
+DVGA
+DWDM
+DWOR
+EAST
+EASY
+EBIT
+ECMO
+EDGE
+EDIT
+EDTA
+EGFR
+EINE
+ELIS
+ELLE
+EMBA
+ENER
+ENGI
+ENTE
+EPDM
+EPIS
+EPON
+EPSO
+EPUB
+ERCP
+ERRO
+ESET
+ESPN
+ETSI
+EVDO
+EVER
+EXCE
+EXIL
+EXPO
+Ever
+Exch
+Exer
+FACE
+FALS
+FANS
+FANU
+FAST
+FDDI
+FIBA
+FIDI
+FIFA
+FIFO
+FILE
+FINA
+FIRE
+FIRS
+FISH
+FIVE
+FLAC
+FLAS
+FLOW
+FMVP
+FORT
+FPGA
+FREE
+FROM
+FTTH
+FULL
+FWVG
+FXCM
+Fuck
+Full
+Fund
+Fung
+Fuzz
+GABA
+GALA
+GAME
+GANK
+GATT
+GEAR
+GENE
+GHOS
+GIRL
+GLON
+GMAT
+GNSS
+GOLD
+GOOD
+GOOG
+GPRS
+GREE
+GROU
+GSMG
+GUCC
+GUND
+GUTS
+Gund
+HACC
+HAPP
+HARD
+HART
+HDCP
+HDMI
+HDPE
+HDTV
+HEAD
+HEAR
+HELL
+HEPA
+HERO
+HIFI
+HIGH
+HIPH
+HKEY
+HOLD
+HOME
+HOST
+HOUS
+HPLC
+HSDP
+HSPA
+HTML
+HTTP
+HUNT
+Hugh
+Hung
+ICAN
+ICMP
+ICON
+IDEA
+IDOL
+IEEE
+IELT
+IETF
+IFPI
+IGBT
+IGMP
+IMAX
+IMDB
+INFO
+INTE
+IPAD
+IPTV
+ISBN
+ISDN
+ISIS
+ISOI
+ISRC
+ISSN
+ISTP
+ITER
+ITIL
+IUCN
+Inte
+Inve
+JACK
+JAPA
+JAVA
+JAZZ
+JBOD
+JOHN
+JOJO
+JOKE
+JOUR
+JPEG
+JUMP
+JUST
+Jack
+Jake
+Jazz
+John
+Joke
+July
+Jump
+Jung
+KING
+KISS
+KONA
+KOYO
+LASI
+LAST
+LEED
+LEEP
+LESS
+LEVE
+LEXU
+LIFE
+LIKE
+LIMI
+LINE
+LINK
+LINU
+LIST
+LIVE
+LLDP
+LOCA
+LOFT
+LOGO
+LOLI
+LONG
+LOOK
+LOVE
+LPGA
+LTPS
+LVDS
+Ligh
+Like
+Lily
+Lind
+Ling
+Liqu
+Live
+Luck
+Luke
+MACD
+MACH
+MAGI
+MALL
+MAMA
+MARK
+MAST
+MATL
+MATX
+MAYA
+MBLA
+MEDI
+MEGA
+MEMS
+MERS
+META
+MIDI
+MIDP
+MIMO
+MINI
+MIPS
+MISS
+MIUI
+MMOR
+MOBA
+MODB
+MODE
+MOMO
+MOOC
+MOON
+MORE
+MOSF
+MOTO
+MOVI
+MPEG
+MPLS
+MSCI
+MSDS
+MTBF
+MUSI
+Mach
+Make
+Maur
+Mazz
+NACH
+NADH
+NADP
+NAMC
+NAME
+NANA
+NAND
+NASA
+NASD
+NATO
+NAVE
+NCAA
+NCAP
+NCIS
+NEDC
+NEOP
+NERV
+NEST
+NEWS
+NEXT
+NICO
+NIGH
+NIKE
+NINE
+NOKI
+NOTE
+NOVA
+NSAI
+NTFS
+NTSC
+NULL
+NURB
+NVID
+NYSE
+Nove
+ODBC
+OECD
+OFDM
+OFFI
+OLAP
+OLED
+ONLI
+ONLY
+OPEC
+OPEN
+OPPO
+ORAC
+ORIC
+ORIG
+OSPF
+OVER
+Oper
+PACS
+PAGE
+PARK
+PART
+PASS
+PCMC
+PDCA
+PEEK
+PERC
+PERF
+PETS
+PHEV
+PHIL
+PHOT
+PICC
+PIEC
+PLAN
+PLAY
+PLUS
+PMMA
+PNAS
+POLO
+POSE
+POST
+POWE
+PPTP
+PPTV
+PRAD
+PROD
+PROF
+PROJ
+PSTN
+PTFE
+PUNK
+PVDF
+Pric
+Prin
+Priv
+Priz
+Prom
+QFII
+QVGA
+QVOD
+QWER
+Quic
+Quin
+Quiz
+RADI
+RAID
+RAIN
+REAC
+READ
+REAL
+REIT
+RESE
+RFID
+RIDE
+RISC
+RMON
+RMRM
+RMVB
+ROAD
+ROCK
+ROHS
+ROOT
+ROSE
+RTEC
+RWBY
+Ruby
+SAAS
+SAMS
+SARS
+SATA
+SCAD
+SCAR
+SCDM
+SCHO
+SCIE
+SCSI
+SDHC
+SDMM
+SDRA
+SDSD
+SDXC
+SECA
+SECC
+SECT
+SEED
+SEGA
+SELE
+SERV
+SEVE
+SFDA
+SHIF
+SHIN
+SHOC
+SHOP
+SHOW
+SIDE
+SIEM
+SING
+SIZE
+SKIP
+SMAP
+SMAR
+SMIL
+SMTP
+SNMP
+SOAP
+SOCK
+SOHO
+SOLO
+SONG
+SONY
+SOSO
+SOUL
+SPAC
+SPCC
+SPDI
+SPEC
+SPEE
+SPIE
+SPOR
+SPSS
+SRAM
+SSCI
+STAF
+STAG
+STAR
+STAT
+STEM
+STEP
+STER
+STOP
+STOR
+STUD
+STYL
+SUMM
+SUPE
+SUSE
+SWAT
+SWIF
+SWOT
+SYST
+Subj
+Sull
+Sund
+Sung
+Supp
+TABL
+TANK
+TCPI
+TDMA
+TEAM
+TECH
+TEST
+TEXT
+TFBO
+TFSI
+TFTP
+THIS
+THRE
+TIFF
+TIME
+TIMK
+TIPS
+TOEF
+TOKY
+TOSH
+TOUC
+TOUR
+TOWN
+TRAC
+TRIP
+TRIZ
+TRUE
+TVBS
+TVOC
+TWIC
+TYPE
+Ther
+Thin
+Thom
+Thou
+UCLA
+UHMW
+ULTR
+UMTS
+UNES
+UNIT
+UNIV
+UNIX
+Unic
+Unit
+Univ
+VAIO
+VCCI
+VEGF
+VERS
+VHDL
+VIDE
+VIER
+VIII
+VISA
+VISI
+VIST
+VIVO
+VLAN
+VLSI
+VOCA
+VOGU
+VOIP
+VRay
+VSAT
+Vick
+Vill
+WANG
+WAPI
+WASD
+WAVE
+WCBA
+WCDM
+WEEK
+WEST
+WHAT
+WHIT
+WIFI
+WIND
+WITH
+WLAN
+WORD
+WORK
+WORL
+WQVG
+WXGA
+Wang
+Wher
+WiMA
+Will
+Wind
+Wing
+XBOX
+XBRL
+XHTM
+XVID
+XXXX
+YAMA
+YANG
+YEAH
+YONE
+YOUN
+YOUR
+YOYO
+Yong
+Your
+ZAFT
+ZARA
+ZERO
+ZGMF
+ZHAN
+ZONE
+Zhon
+Zhou
+abby
+abou
+andr
+appl
+baby
+back
+blic
+call
+char
+chic
+chin
+coff
+coll
+comb
+comm
+comp
+cond
+cons
+cont
+dick
+diff
+ding
+dock
+doin
+dong
+down
+ever
+exch
+find
+foll
+four
+from
+fron
+goin
+good
+goog
+gove
+hack
+hall
+hand
+hang
+happ
+have
+here
+high
+home
+into
+inve
+jack
+java
+jazz
+jump
+jung
+just
+know
+life
+ligh
+like
+lily
+ling
+liqu
+live
+lock
+logo
+lond
+long
+look
+love
+macd
+mach
+make
+mapp
+mmer
+nove
+okay
+only
+oper
+oppo
+othe
+over
+play
+pray
+pric
+prin
+priv
+priz
+prod
+prom
+quic
+real
+requ
+righ
+scho
+shou
+show
+some
+star
+stat
+stay
+stom
+subj
+such
+suff
+supp
+take
+than
+they
+thin
+thou
+toke
+uber
+unic
+univ
+upon
+usdj
+user
+usin
+vill
+vivo
+wake
+wall
+wang
+want
+wave
+were
+what
+when
+wifi
+will
+wind
+wing
+with
+work
+xing
+xxxx
+year
+your
+zhon
+China
+Inter
+Journ
+china
+every
+inter
+iphon
+thing
+think
+where
+which
+Univer
+univer
+Windows
+windows
+##A
+##B
+##C
+##D
+##E
+##F
+##G
+##H
+##I
+##J
+##K
+##L
+##M
+##N
+##O
+##P
+##Q
+##R
+##S
+##T
+##U
+##V
+##W
+##X
+##Y
+##Z
+##a
+##b
+##c
+##d
+##e
+##f
+##g
+##h
+##i
+##j
+##k
+##l
+##m
+##n
+##o
+##p
+##q
+##r
+##s
+##t
+##u
+##v
+##w
+##x
+##y
+##z
+##AA
+##AB
+##AC
+##AD
+##AE
+##AF
+##AG
+##AH
+##AI
+##AK
+##AL
+##AM
+##AN
+##AO
+##AP
+##AQ
+##AR
+##AS
+##AT
+##AV
+##AW
+##AX
+##AY
+##AZ
+##BA
+##BB
+##BC
+##BE
+##BG
+##BI
+##BM
+##BN
+##BO
+##BP
+##BR
+##BS
+##BT
+##BU
+##BY
+##CA
+##CB
+##CC
+##CD
+##CE
+##CF
+##CG
+##CH
+##CI
+##CK
+##CL
+##CM
+##CN
+##CO
+##CP
+##CR
+##CS
+##CT
+##CU
+##DA
+##DB
+##DC
+##DD
+##DE
+##DI
+##DL
+##DM
+##DN
+##DO
+##DP
+##DR
+##DS
+##DT
+##DU
+##DX
+##DY
+##EA
+##EB
+##EC
+##ED
+##EE
+##EF
+##EG
+##EI
+##EK
+##EL
+##EM
+##EN
+##EO
+##EP
+##ER
+##ES
+##ET
+##EV
+##EW
+##EX
+##EY
+##FA
+##FC
+##FD
+##FE
+##FF
+##FI
+##FL
+##FO
+##FP
+##FR
+##FS
+##FT
+##FU
+##FX
+##Fi
+##GA
+##GC
+##GE
+##GF
+##GH
+##GI
+##GL
+##GN
+##GO
+##GP
+##GR
+##GS
+##GU
+##GY
+##HA
+##HC
+##HD
+##HE
+##HG
+##HI
+##HM
+##HN
+##HO
+##HP
+##HR
+##HS
+##HT
+##IA
+##IB
+##IC
+##ID
+##IE
+##IF
+##IG
+##II
+##IK
+##IL
+##IM
+##IN
+##IO
+##IP
+##IR
+##IS
+##IT
+##IU
+##IV
+##IX
+##IZ
+##JI
+##JO
+##Jo
+##Ju
+##KA
+##KE
+##KI
+##KK
+##KO
+##KU
+##KY
+##LA
+##LC
+##LD
+##LE
+##LF
+##LG
+##LI
+##LK
+##LL
+##LM
+##LO
+##LP
+##LS
+##LT
+##LU
+##LV
+##LY
+##MA
+##MB
+##MC
+##MD
+##ME
+##MF
+##MI
+##ML
+##MM
+##MN
+##MO
+##MP
+##MR
+##MS
+##MT
+##MV
+##MY
+##NA
+##NC
+##ND
+##NE
+##NG
+##NI
+##NJ
+##NK
+##NN
+##NO
+##NP
+##NS
+##NT
+##NU
+##NX
+##NY
+##NZ
+##OB
+##OC
+##OD
+##OE
+##OF
+##OG
+##OH
+##OI
+##OK
+##OL
+##OM
+##ON
+##OO
+##OP
+##OR
+##OS
+##OT
+##OU
+##OV
+##OW
+##OX
+##PA
+##PC
+##PD
+##PE
+##PF
+##PG
+##PH
+##PI
+##PL
+##PM
+##PO
+##PP
+##PR
+##PS
+##PT
+##PU
+##QU
+##Qu
+##RA
+##RB
+##RC
+##RD
+##RE
+##RF
+##RG
+##RH
+##RI
+##RK
+##RL
+##RM
+##RN
+##RO
+##RP
+##RR
+##RS
+##RT
+##RU
+##RY
+##SA
+##SB
+##SC
+##SD
+##SE
+##SF
+##SH
+##SI
+##SK
+##SL
+##SM
+##SN
+##SO
+##SP
+##SS
+##ST
+##SU
+##SY
+##TA
+##TC
+##TD
+##TE
+##TH
+##TI
+##TM
+##TO
+##TP
+##TR
+##TS
+##TT
+##TU
+##TV
+##TY
+##Tu
+##UB
+##UC
+##UD
+##UE
+##UF
+##UG
+##UI
+##UK
+##UL
+##UM
+##UN
+##UP
+##UR
+##US
+##UT
+##VA
+##VB
+##VC
+##VD
+##VE
+##VI
+##VO
+##VP
+##VR
+##VT
+##WA
+##WC
+##WE
+##WF
+##WI
+##WL
+##WM
+##WO
+##WS
+##XA
+##XC
+##XE
+##XG
+##XO
+##XP
+##XT
+##XX
+##XY
+##YA
+##YE
+##YL
+##YO
+##YP
+##YS
+##YT
+##ZA
+##ZB
+##ZE
+##ZI
+##ZO
+##ZR
+##ZU
+##ZX
+##ZZ
+##ab
+##ag
+##al
+##am
+##an
+##ar
+##as
+##at
+##ax
+##ay
+##az
+##bi
+##bj
+##bl
+##bo
+##by
+##ce
+##ch
+##ci
+##ck
+##cq
+##ct
+##dj
+##ed
+##en
+##er
+##ew
+##ex
+##ff
+##fi
+##gh
+##gn
+##ha
+##he
+##ho
+##hz
+##ic
+##id
+##im
+##in
+##is
+##it
+##iv
+##ix
+##iz
+##jj
+##jo
+##ke
+##ky
+##kz
+##ld
+##le
+##lf
+##ll
+##ly
+##mb
+##mp
+##na
+##nc
+##nd
+##ng
+##nj
+##nk
+##nn
+##nt
+##nz
+##ob
+##oj
+##ok
+##ol
+##om
+##on
+##op
+##or
+##ou
+##ow
+##ox
+##ph
+##pp
+##pu
+##pv
+##qf
+##ql
+##qq
+##qu
+##re
+##rk
+##ro
+##ry
+##sh
+##sq
+##st
+##th
+##ty
+##ub
+##ul
+##um
+##un
+##ur
+##us
+##uv
+##ux
+##uz
+##ve
+##vi
+##wn
+##ws
+##ww
+##xp
+##xx
+##xy
+##zh
+##zy
+##zz
+##ACE
+##ACH
+##ACT
+##ADE
+##AGE
+##AIN
+##AME
+##AND
+##ANG
+##ANO
+##ANT
+##ARD
+##ARE
+##ASS
+##AST
+##ATE
+##BER
+##BLE
+##BOX
+##BSD
+##Bay
+##CAD
+##CAL
+##CAM
+##COM
+##CSE
+##DEO
+##DER
+##DIA
+##DNA
+##DSL
+##DVD
+##EAM
+##EAR
+##ECT
+##EEN
+##ENS
+##ENT
+##ERA
+##ERS
+##ESE
+##ESS
+##FTA
+##GER
+##GHT
+##GIS
+##IAL
+##IBA
+##IBU
+##ICE
+##ICS
+##IDE
+##INA
+##INE
+##ING
+##INT
+##INY
+##ION
+##IPS
+##ITE
+##IVE
+##KER
+##KON
+##LAY
+##LLA
+##LOR
+##MAN
+##MAS
+##MAX
+##MES
+##NAD
+##NAL
+##NCE
+##NET
+##NEY
+##NIC
+##NNA
+##OCK
+##ODE
+##OME
+##ONE
+##ORA
+##OWS
+##Off
+##PAC
+##PER
+##PRS
+##RAN
+##RIS
+##RNA
+##ROM
+##RON
+##ROR
+##SCO
+##SHI
+##SIC
+##SOL
+##SON
+##SQL
+##TAL
+##TED
+##TER
+##TML
+##TON
+##TRA
+##UND
+##UNG
+##UPA
+##USB
+##USE
+##VEL
+##VER
+##VGA
+##VID
+##WER
+##You
+##abl
+##aby
+##ach
+##ack
+##act
+##ain
+##ake
+##all
+##aly
+##anc
+##and
+##ang
+##ank
+##app
+##ard
+##ark
+##art
+##ary
+##ash
+##ath
+##auv
+##ave
+##avi
+##azi
+##azy
+##azz
+##bVI
+##bby
+##ber
+##bje
+##ble
+##cGI
+##cho
+##com
+##cqu
+##day
+##der
+##ebo
+##ect
+##ell
+##emb
+##enc
+##eng
+##ent
+##erJ
+##ern
+##erv
+##ery
+##eve
+##ews
+##exp
+##ext
+##ezy
+##fer
+##ffe
+##fic
+##for
+##gaz
+##ger
+##ght
+##gin
+##hen
+##her
+##hev
+##hin
+##hon
+##hou
+##iRF
+##ial
+##ica
+##ice
+##ich
+##ick
+##iff
+##igh
+##ike
+##ill
+##ily
+##ime
+##ine
+##ing
+##ink
+##ion
+##iqu
+##ish
+##ith
+##ive
+##iza
+##ize
+##izz
+##jin
+##ker
+##kin
+##lDR
+##lay
+##laz
+##lex
+##lic
+##lin
+##liz
+##llo
+##lly
+##man
+##maz
+##men
+##mer
+##min
+##mpl
+##mpo
+##nGL
+##nRH
+##nal
+##ner
+##ngz
+##niz
+##now
+##nxp
+##oCA
+##obj
+##ock
+##oll
+##omb
+##ome
+##omm
+##omp
+##one
+##ong
+##ook
+##ork
+##orm
+##ort
+##ory
+##oul
+##oup
+##our
+##ous
+##out
+##ove
+##own
+##ows
+##per
+##phe
+##ply
+##por
+##ppl
+##ppy
+##qqu
+##qua
+##que
+##qui
+##raz
+##rch
+##ric
+##rou
+##son
+##tBI
+##tch
+##ter
+##the
+##tic
+##tim
+##tiv
+##tur
+##uch
+##uck
+##uct
+##uff
+##ugh
+##umb
+##ung
+##ure
+##urn
+##vel
+##ven
+##ver
+##vic
+##vid
+##vin
+##war
+##way
+##whe
+##wor
+##www
+##xxx
+##ymb
+##yth
+##zhe
+##zym
+##zzy
+##ATIO
+##CESS
+##CIAT
+##CTIO
+##CTOR
+##ENGI
+##ERSI
+##HCSD
+##INES
+##INUE
+##IONA
+##LOID
+##MENT
+##NEER
+##NOLO
+##NTER
+##NTSC
+##ORMA
+##OSHO
+##RISE
+##RNAT
+##RNET
+##SATA
+##SION
+##TION
+##TTLE
+##VERS
+##ally
+##arch
+##ayer
+##azer
+##azin
+##bert
+##book
+##chin
+##ctor
+##ding
+##echn
+##erPC
+##erVR
+##eriz
+##erve
+##ever
+##ffer
+##ffff
+##ffic
+##fter
+##ghly
+##hell
+##ical
+##iche
+##icke
+##ific
+##ight
+##iver
+##izon
+##izzy
+##king
+##lack
+##land
+##llow
+##mber
+##ngin
+##ning
+##omic
+##onom
+##othe
+##ouch
+##ough
+##ound
+##ower
+##pper
+##ppin
+##pter
+##ster
+##ther
+##tion
+##tive
+##tter
+##ture
+##urch
+##vely
+##ction
+##ctive
+##enter
+##erica
+##ional
+##thing

bert/Erlangshen-MegatronBert-1.3B-Chinese/config.json

@@ -0,0 +1 @@
+{"vocab_size": 21248, "hidden_size": 2048, "num_hidden_layers": 24, "num_attention_heads": 8, "hidden_act": "gelu_new", "intermediate_size": 8192, "hidden_dropout_prob": 0.1, "attention_probs_dropout_prob": 0.1, "max_position_embeddings": 512, "type_vocab_size": 2, "initializer_range": 0.02, "layer_norm_eps": 1e-12, "gradient_checkpointing": false, "position_embedding_type": "absolute", "use_cache": false, "model_type": "megatron-bert"}

bert/Erlangshen-MegatronBert-3.9B-Chinese/config.json

@@ -0,0 +1,21 @@
+{
+    "architectures": [
+        "MegatronBertForMaskedLM"
+    ],
+    "vocab_size": 21248,
+    "hidden_size": 2560,
+    "num_hidden_layers": 48,
+    "num_attention_heads": 40,
+    "hidden_act": "gelu",
+    "intermediate_size": 10240,
+    "hidden_dropout_prob": 0.1,
+    "attention_probs_dropout_prob": 0.1,
+    "max_position_embeddings": 512,
+    "type_vocab_size": 2,
+    "initializer_range": 0.02,
+    "layer_norm_eps": 1e-12,
+    "gradient_checkpointing": false,
+    "position_embedding_type": "absolute",
+    "use_cache": false,
+    "model_type": "megatron-bert"
+}

bert/Erlangshen-MegatronBert-3.9B-Chinese/special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

bert/Erlangshen-MegatronBert-3.9B-Chinese/tokenizer_config.json

@@ -0,0 +1,16 @@
+{
+
+  "cls_token": "[CLS]",
+  "do_basic_tokenize": true,
+  "do_lower_case": true,
+  "mask_token": "[MASK]",
+  "name_or_path": "/cognitive_comp/gaoxinyu/hf_hub/Erlangshen-MegatronBert-3.9B",
+  "never_split": null,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "special_tokens_map_file": null,
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "unk_token": "[UNK]"
+}

bert/bert-base-japanese-v3/.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

bert/bert-base-japanese-v3/README.md

@@ -0,0 +1,53 @@
+---
+license: apache-2.0
+datasets:
+- cc100
+- wikipedia
+language:
+- ja
+widget:
+- text: 東北大学で[MASK]の研究をしています。
+---
+
+# BERT base Japanese (unidic-lite with whole word masking, CC-100 and jawiki-20230102)
+
+This is a [BERT](https://github.com/google-research/bert) model pretrained on texts in the Japanese language.
+
+This version of the model processes input texts with word-level tokenization based on the Unidic 2.1.2 dictionary (available in [unidic-lite](https://pypi.org/project/unidic-lite/) package), followed by the WordPiece subword tokenization.
+Additionally, the model is trained with the whole word masking enabled for the masked language modeling (MLM) objective.
+
+The codes for the pretraining are available at [cl-tohoku/bert-japanese](https://github.com/cl-tohoku/bert-japanese/).
+
+## Model architecture
+
+The model architecture is the same as the original BERT base model; 12 layers, 768 dimensions of hidden states, and 12 attention heads.
+
+## Training Data
+
+The model is trained on the Japanese portion of [CC-100 dataset](https://data.statmt.org/cc-100/) and the Japanese version of Wikipedia.
+For Wikipedia, we generated a text corpus from the [Wikipedia Cirrussearch dump file](https://dumps.wikimedia.org/other/cirrussearch/) as of January 2, 2023.
+The corpus files generated from CC-100 and Wikipedia are 74.3GB and 4.9GB in size and consist of approximately 392M and 34M sentences, respectively.
+
+For the purpose of splitting texts into sentences, we used [fugashi](https://github.com/polm/fugashi) with [mecab-ipadic-NEologd](https://github.com/neologd/mecab-ipadic-neologd) dictionary (v0.0.7).
+
+## Tokenization
+
+The texts are first tokenized by MeCab with the Unidic 2.1.2 dictionary and then split into subwords by the WordPiece algorithm.
+The vocabulary size is 32768.
+
+We used [fugashi](https://github.com/polm/fugashi) and [unidic-lite](https://github.com/polm/unidic-lite) packages for the tokenization.
+
+## Training
+
+We trained the model first on the CC-100 corpus for 1M steps and then on the Wikipedia corpus for another 1M steps.
+For training of the MLM (masked language modeling) objective, we introduced whole word masking in which all of the subword tokens corresponding to a single word (tokenized by MeCab) are masked at once.
+
+For training of each model, we used a v3-8 instance of Cloud TPUs provided by [TPU Research Cloud](https://sites.research.google/trc/about/).
+
+## Licenses
+
+The pretrained models are distributed under the Apache License 2.0.
+
+## Acknowledgments
+
+This model is trained with Cloud TPUs provided by [TPU Research Cloud](https://sites.research.google/trc/about/) program.

bert/bert-base-japanese-v3/config.json

@@ -0,0 +1,19 @@
+{
+    "architectures": [
+        "BertForPreTraining"
+    ],
+    "attention_probs_dropout_prob": 0.1,
+    "hidden_act": "gelu",
+    "hidden_dropout_prob": 0.1,
+    "hidden_size": 768,
+    "initializer_range": 0.02,
+    "intermediate_size": 3072,
+    "layer_norm_eps": 1e-12,
+    "max_position_embeddings": 512,
+    "model_type": "bert",
+    "num_attention_heads": 12,
+    "num_hidden_layers": 12,
+    "pad_token_id": 0,
+    "type_vocab_size": 2,
+    "vocab_size": 32768
+}

bert/bert-base-japanese-v3/tokenizer_config.json

@@ -0,0 +1,10 @@
+{
+    "tokenizer_class": "BertJapaneseTokenizer",
+    "model_max_length": 512,
+    "do_lower_case": false,
+    "word_tokenizer_type": "mecab",
+    "subword_tokenizer_type": "wordpiece",
+    "mecab_kwargs": {
+        "mecab_dic": "unidic_lite"
+    }
+}

bert/bert-large-japanese-v2/.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

bert/bert-large-japanese-v2/README.md

@@ -0,0 +1,53 @@
+---
+license: apache-2.0
+datasets:
+- cc100
+- wikipedia
+language:
+- ja
+widget:
+- text: 東北大学で[MASK]の研究をしています。
+---
+
+# BERT large Japanese (unidic-lite with whole word masking, CC-100 and jawiki-20230102)
+
+This is a [BERT](https://github.com/google-research/bert) model pretrained on texts in the Japanese language.
+
+This version of the model processes input texts with word-level tokenization based on the Unidic 2.1.2 dictionary (available in [unidic-lite](https://pypi.org/project/unidic-lite/) package), followed by the WordPiece subword tokenization.
+Additionally, the model is trained with the whole word masking enabled for the masked language modeling (MLM) objective.
+
+The codes for the pretraining are available at [cl-tohoku/bert-japanese](https://github.com/cl-tohoku/bert-japanese/).
+
+## Model architecture
+
+The model architecture is the same as the original BERT large model; 24 layers, 1024 dimensions of hidden states, and 16 attention heads.
+
+## Training Data
+
+The model is trained on the Japanese portion of [CC-100 dataset](https://data.statmt.org/cc-100/) and the Japanese version of Wikipedia.
+For Wikipedia, we generated a text corpus from the [Wikipedia Cirrussearch dump file](https://dumps.wikimedia.org/other/cirrussearch/) as of January 2, 2023.
+The corpus files generated from CC-100 and Wikipedia are 74.3GB and 4.9GB in size and consist of approximately 392M and 34M sentences, respectively.
+
+For the purpose of splitting texts into sentences, we used [fugashi](https://github.com/polm/fugashi) with [mecab-ipadic-NEologd](https://github.com/neologd/mecab-ipadic-neologd) dictionary (v0.0.7).
+
+## Tokenization
+
+The texts are first tokenized by MeCab with the Unidic 2.1.2 dictionary and then split into subwords by the WordPiece algorithm.
+The vocabulary size is 32768.
+
+We used [fugashi](https://github.com/polm/fugashi) and [unidic-lite](https://github.com/polm/unidic-lite) packages for the tokenization.
+
+## Training
+
+We trained the model first on the CC-100 corpus for 1M steps and then on the Wikipedia corpus for another 1M steps.
+For training of the MLM (masked language modeling) objective, we introduced whole word masking in which all of the subword tokens corresponding to a single word (tokenized by MeCab) are masked at once.
+
+For training of each model, we used a v3-8 instance of Cloud TPUs provided by [TPU Research Cloud](https://sites.research.google/trc/about/).
+
+## Licenses
+
+The pretrained models are distributed under the Apache License 2.0.
+
+## Acknowledgments
+
+This model is trained with Cloud TPUs provided by [TPU Research Cloud](https://sites.research.google/trc/about/) program.

bert/bert-large-japanese-v2/config.json

@@ -0,0 +1,19 @@
+{
+    "architectures": [
+        "BertForPreTraining"
+    ],
+    "attention_probs_dropout_prob": 0.1,
+    "hidden_act": "gelu",
+    "hidden_dropout_prob": 0.1,
+    "hidden_size": 1024,
+    "initializer_range": 0.02,
+    "intermediate_size": 4096,
+    "layer_norm_eps": 1e-12,
+    "max_position_embeddings": 512,
+    "model_type": "bert",
+    "num_attention_heads": 16,
+    "num_hidden_layers": 24,
+    "pad_token_id": 0,
+    "type_vocab_size": 2,
+    "vocab_size": 32768
+}

bert/bert-large-japanese-v2/tokenizer_config.json

@@ -0,0 +1,10 @@
+{
+    "tokenizer_class": "BertJapaneseTokenizer",
+    "model_max_length": 512,
+    "do_lower_case": false,
+    "word_tokenizer_type": "mecab",
+    "subword_tokenizer_type": "wordpiece",
+    "mecab_kwargs": {
+        "mecab_dic": "unidic_lite"
+    }
+}

bert/bert_models.json

@@ -0,0 +1,14 @@
+{
+    "deberta-v2-large-japanese-char-wwm": {
+        "repo_id": "ku-nlp/deberta-v2-large-japanese-char-wwm",
+        "files": ["pytorch_model.bin"]
+    },
+    "chinese-roberta-wwm-ext-large": {
+        "repo_id": "hfl/chinese-roberta-wwm-ext-large",
+        "files": ["pytorch_model.bin"]
+    },
+    "deberta-v3-large": {
+        "repo_id": "microsoft/deberta-v3-large",
+        "files": ["spm.model", "pytorch_model.bin"]
+    }
+}

bert/chinese-roberta-wwm-ext-large/.gitattributes

@@ -0,0 +1,9 @@
+*.bin.* filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tar.gz filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text

bert/chinese-roberta-wwm-ext-large/README.md

@@ -0,0 +1,57 @@
+---
+language:
+- zh
+tags:
+- bert
+license: "apache-2.0"
+---
+
+# Please use 'Bert' related functions to load this model!
+
+## Chinese BERT with Whole Word Masking
+For further accelerating Chinese natural language processing, we provide **Chinese pre-trained BERT with Whole Word Masking**.
+
+**[Pre-Training with Whole Word Masking for Chinese BERT](https://arxiv.org/abs/1906.08101)**
+Yiming Cui, Wanxiang Che, Ting Liu, Bing Qin, Ziqing Yang, Shijin Wang, Guoping Hu
+
+This repository is developed based on：https://github.com/google-research/bert
+
+You may also interested in,
+- Chinese BERT series: https://github.com/ymcui/Chinese-BERT-wwm
+- Chinese MacBERT: https://github.com/ymcui/MacBERT
+- Chinese ELECTRA: https://github.com/ymcui/Chinese-ELECTRA
+- Chinese XLNet: https://github.com/ymcui/Chinese-XLNet
+- Knowledge Distillation Toolkit - TextBrewer: https://github.com/airaria/TextBrewer
+
+More resources by HFL: https://github.com/ymcui/HFL-Anthology
+
+## Citation
+If you find the technical report or resource is useful, please cite the following technical report in your paper.
+- Primary: https://arxiv.org/abs/2004.13922
+```
+@inproceedings{cui-etal-2020-revisiting,
+    title = "Revisiting Pre-Trained Models for {C}hinese Natural Language Processing",
+    author = "Cui, Yiming  and
+      Che, Wanxiang  and
+      Liu, Ting  and
+      Qin, Bing  and
+      Wang, Shijin  and
+      Hu, Guoping",
+    booktitle = "Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing: Findings",
+    month = nov,
+    year = "2020",
+    address = "Online",
+    publisher = "Association for Computational Linguistics",
+    url = "https://www.aclweb.org/anthology/2020.findings-emnlp.58",
+    pages = "657--668",
+}
+```
+- Secondary: https://arxiv.org/abs/1906.08101
+```
+@article{chinese-bert-wwm,
+  title={Pre-Training with Whole Word Masking for Chinese BERT},
+  author={Cui, Yiming and Che, Wanxiang and Liu, Ting and Qin, Bing and Yang, Ziqing and Wang, Shijin and Hu, Guoping},
+  journal={arXiv preprint arXiv:1906.08101},
+  year={2019}
+ }
+```

bert/chinese-roberta-wwm-ext-large/added_tokens.json

@@ -0,0 +1 @@
+{}

bert/chinese-roberta-wwm-ext-large/config.json

@@ -0,0 +1,28 @@
+{
+  "architectures": [
+    "BertForMaskedLM"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
+  "directionality": "bidi",
+  "eos_token_id": 2,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 1024,
+  "initializer_range": 0.02,
+  "intermediate_size": 4096,
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 24,
+  "output_past": true,
+  "pad_token_id": 0,
+  "pooler_fc_size": 768,
+  "pooler_num_attention_heads": 12,
+  "pooler_num_fc_layers": 3,
+  "pooler_size_per_head": 128,
+  "pooler_type": "first_token_transform",
+  "type_vocab_size": 2,
+  "vocab_size": 21128
+}

bert/chinese-roberta-wwm-ext-large/special_tokens_map.json

@@ -0,0 +1 @@
+{"unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]"}

bert/chinese-roberta-wwm-ext-large/tokenizer_config.json

@@ -0,0 +1 @@
+{"init_inputs": []}

bert/deberta-v2-large-japanese-char-wwm/.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

bert/deberta-v2-large-japanese-char-wwm/README.md

@@ -0,0 +1,89 @@
+---
+language: ja
+license: cc-by-sa-4.0
+library_name: transformers
+tags:
+  - deberta
+  - deberta-v2
+  - fill-mask
+  - character
+  - wwm
+datasets:
+  - wikipedia
+  - cc100
+  - oscar
+metrics:
+  - accuracy
+mask_token: "[MASK]"
+widget:
+    - text: "京都大学で自然言語処理を[MASK][MASK]する。"
+---
+
+# Model Card for Japanese character-level DeBERTa V2 large
+
+## Model description
+
+This is a Japanese DeBERTa V2 large model pre-trained on Japanese Wikipedia, the Japanese portion of CC-100, and the Japanese portion of OSCAR.
+This model is trained with character-level tokenization and whole word masking.
+
+## How to use
+
+You can use this model for masked language modeling as follows:
+
+```python
+from transformers import AutoTokenizer, AutoModelForMaskedLM
+tokenizer = AutoTokenizer.from_pretrained('ku-nlp/deberta-v2-large-japanese-char-wwm')
+model = AutoModelForMaskedLM.from_pretrained('ku-nlp/deberta-v2-large-japanese-char-wwm')
+
+sentence = '京都大学で自然言語処理を[MASK][MASK]する。'
+encoding = tokenizer(sentence, return_tensors='pt')
+...
+```
+
+You can also fine-tune this model on downstream tasks.
+
+## Tokenization
+
+There is no need to tokenize texts in advance, and you can give raw texts to the tokenizer.
+The texts are tokenized into character-level tokens by [sentencepiece](https://github.com/google/sentencepiece).
+
+## Training data
+
+We used the following corpora for pre-training:
+
+- Japanese Wikipedia (as of 20221020, 3.2GB, 27M sentences, 1.3M documents)
+- Japanese portion of CC-100 (85GB, 619M sentences, 66M documents)
+- Japanese portion of OSCAR (54GB, 326M sentences, 25M documents)
+
+Note that we filtered out documents annotated with "header", "footer", or "noisy" tags in OSCAR.
+Also note that Japanese Wikipedia was duplicated 10 times to make the total size of the corpus comparable to that of CC-100 and OSCAR. As a result, the total size of the training data is 171GB.
+
+## Training procedure
+
+We first segmented texts in the corpora into words using [Juman++ 2.0.0-rc3](https://github.com/ku-nlp/jumanpp/releases/tag/v2.0.0-rc3) for whole word masking.
+Then, we built a sentencepiece model with 22,012 tokens including all characters that appear in the training corpus.
+
+We tokenized raw corpora into character-level subwords using the sentencepiece model and trained the Japanese DeBERTa model using [transformers](https://github.com/huggingface/transformers) library.
+The training took 26 days using 16 NVIDIA A100-SXM4-40GB GPUs.
+
+The following hyperparameters were used during pre-training:
+
+- learning_rate: 1e-4
+- per_device_train_batch_size: 26
+- distributed_type: multi-GPU
+- num_devices: 16
+- gradient_accumulation_steps: 8
+- total_train_batch_size: 3,328
+- max_seq_length: 512
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-06
+- lr_scheduler_type: linear schedule with warmup (lr = 0 at 300k steps)
+- training_steps: 260,000
+- warmup_steps: 10,000
+
+The accuracy of the trained model on the masked language modeling task was 0.795.
+The evaluation set consists of 5,000 randomly sampled documents from each of the training corpora.
+
+## Acknowledgments
+
+This work was supported by Joint Usage/Research Center for Interdisciplinary Large-scale Information Infrastructures (JHPCN) through General Collaboration Project no. jh221004, "Developing a Platform for Constructing and Sharing of Large-Scale Japanese Language Models".
+For training models, we used the mdx: a platform for the data-driven future.

bert/deberta-v2-large-japanese-char-wwm/config.json

@@ -0,0 +1,37 @@
+{
+  "architectures": [
+    "DebertaV2ForMaskedLM"
+  ],
+  "attention_head_size": 64,
+  "attention_probs_dropout_prob": 0.1,
+  "conv_act": "gelu",
+  "conv_kernel_size": 3,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 1024,
+  "initializer_range": 0.02,
+  "intermediate_size": 4096,
+  "layer_norm_eps": 1e-07,
+  "max_position_embeddings": 512,
+  "max_relative_positions": -1,
+  "model_type": "deberta-v2",
+  "norm_rel_ebd": "layer_norm",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 24,
+  "pad_token_id": 0,
+  "pooler_dropout": 0,
+  "pooler_hidden_act": "gelu",
+  "pooler_hidden_size": 1024,
+  "pos_att_type": [
+    "p2c",
+    "c2p"
+  ],
+  "position_biased_input": false,
+  "position_buckets": 256,
+  "relative_attention": true,
+  "share_att_key": true,
+  "torch_dtype": "float16",
+  "transformers_version": "4.25.1",
+  "type_vocab_size": 0,
+  "vocab_size": 22012
+}

bert/deberta-v2-large-japanese-char-wwm/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf0dab8ad87bd7c22e85ec71e04f2240804fda6d33196157d6b5923af6ea1201
+size 1318456639

bert/deberta-v2-large-japanese-char-wwm/special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

bert/deberta-v2-large-japanese-char-wwm/tokenizer_config.json

@@ -0,0 +1,19 @@
+{
+  "cls_token": "[CLS]",
+  "do_lower_case": false,
+  "do_subword_tokenize": true,
+  "do_word_tokenize": true,
+  "jumanpp_kwargs": null,
+  "mask_token": "[MASK]",
+  "mecab_kwargs": null,
+  "model_max_length": 1000000000000000019884624838656,
+  "never_split": null,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "special_tokens_map_file": null,
+  "subword_tokenizer_type": "character",
+  "sudachi_kwargs": null,
+  "tokenizer_class": "BertJapaneseTokenizer",
+  "unk_token": "[UNK]",
+  "word_tokenizer_type": "basic"
+}

bert/deberta-v2-large-japanese/.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

bert/deberta-v2-large-japanese/README.md

@@ -0,0 +1,111 @@
+---
+language: ja
+license: cc-by-sa-4.0
+library_name: transformers
+tags:
+  - deberta
+  - deberta-v2
+  - fill-mask
+datasets:
+  - wikipedia
+  - cc100
+  - oscar
+metrics:
+  - accuracy
+mask_token: "[MASK]"
+widget:
+  - text: "京都 大学 で 自然 言語 処理 を [MASK] する 。"
+---
+
+# Model Card for Japanese DeBERTa V2 large
+
+## Model description
+
+This is a Japanese DeBERTa V2 large model pre-trained on Japanese Wikipedia, the Japanese portion of CC-100, and the
+Japanese portion of OSCAR.
+
+## How to use
+
+You can use this model for masked language modeling as follows:
+
+```python
+from transformers import AutoTokenizer, AutoModelForMaskedLM
+
+tokenizer = AutoTokenizer.from_pretrained('ku-nlp/deberta-v2-large-japanese')
+model = AutoModelForMaskedLM.from_pretrained('ku-nlp/deberta-v2-large-japanese')
+
+sentence = '京都 大学 で 自然 言語 処理 を [MASK] する 。'  # input should be segmented into words by Juman++ in advance
+encoding = tokenizer(sentence, return_tensors='pt')
+...
+```
+
+You can also fine-tune this model on downstream tasks.
+
+## Tokenization
+
+The input text should be segmented into words by [Juman++](https://github.com/ku-nlp/jumanpp) in
+advance. [Juman++ 2.0.0-rc3](https://github.com/ku-nlp/jumanpp/releases/tag/v2.0.0-rc3) was used for pre-training. Each
+word is tokenized into subwords by [sentencepiece](https://github.com/google/sentencepiece).
+
+## Training data
+
+We used the following corpora for pre-training:
+
+- Japanese Wikipedia (as of 20221020, 3.2GB, 27M sentences, 1.3M documents)
+- Japanese portion of CC-100 (85GB, 619M sentences, 66M documents)
+- Japanese portion of OSCAR (54GB, 326M sentences, 25M documents)
+
+Note that we filtered out documents annotated with "header", "footer", or "noisy" tags in OSCAR.
+Also note that Japanese Wikipedia was duplicated 10 times to make the total size of the corpus comparable to that of
+CC-100 and OSCAR. As a result, the total size of the training data is 171GB.
+
+## Training procedure
+
+We first segmented texts in the corpora into words using [Juman++](https://github.com/ku-nlp/jumanpp).
+Then, we built a sentencepiece model with 32000 tokens including words ([JumanDIC](https://github.com/ku-nlp/JumanDIC))
+and subwords induced by the unigram language model of [sentencepiece](https://github.com/google/sentencepiece).
+
+We tokenized the segmented corpora into subwords using the sentencepiece model and trained the Japanese DeBERTa model
+using [transformers](https://github.com/huggingface/transformers) library.
+The training took 36 days using 8 NVIDIA A100-SXM4-40GB GPUs.
+
+The following hyperparameters were used during pre-training:
+
+- learning_rate: 1e-4
+- per_device_train_batch_size: 18
+- distributed_type: multi-GPU
+- num_devices: 8
+- gradient_accumulation_steps: 16
+- total_train_batch_size: 2,304
+- max_seq_length: 512
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-06
+- lr_scheduler_type: linear schedule with warmup
+- training_steps: 300,000
+- warmup_steps: 10,000
+
+The accuracy of the trained model on the masked language modeling task was 0.799.
+The evaluation set consists of 5,000 randomly sampled documents from each of the training corpora.
+
+## Fine-tuning on NLU tasks
+
+We fine-tuned the following models and evaluated them on the dev set of JGLUE.
+We tuned learning rate and training epochs for each model and task
+following [the JGLUE paper](https://www.jstage.jst.go.jp/article/jnlp/30/1/30_63/_pdf/-char/ja).
+
+| Model                         | MARC-ja/acc | JSTS/pearson | JSTS/spearman | JNLI/acc | JSQuAD/EM | JSQuAD/F1 | JComQA/acc |
+|-------------------------------|-------------|--------------|---------------|----------|-----------|-----------|------------|
+| Waseda RoBERTa base           | 0.965       | 0.913        | 0.876         | 0.905    | 0.853     | 0.916     | 0.853      |
+| Waseda RoBERTa large (seq512) | 0.969       | 0.925        | 0.890         | 0.928    | 0.910     | 0.955     | 0.900      |
+| LUKE Japanese base*           | 0.965       | 0.916        | 0.877         | 0.912    | -         | -         | 0.842      |
+| LUKE Japanese large*          | 0.965       | 0.932        | 0.902         | 0.927    | -         | -         | 0.893      |
+| DeBERTaV2 base                | 0.970       | 0.922        | 0.886         | 0.922    | 0.899     | 0.951     | 0.873      |
+| DeBERTaV2 large               | 0.968       | 0.925        | 0.892         | 0.924    | 0.912     | 0.959     | 0.890      |
+
+*The scores of LUKE are from [the official repository](https://github.com/studio-ousia/luke).
+
+## Acknowledgments
+
+This work was supported by Joint Usage/Research Center for Interdisciplinary Large-scale Information Infrastructures (
+JHPCN) through General Collaboration Project no. jh221004, "Developing a Platform for Constructing and Sharing of
+Large-Scale Japanese Language Models".
+For training models, we used the mdx: a platform for the data-driven future.

bert/deberta-v2-large-japanese/config.json

@@ -0,0 +1,38 @@
+{
+  "_name_or_path": "configs/deberta_v2_large.json",
+  "architectures": [
+    "DebertaV2ForMaskedLM"
+  ],
+  "attention_head_size": 64,
+  "attention_probs_dropout_prob": 0.1,
+  "conv_act": "gelu",
+  "conv_kernel_size": 3,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 1024,
+  "initializer_range": 0.02,
+  "intermediate_size": 4096,
+  "layer_norm_eps": 1e-07,
+  "max_position_embeddings": 512,
+  "max_relative_positions": -1,
+  "model_type": "deberta-v2",
+  "norm_rel_ebd": "layer_norm",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 24,
+  "pad_token_id": 0,
+  "pooler_dropout": 0,
+  "pooler_hidden_act": "gelu",
+  "pooler_hidden_size": 1024,
+  "pos_att_type": [
+    "p2c",
+    "c2p"
+  ],
+  "position_biased_input": false,
+  "position_buckets": 256,
+  "relative_attention": true,
+  "share_att_key": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.23.1",
+  "type_vocab_size": 0,
+  "vocab_size": 32000
+}

bert/deberta-v2-large-japanese/special_tokens_map.json

@@ -0,0 +1,9 @@
+{
+  "bos_token": "[CLS]",
+  "cls_token": "[CLS]",
+  "eos_token": "[SEP]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

bert/deberta-v2-large-japanese/tokenizer_config.json

@@ -0,0 +1,15 @@
+{
+  "bos_token": "[CLS]",
+  "cls_token": "[CLS]",
+  "do_lower_case": false,
+  "eos_token": "[SEP]",
+  "keep_accents": true,
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "sp_model_kwargs": {},
+  "special_tokens_map_file": null,
+  "split_by_punct": false,
+  "tokenizer_class": "DebertaV2Tokenizer",
+  "unk_token": "[UNK]"
+}

bert/deberta-v3-large/.gitattributes

@@ -0,0 +1,27 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bin.* filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zstandard filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

bert/deberta-v3-large/README.md

@@ -0,0 +1,93 @@
+---
+language: en
+tags:
+  - deberta
+  - deberta-v3
+  - fill-mask
+thumbnail: https://huggingface.co/front/thumbnails/microsoft.png
+license: mit
+---
+
+## DeBERTaV3: Improving DeBERTa using ELECTRA-Style Pre-Training with Gradient-Disentangled Embedding Sharing
+
+[DeBERTa](https://arxiv.org/abs/2006.03654) improves the BERT and RoBERTa models using disentangled attention and enhanced mask decoder. With those two improvements, DeBERTa out perform RoBERTa on a majority of NLU tasks with 80GB training data.
+
+In [DeBERTa V3](https://arxiv.org/abs/2111.09543), we further improved the efficiency of DeBERTa using ELECTRA-Style pre-training with Gradient Disentangled Embedding Sharing. Compared to DeBERTa,  our V3 version significantly improves the model performance on downstream tasks.  You can find more technique details about the new model from our [paper](https://arxiv.org/abs/2111.09543).
+
+Please check the [official repository](https://github.com/microsoft/DeBERTa) for more implementation details and updates.
+
+The DeBERTa V3 large model comes with 24 layers and a hidden size of 1024. It has 304M backbone parameters  with a vocabulary containing 128K tokens which introduces 131M parameters in the Embedding layer.  This model was trained using the 160GB data as DeBERTa V2.
+
+
+#### Fine-tuning on NLU tasks
+
+We present the dev results on SQuAD 2.0 and MNLI tasks.
+
+| Model             |Vocabulary(K)|Backbone #Params(M)| SQuAD 2.0(F1/EM) | MNLI-m/mm(ACC)|
+|-------------------|----------|-------------------|-----------|----------|
+| RoBERTa-large     |50     |304                | 89.4/86.5 | 90.2   |
+| XLNet-large       |32     |-                  | 90.6/87.9 | 90.8   |
+| DeBERTa-large     |50     |-                  | 90.7/88.0 | 91.3   |
+| **DeBERTa-v3-large**|128|304                  |  **91.5/89.0**| **91.8/91.9**|
+
+
+#### Fine-tuning with HF transformers
+
+```bash
+#!/bin/bash
+
+cd transformers/examples/pytorch/text-classification/
+
+pip install datasets
+export TASK_NAME=mnli
+
+output_dir="ds_results"
+
+num_gpus=8
+
+batch_size=8
+
+python -m torch.distributed.launch --nproc_per_node=${num_gpus} \
+  run_glue.py \
+  --model_name_or_path microsoft/deberta-v3-large \
+  --task_name $TASK_NAME \
+  --do_train \
+  --do_eval \
+  --evaluation_strategy steps \
+  --max_seq_length 256 \
+  --warmup_steps 50 \
+  --per_device_train_batch_size ${batch_size} \
+  --learning_rate 6e-6 \
+  --num_train_epochs 2 \
+  --output_dir $output_dir \
+  --overwrite_output_dir \
+  --logging_steps 1000 \
+  --logging_dir $output_dir
+
+```
+
+### Citation
+
+If you find DeBERTa useful for your work, please cite the following papers:
+
+``` latex
+@misc{he2021debertav3,
+      title={DeBERTaV3: Improving DeBERTa using ELECTRA-Style Pre-Training with Gradient-Disentangled Embedding Sharing},
+      author={Pengcheng He and Jianfeng Gao and Weizhu Chen},
+      year={2021},
+      eprint={2111.09543},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}
+```
+
+``` latex
+@inproceedings{
+he2021deberta,
+title={DEBERTA: DECODING-ENHANCED BERT WITH DISENTANGLED ATTENTION},
+author={Pengcheng He and Xiaodong Liu and Jianfeng Gao and Weizhu Chen},
+booktitle={International Conference on Learning Representations},
+year={2021},
+url={https://openreview.net/forum?id=XPZIaotutsD}
+}
+```

bert/deberta-v3-large/config.json

@@ -0,0 +1,22 @@
+{
+		"model_type": "deberta-v2",
+		"attention_probs_dropout_prob": 0.1,
+		"hidden_act": "gelu",
+		"hidden_dropout_prob": 0.1,
+		"hidden_size": 1024,
+		"initializer_range": 0.02,
+		"intermediate_size": 4096,
+		"max_position_embeddings": 512,
+		"relative_attention": true,
+		"position_buckets": 256,
+		"norm_rel_ebd": "layer_norm",
+		"share_att_key": true,
+		"pos_att_type": "p2c|c2p",
+		"layer_norm_eps": 1e-7,
+		"max_relative_positions": -1,
+		"position_biased_input": false,
+		"num_attention_heads": 16,
+		"num_hidden_layers": 24,
+		"type_vocab_size": 0,
+		"vocab_size": 128100
+}

bert/deberta-v3-large/generator_config.json

@@ -0,0 +1,22 @@
+{
+	"model_type": "deberta-v2",
+	"attention_probs_dropout_prob": 0.1,
+	"hidden_act": "gelu",
+	"hidden_dropout_prob": 0.1,
+	"hidden_size": 1024,
+	"initializer_range": 0.02,
+	"intermediate_size": 4096,
+	"max_position_embeddings": 512,
+	"relative_attention": true,
+	"position_buckets": 256,
+	"norm_rel_ebd": "layer_norm",
+	"share_att_key": true,
+	"pos_att_type": "p2c|c2p",
+	"layer_norm_eps": 1e-7,
+	"max_relative_positions": -1,
+	"position_biased_input": false,
+	"num_attention_heads": 16,
+	"num_hidden_layers": 12,
+	"type_vocab_size": 0,
+	"vocab_size": 128100
+}