Upload 7 files

README.md

@@ -1,3 +1,49 @@
----
-license: cc-by-nc-sa-4.0
----
+---
+license: cc-by-nc-sa-4.0
+widget:
+- text: AGTCGCCGCAACCCACACACGGACGGCTCGACGTGGCGATCTTAGCGGCTCATCCGCCCGGCCTCCCTCGCGCTCGATCGCTACGCAGCCTACGCTCGTTTCGCTCGGTTCGGTGGGTCGCCGATCTGGCGCCACGGCGGCTACCAACGACACCGCGATTGAGAAGGGTGCGTGGCCGTGGAGTCGTGGAGAAACGCCCGCGCGCGCGGGTGCGGCGAGGGACGACGACCGCGTCGTGCGGATCGATTGGCGGGGCAGCTCGGCGCCCCG
+tags:
+- DNA
+- biology
+- genomics
+---
+# Plant foundation DNA large language models
+
+The plant DNA large language models (LLMs) contain a series of foundation models based on different model architectures, which are pre-trained on various plant reference genomes.  
+All the models have a comparable model size between 90 MB and 150 MB, BPE tokenizer is used for tokenization and 8000 tokens are included in the vocabulary.  
+
+
+**Developed by:** zhangtaolab
+
+### Model Sources
+
+- **Repository:** [Plant DNA LLMs](https://github.com/zhangtaolab/plant_DNA_LLMs)
+- **Manuscript:** [Versatile applications of foundation DNA large language models in plant genomes]() 
+
+### Architecture
+
+The model is trained based on the State-Space Mamba-130m model with modified tokenizer specific for DNA sequence.
+
+This model is fine-tuned for predicting open chromatin.
+
+### How to use
+
+Install the runtime library first:
+```bash
+pip install transformers
+pip install causal-conv1d<=1.2.0
+pip install mamba-ssm<2.0.0
+```
+
+Since `transformers` library (version < 4.43.0) does not provide a MambaForSequenceClassification function, we wrote a script to train Mamba model for sequence classification.  
+An inference code can be found in our [GitHub](https://github.com/zhangtaolab/plant_DNA_LLMs).  
+Note that Plant DNAMamba model requires NVIDIA GPU to run.
+
+
+### Training data
+We use a custom MambaForSequenceClassification script to fine-tune the model.  
+Detailed training procedure can be found in our manuscript.
+
+
+#### Hardware
+Model was trained on a NVIDIA GTX4090 GPU (24 GB).

config.json

@@ -0,0 +1 @@
+{"d_model": 768, "n_layer": 24, "vocab_size": 1035, "ssm_cfg": {}, "rms_norm": true, "residual_in_fp32": true, "fused_add_norm": true, "pad_vocab_size_multiple": 1, "tie_embeddings": true}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b3db0c36ead21af8cb2d8cdd9c2c44d79b336870f2b81f18d77832b313581bf4
+size 365362714

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "cls_token": {
+    "content": "<cls>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

test_metrics.json

@@ -0,0 +1 @@
+{'test_loss': 0.9127596616744995, 'test_precision': 0.6193093093093093, 'test_recall': 0.6193093093093093, 'test_f1': 0.6193093093093093, 'test_matthews_correlation': 0.42900582085509864, 'test_AUROC_ovr': 0.7981367374130888, 'test_AUROC_ovo': 0.7981367374130888, 'test_runtime': 203.4055, 'test_samples_per_second': 163.712, 'test_steps_per_second': 10.236}

tokenizer_config.json

@@ -0,0 +1,44 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<mask>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<cls>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "<cls>",
+  "eos_token": null,
+  "mask_token": "<mask>",
+  "model_max_length": 512,
+  "pad_token": "<pad>",
+  "tokenizer_class": "EsmTokenizer",
+  "unk_token": "<unk>"
+}

vocab.txt

@@ -0,0 +1,1035 @@
+<unk>
+<pad>
+<mask>
+<cls>
+AAAAA
+AAAAT
+AAAAC
+AAAAG
+AAATA
+AAATT
+AAATC
+AAATG
+AAACA
+AAACT
+AAACC
+AAACG
+AAAGA
+AAAGT
+AAAGC
+AAAGG
+AATAA
+AATAT
+AATAC
+AATAG
+AATTA
+AATTT
+AATTC
+AATTG
+AATCA
+AATCT
+AATCC
+AATCG
+AATGA
+AATGT
+AATGC
+AATGG
+AACAA
+AACAT
+AACAC
+AACAG
+AACTA
+AACTT
+AACTC
+AACTG
+AACCA
+AACCT
+AACCC
+AACCG
+AACGA
+AACGT
+AACGC
+AACGG
+AAGAA
+AAGAT
+AAGAC
+AAGAG
+AAGTA
+AAGTT
+AAGTC
+AAGTG
+AAGCA
+AAGCT
+AAGCC
+AAGCG
+AAGGA
+AAGGT
+AAGGC
+AAGGG
+ATAAA
+ATAAT
+ATAAC
+ATAAG
+ATATA
+ATATT
+ATATC
+ATATG
+ATACA
+ATACT
+ATACC
+ATACG
+ATAGA
+ATAGT
+ATAGC
+ATAGG
+ATTAA
+ATTAT
+ATTAC
+ATTAG
+ATTTA
+ATTTT
+ATTTC
+ATTTG
+ATTCA
+ATTCT
+ATTCC
+ATTCG
+ATTGA
+ATTGT
+ATTGC
+ATTGG
+ATCAA
+ATCAT
+ATCAC
+ATCAG
+ATCTA
+ATCTT
+ATCTC
+ATCTG
+ATCCA
+ATCCT
+ATCCC
+ATCCG
+ATCGA
+ATCGT
+ATCGC
+ATCGG
+ATGAA
+ATGAT
+ATGAC
+ATGAG
+ATGTA
+ATGTT
+ATGTC
+ATGTG
+ATGCA
+ATGCT
+ATGCC
+ATGCG
+ATGGA
+ATGGT
+ATGGC
+ATGGG
+ACAAA
+ACAAT
+ACAAC
+ACAAG
+ACATA
+ACATT
+ACATC
+ACATG
+ACACA
+ACACT
+ACACC
+ACACG
+ACAGA
+ACAGT
+ACAGC
+ACAGG
+ACTAA
+ACTAT
+ACTAC
+ACTAG
+ACTTA
+ACTTT
+ACTTC
+ACTTG
+ACTCA
+ACTCT
+ACTCC
+ACTCG
+ACTGA
+ACTGT
+ACTGC
+ACTGG
+ACCAA
+ACCAT
+ACCAC
+ACCAG
+ACCTA
+ACCTT
+ACCTC
+ACCTG
+ACCCA
+ACCCT
+ACCCC
+ACCCG
+ACCGA
+ACCGT
+ACCGC
+ACCGG
+ACGAA
+ACGAT
+ACGAC
+ACGAG
+ACGTA
+ACGTT
+ACGTC
+ACGTG
+ACGCA
+ACGCT
+ACGCC
+ACGCG
+ACGGA
+ACGGT
+ACGGC
+ACGGG
+AGAAA
+AGAAT
+AGAAC
+AGAAG
+AGATA
+AGATT
+AGATC
+AGATG
+AGACA
+AGACT
+AGACC
+AGACG
+AGAGA
+AGAGT
+AGAGC
+AGAGG
+AGTAA
+AGTAT
+AGTAC
+AGTAG
+AGTTA
+AGTTT
+AGTTC
+AGTTG
+AGTCA
+AGTCT
+AGTCC
+AGTCG
+AGTGA
+AGTGT
+AGTGC
+AGTGG
+AGCAA
+AGCAT
+AGCAC
+AGCAG
+AGCTA
+AGCTT
+AGCTC
+AGCTG
+AGCCA
+AGCCT
+AGCCC
+AGCCG
+AGCGA
+AGCGT
+AGCGC
+AGCGG
+AGGAA
+AGGAT
+AGGAC
+AGGAG
+AGGTA
+AGGTT
+AGGTC
+AGGTG
+AGGCA
+AGGCT
+AGGCC
+AGGCG
+AGGGA
+AGGGT
+AGGGC
+AGGGG
+TAAAA
+TAAAT
+TAAAC
+TAAAG
+TAATA
+TAATT
+TAATC
+TAATG
+TAACA
+TAACT
+TAACC
+TAACG
+TAAGA
+TAAGT
+TAAGC
+TAAGG
+TATAA
+TATAT
+TATAC
+TATAG
+TATTA
+TATTT
+TATTC
+TATTG
+TATCA
+TATCT
+TATCC
+TATCG
+TATGA
+TATGT
+TATGC
+TATGG
+TACAA
+TACAT
+TACAC
+TACAG
+TACTA
+TACTT
+TACTC
+TACTG
+TACCA
+TACCT
+TACCC
+TACCG
+TACGA
+TACGT
+TACGC
+TACGG
+TAGAA
+TAGAT
+TAGAC
+TAGAG
+TAGTA
+TAGTT
+TAGTC
+TAGTG
+TAGCA
+TAGCT
+TAGCC
+TAGCG
+TAGGA
+TAGGT
+TAGGC
+TAGGG
+TTAAA
+TTAAT
+TTAAC
+TTAAG
+TTATA
+TTATT
+TTATC
+TTATG
+TTACA
+TTACT
+TTACC
+TTACG
+TTAGA
+TTAGT
+TTAGC
+TTAGG
+TTTAA
+TTTAT
+TTTAC
+TTTAG
+TTTTA
+TTTTT
+TTTTC
+TTTTG
+TTTCA
+TTTCT
+TTTCC
+TTTCG
+TTTGA
+TTTGT
+TTTGC
+TTTGG
+TTCAA
+TTCAT
+TTCAC
+TTCAG
+TTCTA
+TTCTT
+TTCTC
+TTCTG
+TTCCA
+TTCCT
+TTCCC
+TTCCG
+TTCGA
+TTCGT
+TTCGC
+TTCGG
+TTGAA
+TTGAT
+TTGAC
+TTGAG
+TTGTA
+TTGTT
+TTGTC
+TTGTG
+TTGCA
+TTGCT
+TTGCC
+TTGCG
+TTGGA
+TTGGT
+TTGGC
+TTGGG
+TCAAA
+TCAAT
+TCAAC
+TCAAG
+TCATA
+TCATT
+TCATC
+TCATG
+TCACA
+TCACT
+TCACC
+TCACG
+TCAGA
+TCAGT
+TCAGC
+TCAGG
+TCTAA
+TCTAT
+TCTAC
+TCTAG
+TCTTA
+TCTTT
+TCTTC
+TCTTG
+TCTCA
+TCTCT
+TCTCC
+TCTCG
+TCTGA
+TCTGT
+TCTGC
+TCTGG
+TCCAA
+TCCAT
+TCCAC
+TCCAG
+TCCTA
+TCCTT
+TCCTC
+TCCTG
+TCCCA
+TCCCT
+TCCCC
+TCCCG
+TCCGA
+TCCGT
+TCCGC
+TCCGG
+TCGAA
+TCGAT
+TCGAC
+TCGAG
+TCGTA
+TCGTT
+TCGTC
+TCGTG
+TCGCA
+TCGCT
+TCGCC
+TCGCG
+TCGGA
+TCGGT
+TCGGC
+TCGGG
+TGAAA
+TGAAT
+TGAAC
+TGAAG
+TGATA
+TGATT
+TGATC
+TGATG
+TGACA
+TGACT
+TGACC
+TGACG
+TGAGA
+TGAGT
+TGAGC
+TGAGG
+TGTAA
+TGTAT
+TGTAC
+TGTAG
+TGTTA
+TGTTT
+TGTTC
+TGTTG
+TGTCA
+TGTCT
+TGTCC
+TGTCG
+TGTGA
+TGTGT
+TGTGC
+TGTGG
+TGCAA
+TGCAT
+TGCAC
+TGCAG
+TGCTA
+TGCTT
+TGCTC
+TGCTG
+TGCCA
+TGCCT
+TGCCC
+TGCCG
+TGCGA
+TGCGT
+TGCGC
+TGCGG
+TGGAA
+TGGAT
+TGGAC
+TGGAG
+TGGTA
+TGGTT
+TGGTC
+TGGTG
+TGGCA
+TGGCT
+TGGCC
+TGGCG
+TGGGA
+TGGGT
+TGGGC
+TGGGG
+CAAAA
+CAAAT
+CAAAC
+CAAAG
+CAATA
+CAATT
+CAATC
+CAATG
+CAACA
+CAACT
+CAACC
+CAACG
+CAAGA
+CAAGT
+CAAGC
+CAAGG
+CATAA
+CATAT
+CATAC
+CATAG
+CATTA
+CATTT
+CATTC
+CATTG
+CATCA
+CATCT
+CATCC
+CATCG
+CATGA
+CATGT
+CATGC
+CATGG
+CACAA
+CACAT
+CACAC
+CACAG
+CACTA
+CACTT
+CACTC
+CACTG
+CACCA
+CACCT
+CACCC
+CACCG
+CACGA
+CACGT
+CACGC
+CACGG
+CAGAA
+CAGAT
+CAGAC
+CAGAG
+CAGTA
+CAGTT
+CAGTC
+CAGTG
+CAGCA
+CAGCT
+CAGCC
+CAGCG
+CAGGA
+CAGGT
+CAGGC
+CAGGG
+CTAAA
+CTAAT
+CTAAC
+CTAAG
+CTATA
+CTATT
+CTATC
+CTATG
+CTACA
+CTACT
+CTACC
+CTACG
+CTAGA
+CTAGT
+CTAGC
+CTAGG
+CTTAA
+CTTAT
+CTTAC
+CTTAG
+CTTTA
+CTTTT
+CTTTC
+CTTTG
+CTTCA
+CTTCT
+CTTCC
+CTTCG
+CTTGA
+CTTGT
+CTTGC
+CTTGG
+CTCAA
+CTCAT
+CTCAC
+CTCAG
+CTCTA
+CTCTT
+CTCTC
+CTCTG
+CTCCA
+CTCCT
+CTCCC
+CTCCG
+CTCGA
+CTCGT
+CTCGC
+CTCGG
+CTGAA
+CTGAT
+CTGAC
+CTGAG
+CTGTA
+CTGTT
+CTGTC
+CTGTG
+CTGCA
+CTGCT
+CTGCC
+CTGCG
+CTGGA
+CTGGT
+CTGGC
+CTGGG
+CCAAA
+CCAAT
+CCAAC
+CCAAG
+CCATA
+CCATT
+CCATC
+CCATG
+CCACA
+CCACT
+CCACC
+CCACG
+CCAGA
+CCAGT
+CCAGC
+CCAGG
+CCTAA
+CCTAT
+CCTAC
+CCTAG
+CCTTA
+CCTTT
+CCTTC
+CCTTG
+CCTCA
+CCTCT
+CCTCC
+CCTCG
+CCTGA
+CCTGT
+CCTGC
+CCTGG
+CCCAA
+CCCAT
+CCCAC
+CCCAG
+CCCTA
+CCCTT
+CCCTC
+CCCTG
+CCCCA
+CCCCT
+CCCCC
+CCCCG
+CCCGA
+CCCGT
+CCCGC
+CCCGG
+CCGAA
+CCGAT
+CCGAC
+CCGAG
+CCGTA
+CCGTT
+CCGTC
+CCGTG
+CCGCA
+CCGCT
+CCGCC
+CCGCG
+CCGGA
+CCGGT
+CCGGC
+CCGGG
+CGAAA
+CGAAT
+CGAAC
+CGAAG
+CGATA
+CGATT
+CGATC
+CGATG
+CGACA
+CGACT
+CGACC
+CGACG
+CGAGA
+CGAGT
+CGAGC
+CGAGG
+CGTAA
+CGTAT
+CGTAC
+CGTAG
+CGTTA
+CGTTT
+CGTTC
+CGTTG
+CGTCA
+CGTCT
+CGTCC
+CGTCG
+CGTGA
+CGTGT
+CGTGC
+CGTGG
+CGCAA
+CGCAT
+CGCAC
+CGCAG
+CGCTA
+CGCTT
+CGCTC
+CGCTG
+CGCCA
+CGCCT
+CGCCC
+CGCCG
+CGCGA
+CGCGT
+CGCGC
+CGCGG
+CGGAA
+CGGAT
+CGGAC
+CGGAG
+CGGTA
+CGGTT
+CGGTC
+CGGTG
+CGGCA
+CGGCT
+CGGCC
+CGGCG
+CGGGA
+CGGGT
+CGGGC
+CGGGG
+GAAAA
+GAAAT
+GAAAC
+GAAAG
+GAATA
+GAATT
+GAATC
+GAATG
+GAACA
+GAACT
+GAACC
+GAACG
+GAAGA
+GAAGT
+GAAGC
+GAAGG
+GATAA
+GATAT
+GATAC
+GATAG
+GATTA
+GATTT
+GATTC
+GATTG
+GATCA
+GATCT
+GATCC
+GATCG
+GATGA
+GATGT
+GATGC
+GATGG
+GACAA
+GACAT
+GACAC
+GACAG
+GACTA
+GACTT
+GACTC
+GACTG
+GACCA
+GACCT
+GACCC
+GACCG
+GACGA
+GACGT
+GACGC
+GACGG
+GAGAA
+GAGAT
+GAGAC
+GAGAG
+GAGTA
+GAGTT
+GAGTC
+GAGTG
+GAGCA
+GAGCT
+GAGCC
+GAGCG
+GAGGA
+GAGGT
+GAGGC
+GAGGG
+GTAAA
+GTAAT
+GTAAC
+GTAAG
+GTATA
+GTATT
+GTATC
+GTATG
+GTACA
+GTACT
+GTACC
+GTACG
+GTAGA
+GTAGT
+GTAGC
+GTAGG
+GTTAA
+GTTAT
+GTTAC
+GTTAG
+GTTTA
+GTTTT
+GTTTC
+GTTTG
+GTTCA
+GTTCT
+GTTCC
+GTTCG
+GTTGA
+GTTGT
+GTTGC
+GTTGG
+GTCAA
+GTCAT
+GTCAC
+GTCAG
+GTCTA
+GTCTT
+GTCTC
+GTCTG
+GTCCA
+GTCCT
+GTCCC
+GTCCG
+GTCGA
+GTCGT
+GTCGC
+GTCGG
+GTGAA
+GTGAT
+GTGAC
+GTGAG
+GTGTA
+GTGTT
+GTGTC
+GTGTG
+GTGCA
+GTGCT
+GTGCC
+GTGCG
+GTGGA
+GTGGT
+GTGGC
+GTGGG
+GCAAA
+GCAAT
+GCAAC
+GCAAG
+GCATA
+GCATT
+GCATC
+GCATG
+GCACA
+GCACT
+GCACC
+GCACG
+GCAGA
+GCAGT
+GCAGC
+GCAGG
+GCTAA
+GCTAT
+GCTAC
+GCTAG
+GCTTA
+GCTTT
+GCTTC
+GCTTG
+GCTCA
+GCTCT
+GCTCC
+GCTCG
+GCTGA
+GCTGT
+GCTGC
+GCTGG
+GCCAA
+GCCAT
+GCCAC
+GCCAG
+GCCTA
+GCCTT
+GCCTC
+GCCTG
+GCCCA
+GCCCT
+GCCCC
+GCCCG
+GCCGA
+GCCGT
+GCCGC
+GCCGG
+GCGAA
+GCGAT
+GCGAC
+GCGAG
+GCGTA
+GCGTT
+GCGTC
+GCGTG
+GCGCA
+GCGCT
+GCGCC
+GCGCG
+GCGGA
+GCGGT
+GCGGC
+GCGGG
+GGAAA
+GGAAT
+GGAAC
+GGAAG
+GGATA
+GGATT
+GGATC
+GGATG
+GGACA
+GGACT
+GGACC
+GGACG
+GGAGA
+GGAGT
+GGAGC
+GGAGG
+GGTAA
+GGTAT
+GGTAC
+GGTAG
+GGTTA
+GGTTT
+GGTTC
+GGTTG
+GGTCA
+GGTCT
+GGTCC
+GGTCG
+GGTGA
+GGTGT
+GGTGC
+GGTGG
+GGCAA
+GGCAT
+GGCAC
+GGCAG
+GGCTA
+GGCTT
+GGCTC
+GGCTG
+GGCCA
+GGCCT
+GGCCC
+GGCCG
+GGCGA
+GGCGT
+GGCGC
+GGCGG
+GGGAA
+GGGAT
+GGGAC
+GGGAG
+GGGTA
+GGGTT
+GGGTC
+GGGTG
+GGGCA
+GGGCT
+GGGCC
+GGGCG
+GGGGA
+GGGGT
+GGGGC
+GGGGG
+A
+T
+C
+G
+N
+<eos>
+<bos>