Upload weights and config

README.md

@@ -1,3 +1,64 @@
 ---
 license: cc-by-nc-sa-4.0
+tags:
+- Helical
+- RNA
+- Biology
+- Transformers
+- Genomics
+- Sequence
+library_name: transformers
 ---
+ # Helix-mRNA
+Helix-mRNA emerges as a hybrid state-space and transformer based model, leveraging both the efficient sequence processing capabilities of Mamba2's state-space architecture and the contextual understanding of transformer attention mechanisms, allowing for the best of both worlds between these two approaches. These traits make it particularly suitable for studying full-length transcripts, splice variants, and complex mRNA structural elements.
+
+We tokenize mRNA sequences at single-nucleotide resolution by mapping each nucleotide (A, C, U, G) and ambiguous base (N) to a unique integer. A further special character E is incorporated into the sequence, denoting the start of each codon. This fine-grained approach maximizes the model's ability to extract patterns from the sequences. Unlike coarser tokenization methods that might group nucleotides together or use k-mer based approaches, our single-nucleotide resolution preserves the full sequential information of the mRNA molecule. This simple yet effective encoding scheme ensures that no information is lost during the preprocessing stage, allowing the downstream model to learn directly from the raw sequence composition.
+
+# Helical<a name="helical"></a>
+
+#### Install the package
+
+Run the following to install the [Helical](https://github.com/helicalAI/helical) package via pip:
+```console
+pip install --upgrade helical
+```
+
+#### Generate Embeddings
+```python
+from helical import HelixmRNA, HelixmRNAConfig
+import torch
+
+device = "cuda" if torch.cuda.is_available() else "cpu"
+
+input_sequences = ["EACU"*20, "EAUG"*20, "EAUG"*20, "EACU"*20, "EAUU"*20]
+
+helix_mrna_config = HelixmRNAConfig(batch_size=5, device=device, max_length=100)
+helix_mrna = HelixmRNA()
+
+# prepare data for input to the model
+processed_input_data = helix_mrna.process_data(input_sequences)
+
+# generate the embeddings for the processed data
+embeddings = helix_mrna.get_embeddings(processed_input_data)
+```
+
+#### Fine-Tuning
+Classification fine-tuning example:
+```python
+from helical import HelixmRNAFineTuningModel, HelixmRNAConfig
+import torch
+
+device = "cuda" if torch.cuda.is_available() else "cpu"
+
+input_sequences = ["EACU"*20, "EAUG"*20, "EAUG"*20, "EACU"*20, "EAUU"*20]
+labels = [0, 2, 2, 0, 1]
+
+helixr_config = HelixmRNAConfig(batch_size=5, device=device)
+helixr_fine_tune = HelixmRNAFineTuningModel(helix_mrna_config=helixr_config, output_size=3, max_length=100)
+
+train_dataset = helixr_fine_tune.process_data(input_sequences)
+
+helixr_fine_tune.train(train_dataset=train_dataset, train_labels=labels)
+
+outputs = helixr_fine_tune.get_outputs(train_dataset)
+```

config.json

@@ -0,0 +1,46 @@
+{
+  "_name_or_path": "/home/models/helix-mRNA/helix_mrna_8l_256h_5.1883Mio_12288_2024-11-30_18-51-46/checkpoint-23000",
+  "architectures": [
+    "HelixmRNAForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "attn_implementation": "flash_attention_2",
+  "bos_token_id": 0,
+  "chunk_size": 256,
+  "conv_kernel": 4,
+  "eos_token_id": 2,
+  "expand": 2,
+  "head_dim": 32,
+  "hidden_act": "silu",
+  "hidden_size": 256,
+  "initializer_range": 0.1,
+  "intermediate_size": 512,
+  "layer_norm_epsilon": 1e-05,
+  "layers_block_type_string": "M+M*M+M+",
+  "model_type": "mamba2",
+  "n_groups": 1,
+  "num_attention_heads": 32,
+  "num_heads": 16,
+  "num_hidden_layers": 8,
+  "num_key_value_heads": 8,
+  "pad_token_id": 1,
+  "rescale_prenorm_residual": false,
+  "residual_in_fp32": true,
+  "rms_norm": true,
+  "state_size": 128,
+  "tie_word_embeddings": false,
+  "time_step_floor": 0.0001,
+  "time_step_limit": [
+    0.0,
+    Infinity
+  ],
+  "time_step_max": 0.1,
+  "time_step_min": 0.001,
+  "time_step_rank": 16,
+  "torch_dtype": "float32",
+  "transformers_version": "4.45.1",
+  "use_bias": false,
+  "use_cache": false,
+  "use_conv_bias": true,
+  "vocab_size": 14
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c0e977c4a14516495048cb5e08ec7998ec8a2a02c0f274e15c3515cae254be4
+size 10382896