Validation: 0%| | 0/6 [00:00<?, ?it/s]Training loss: 0.30

Validation Loss: 0.28
Validation Accuracy: 0.90
traing end, save model to :./saved_models/model_20240302-214915_lr1e-05_optAdamW_lossBCEWithLogitsLoss_batch16_epoch10.pt

app.py

@@ -1,4 +1,31 @@
 import streamlit as st
+from transformers import AutoTokenizer, AutoModelForSequenceClassification
+import torch
 
-x = st.slider('Select a value')
-st.write(x, 'squared is', x * x)
+# 加载模型和tokenizer
+tokenizer = AutoTokenizer.from_pretrained("your_model_directory")
+model = AutoModelForSequenceClassification.from_pretrained("your_model_directory", num_labels=8)
+model.eval()
+
+def predict(text):
+    inputs = tokenizer(text, return_tensors="pt", padding=True, truncation=True, max_length=512)
+    with torch.no_grad():
+        outputs = model(**inputs)
+        logits = outputs.logits
+    probabilities = torch.softmax(logits, dim=1).squeeze()
+    # 假设每个类别（SAS_Class和SDS_Class）都有4个概率值
+    sas_probs = probabilities[:4]  # 获取SAS_Class的概率
+    sds_probs = probabilities[4:]  # 获取SDS_Class的概率
+    return sas_probs, sds_probs
+
+# 创建Streamlit应用
+st.title("Multi-label Classification App")
+
+# 用户输入文本
+user_input = st.text_area("Enter text here", "Type something...")
+
+if st.button("Predict"):
+    # 显示预测结果
+    sas_probs, sds_probs = predict(user_input)
+    st.write("SAS_Class probabilities:", sas_probs.numpy())
+    st.write("SDS_Class probabilities:", sds_probs.numpy())

dataset/CustomDataset.py

@@ -1,40 +0,0 @@
-import torch
-from torch.utils.data import Dataset
-
-n_classes = 2
-
-class CustomDataset(Dataset):
-    def __init__(self, data, tokenizer, max_length=512):
-        self.data = data
-        self.tokenizer = tokenizer
-        self.max_length = max_length
-
-    def __len__(self):
-        return len(self.data)
-
-    def __getitem__(self, idx):
-        item = self.data[idx]
-        # 假设我们的数据是一个字典，包含"text"和"label"
-        text = item['Description']
-        label = item['label']
-
-        # 对文本进行编码
-        encoded = self.tokenizer.encode_plus(
-            text,
-            add_special_tokens=True,
-            max_length=self.max_length,
-            padding='max_length',
-            truncation=True,
-            return_attention_mask=True,
-            return_tensors='pt',
-        )
-        # 在这里添加任何需要的数据检查逻辑
-        # 例如，检查标签是否在预期的范围内
-        if label < 0 or label > n_classes:  # 假设n_classes是标签的数量
-            raise ValueError("Found an invalid label")
-
-        return {
-            'input_ids': encoded['input_ids'].flatten(),
-            'attention_mask': encoded['attention_mask'].flatten(),
-            'labels': torch.tensor(label, dtype=torch.long)
-        }

new.py

@@ -1,108 +0,0 @@
-from transformers import AdamW, get_linear_schedule_with_warmup, AutoTokenizer, AutoModelForSequenceClassification
-from torch.utils.data import DataLoader, RandomSampler, SequentialSampler
-import torch
-from sklearn.model_selection import train_test_split
-from dataset.load_dataset import df, prepare_dataset
-from torch.nn import BCEWithLogitsLoss
-from transformers import BertForSequenceClassification, BertConfig
-from tqdm.auto import tqdm
-from torch.cuda.amp import GradScaler, autocast
-
-
-epochs = 10
-tokenizer = AutoTokenizer.from_pretrained(
-    "pretrained_models/Bio_ClinicalBERT-finetuned-medicalcondition")  # 用于将文本转换为模型所需输入格式的tokenizer
-device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # 创建一个device对象，如果有可用的GPU就使用它，否则使用CPU
-
-# 分割数据集
-train_df, val_df = train_test_split(df, test_size=0.1)  # 以90%训练，10%验证的比例分割数据集
-
-# 准备训练和验证数据集
-train_dataset = prepare_dataset(train_df, tokenizer)
-val_dataset = prepare_dataset(val_df, tokenizer)
-# 现在train_dataloader和validation_dataloader已准备好，可用于模型训练和验证
-train_dataloader = DataLoader(train_dataset, sampler=RandomSampler(train_dataset), batch_size=16)
-validation_dataloader = DataLoader(val_dataset, sampler=SequentialSampler(val_dataset), batch_size=16)
-
-
-# 加载配置
-config = BertConfig.from_pretrained("pretrained_models/Bio_ClinicalBERT-finetuned-medicalcondition")
-config.num_labels = 8  # 调整为你的标签数量
-
-model = AutoModelForSequenceClassification.from_pretrained(
-    "pretrained_models/Bio_ClinicalBERT-finetuned-medicalcondition",config=config,ignore_mismatched_sizes=True).to(device)
-# 准备优化器和学习率调度器
-optimizer = AdamW(model.parameters(), lr=1e-5, eps=1e-8)
-total_steps = len(train_dataloader) * epochs  # epochs是您想要训练的轮数
-scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=0, num_training_steps=total_steps)
-loss_fn = BCEWithLogitsLoss()
-# 微调模型
-scaler = GradScaler()
-
-for epoch in range(epochs):  # 迭代多个epoch
-    print(f"\nEpoch {epoch + 1}/{epochs}")
-    print('-------------------------------')
-    model.train()
-    total_loss = 0
-    train_progress_bar = tqdm(train_dataloader, desc="Training", leave=False)
-    for step, batch in enumerate(train_progress_bar):
-        # 将数据加载到GPU
-        batch = tuple(t.to(device) for t in batch)
-        b_input_ids, b_input_mask, b_labels = batch
-        model.zero_grad()
-        # 前向传播
-        outputs = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask)
-        logits = outputs.logits
-        # 反向传播
-        loss = loss_fn(logits, b_labels)
-        total_loss += loss.item()
-        # loss.backward()
-        # optimizer.step()
-        # scheduler.step()
-        torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)
-        # 检查loss是否为nan
-        if torch.isnan(loss).any():
-            print(f"Loss is nan in epoch {epoch + 1}, step {step}.")
-            # 可选：打印出问题数据的更多信息或采取其他措施
-            # 注意：直接跳过可能不解决根本问题，最好检查为何loss会是nan
-            continue  # 跳过当前批次的反向传播和优化器步骤
-        scaler.scale(loss).backward()
-        scaler.step(optimizer)
-        scaler.update()
-        train_progress_bar.set_postfix({'loss': f"{loss.item():.2f}"})
-
-    # 评估阶段
-    avg_train_loss = total_loss / len(train_dataloader)
-    print(f"Training loss: {avg_train_loss:.2f}")
-
-    # 验证阶段
-    model.eval()
-    total_eval_accuracy = 0
-    eval_progress_bar = tqdm(validation_dataloader, desc="Validation", leave=False)
-
-    for batch in eval_progress_bar:
-        batch = tuple(t.to(device) for t in batch)
-        b_input_ids, b_input_mask, b_labels = batch
-        with torch.no_grad():
-            outputs = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask)
-
-        logits = outputs.logits
-        # predictions = torch.argmax(logits, dim=1).flatten()
-        # labels = b_labels.flatten()
-        # accuracy = (predictions == labels).cpu().numpy().mean()
-        # 使用sigmoid函数将logits转换为概率值
-        probs = torch.sigmoid(logits)
-        # 将概率高于0.5的预测为正类（1），低于0.5的预测为负类（0）
-        predictions = (probs > 0.5).int()
-
-        # 比较预测和真实标签
-        correct_predictions = (predictions == b_labels.int()).float()  # 确保标签也是整数类型
-        # 计算每个样本的正确预测的平均数，然后计算整个批次的平均值
-        accuracy_per_sample = correct_predictions.mean(dim=1)
-        accuracy = accuracy_per_sample.mean().item()
-        total_eval_accuracy += accuracy
-        # 更新进度条
-        eval_progress_bar.set_postfix({'accuracy': f"{accuracy:.2f}"})
-
-    avg_val_accuracy = total_eval_accuracy / len(validation_dataloader)
-    print(f"Validation Accuracy: {avg_val_accuracy:.2f}")

test.py

@@ -1,18 +0,0 @@
-# Load model directly
-import ast
-
-from transformers import AutoTokenizer, AutoModelForSequenceClassification
-import torch
-# assets_path = cached_assets_path(library_name="datasets", namespace="SQuAD", subfolder="download")
-# something_path = assets_path / "config.json" # Do anything you like in your assets folder !
-
-tokenizer = AutoTokenizer.from_pretrained("pretrained_models/Bio_ClinicalBERT-finetuned-medicalcondition")
-model = AutoModelForSequenceClassification.from_pretrained("pretrained_models/Bio_ClinicalBERT-finetuned-medicalcondition")
-input=tokenizer("I love using transformers for natural language processing.", return_tensors="pt")
-
-# 使用模型进行预测
-with torch.no_grad():
-    logits = model(**input).logits
-# 解析预测结果
-predicted_class_id = logits.argmax().item()
-print(f"Predicted class id: {predicted_class_id}")

test2.py

@@ -1,23 +0,0 @@
-from transformers import AutoTokenizer, AutoModelForSequenceClassification
-import torch
-
-# 指定预训练模型
-model_name = "bert-base-uncased"
-
-# 加载分词器和模型
-tokenizer = AutoTokenizer.from_pretrained(model_name,force_download=True, resume_download=False)
-model = AutoModelForSequenceClassification.from_pretrained(model_name,force_download=True, resume_download=False)
-
-# 要进行分类的文本
-text = "I love using transformers for natural language processing."
-
-# 使用分词器处理文本
-inputs = tokenizer(text, return_tensors="pt")
-
-# 使用模型进行预测
-with torch.no_grad():
-    logits = model(**inputs).logits
-
-# 解析预测结果
-predicted_class_id = logits.argmax().item()
-print(f"Predicted class id: {predicted_class_id}")

train.py

@@ -1,36 +1,131 @@
-import sagemaker
-import boto3
-from sagemaker.huggingface import HuggingFace
-
-try:
-    role = sagemaker.get_execution_role()
-except ValueError:
-    iam = boto3.client('iam')
-    role = iam.get_role(RoleName='sagemaker_execution_role')['Role']['Arn']
-
-hyperparameters = {
-    'model_name_or_path': 'emilyalsentzer/Bio_ClinicalBERT',
-    'output_dir': '/opt/ml/model'
-    # add your remaining hyperparameters
-    # more info here https://github.com/huggingface/transformers/tree/v4.37.0/examples/pytorch/text-classification
-}
-
-# git configuration to download our fine-tuning script
-git_config = {'repo': 'https://github.com/huggingface/transformers.git', 'branch': 'v4.37.0'}
-
-# creates Hugging Face estimator
-huggingface_estimator = HuggingFace(
-    entry_point='run_glue.py',
-    source_dir='./examples/pytorch/text-classification',
-    instance_type='ml.p3.2xlarge',
-    instance_count=1,
-    role=role,
-    git_config=git_config,
-    transformers_version='4.37.0',
-    pytorch_version='2.1.0',
-    py_version='py310',
-    hyperparameters=hyperparameters
-)
-
-# starting the train job
-huggingface_estimator.fit()
+from transformers import AdamW, get_linear_schedule_with_warmup, AutoTokenizer, AutoModelForSequenceClassification
+from torch.utils.data import DataLoader, RandomSampler, SequentialSampler
+import torch
+from sklearn.model_selection import train_test_split
+from dataset.load_dataset import df, prepare_dataset
+from torch.nn import BCEWithLogitsLoss
+from transformers import BertForSequenceClassification, BertConfig
+from tqdm.auto import tqdm
+from torch.cuda.amp import GradScaler, autocast
+from torch.utils.tensorboard import SummaryWriter
+import datetime
+
+# 初始化TensorBoard SummaryWriter
+current_time = datetime.datetime.now().strftime('%Y%m%d-%H%M%S')
+log_dir = f'runs/train_{current_time}'
+writer = SummaryWriter(log_dir)
+epochs = 10
+lr = 1e-5
+optimizer_name = 'AdamW'
+loss_fn_name = 'BCEWithLogitsLoss'
+batch_size = 16
+
+# 构建模型保存路径，包括重要参数
+model_save_name = f'model_{current_time}_lr{lr}_opt{optimizer_name}_loss{loss_fn_name}_batch{batch_size}_epoch{epochs}.pt'
+model_save_path = f'./saved_models/{model_save_name}'
+
+tokenizer = AutoTokenizer.from_pretrained(
+    "pretrained_models/Bio_ClinicalBERT-finetuned-medicalcondition")  # 用于将文本转换为模型所需输入格式的tokenizer
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")  # 创建一个device对象，如果有可用的GPU就使用它，否则使用CPU
+
+# 分割数据集
+train_df, val_df = train_test_split(df, test_size=0.1)  # 以90%训练，10%验证的比例分割数据集
+
+# 准备训练和验证数据集
+train_dataset = prepare_dataset(train_df, tokenizer)
+val_dataset = prepare_dataset(val_df, tokenizer)
+# 现在train_dataloader和validation_dataloader已准备好，可用于模型训练和验证
+train_dataloader = DataLoader(train_dataset, sampler=RandomSampler(train_dataset), batch_size=batch_size)
+validation_dataloader = DataLoader(val_dataset, sampler=SequentialSampler(val_dataset), batch_size=batch_size)
+
+# 加载配置
+config = BertConfig.from_pretrained("pretrained_models/Bio_ClinicalBERT-finetuned-medicalcondition")
+config.num_labels = 8  # 调整为你的标签数量
+
+model = AutoModelForSequenceClassification.from_pretrained(
+    "pretrained_models/Bio_ClinicalBERT-finetuned-medicalcondition", config=config, ignore_mismatched_sizes=True).to(
+    device)
+# 准备优化器和学习率调度器
+optimizer = AdamW(model.parameters(), lr=1e-5, eps=1e-8)
+total_steps = len(train_dataloader) * epochs  # epochs是您想要训练的轮数
+scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=0, num_training_steps=total_steps)
+loss_fn = BCEWithLogitsLoss()
+# 微调模型
+scaler = GradScaler()
+
+for epoch in range(epochs):  # 迭代多个epoch
+    print(f"\nEpoch {epoch + 1}/{epochs}")
+    print('-------------------------------')
+    model.train()
+    total_loss = 0
+    train_progress_bar = tqdm(train_dataloader, desc="Training", leave=False)
+    for step, batch in enumerate(train_progress_bar):
+        # 将数据加载到GPU
+        batch = tuple(t.to(device) for t in batch)
+        b_input_ids, b_input_mask, b_labels = batch
+        model.zero_grad()
+        # 前向传播
+        outputs = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask)
+        logits = outputs.logits
+        # 反向传播
+        loss = loss_fn(logits, b_labels)
+        total_loss += loss.item()
+        # loss.backward()
+        # optimizer.step()
+        # scheduler.step()
+        torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)
+        # 检查loss是否为nan
+        if torch.isnan(loss).any():
+            print(f"Loss is nan in epoch {epoch + 1}, step {step}.")
+            # 可选：打印出问题数据的更多信息或采取其他措施
+            # 注意：直接跳过可能不解决根本问题，最好检查为何loss会是nan
+            continue  # 跳过当前批次的反向传播和优化器步骤
+        scaler.scale(loss).backward()
+        scaler.step(optimizer)
+        scaler.update()
+        train_progress_bar.set_postfix({'loss': f"{loss.item():.2f}"})
+        # 添加训练损失到TensorBoard
+        writer.add_scalar('Loss/train', loss.item(), epoch * len(train_dataloader) + step)
+
+    # 评估阶段
+    avg_train_loss = total_loss / len(train_dataloader)
+    print(f"Training loss: {avg_train_loss:.2f}")
+
+    # 验证阶段
+    model.eval()
+    total_eval_accuracy = 0
+    eval_progress_bar = tqdm(validation_dataloader, desc="Validation", leave=False)
+    total_eval_loss = 0  # 初始化验证集总损失
+
+    for batch in eval_progress_bar:
+        batch = tuple(t.to(device) for t in batch)
+        b_input_ids, b_input_mask, b_labels = batch
+        with torch.no_grad():
+            outputs = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask)
+        logits = outputs.logits
+        # 注意：如果你的损失函数和模型的输出有不同的形状要求，可能需要对下面的损失计算进行调整
+        loss = loss_fn(logits, b_labels)
+        total_eval_loss += loss.item()  # 累加批次损失到总损失
+        # 使用sigmoid函数将logits转换为概率值
+        probs = torch.sigmoid(logits)
+        # 将概率高于0.5的预测为正类（1），低于0.5的预测为负类（0）
+        predictions = (probs > 0.5).int()
+        # 比较预测和真实标签
+        correct_predictions = (predictions == b_labels.int()).float()  # 确保标签也是整数类型
+        # 计算每个样本的正确预测的平均数，然后计算整个批次的平均值
+        accuracy_per_sample = correct_predictions.mean(dim=1)
+        accuracy = accuracy_per_sample.mean().item()
+        total_eval_accuracy += accuracy
+        # 更新进度条
+        eval_progress_bar.set_postfix({'accuracy': f"{accuracy:.2f}"})
+    # 计算整个验证集的平均损失
+    avg_val_loss = total_eval_loss / len(validation_dataloader)
+    print(f"Validation Loss: {avg_val_loss:.2f}")
+    avg_val_accuracy = total_eval_accuracy / len(validation_dataloader)
+    writer.add_scalar('Loss/val', avg_val_loss, epoch)  # 确保在TensorBoard中记录验证损失
+    print(f"Validation Accuracy: {avg_val_accuracy:.2f}")
+
+writer.close()
+# 保存模型
+torch.save(model.state_dict(), model_save_path)
+print(f"traing end, save model to :{model_save_path}")