实习管理平台与大模型训练的融合实践

import pandas as pd

from nltk.corpus import stopwords

from nltk.tokenize import word_tokenize

import string

# 读取实习日志数据

df = pd.read_csv('internship_logs.csv')

# 文本预处理函数

def preprocess_text(text):

# 去除标点符号

text = ''.join([char for char in text if char not in string.punctuation])

# 转换为小写

text = text.lower()

# 分词

tokens = word_tokenize(text)

# 去除停用词

stop_words = set(stopwords.words('english'))

tokens = [word for word in tokens if word not in stop_words]

return ' '.join(tokens)

# 应用预处理

df['processed_text'] = df['log'].apply(preprocess_text)

# 输出处理后的数据

print(df[['student_id', 'processed_text']].head())

from transformers import AutoTokenizer, AutoModelForSequenceClassification, TrainingArguments, Trainer

from sklearn.model_selection import train_test_split

# 加载预训练模型和分词器

model_name = "bert-base-uncased"

tokenizer = AutoTokenizer.from_pretrained(model_name)

model = AutoModelForSequenceClassification.from_pretrained(model_name, num_labels=2) # 假设是二分类任务

# 准备数据

texts = df['processed_text'].tolist()

labels = df['sentiment'].tolist() # 假设有一个情感标签列

# 划分训练集和测试集

train_texts, val_texts, train_labels, val_labels = train_test_split(texts, labels, test_size=0.2)

# 对数据进行编码

train_encodings = tokenizer(train_texts, truncation=True, padding=True, max_length=512)

val_encodings = tokenizer(val_texts, truncation=True, padding=True, max_length=512)

# 创建Dataset对象

class InternshipDataset:

def __init__(self, encodings, labels):

self.encodings = encodings

self.labels = labels

def __getitem__(self, idx):

item = {key: torch.tensor(self.encodings[key][idx]) for key in self.encodings}

item['labels'] = torch.tensor(self.labels[idx])

return item

def __len__(self):

return len(self.labels)

train_dataset = InternshipDataset(train_encodings, train_labels)

val_dataset = InternshipDataset(val_encodings, val_labels)

# 设置训练参数

training_args = TrainingArguments(

output_dir='./results',

num_train_epochs=3,

per_device_train_batch_size=16,

per_device_eval_batch_size=16,

warmup_steps=500,

weight_decay=0.01,

logging_dir='./logs',

logging_steps=10,

)

# 定义Trainer

trainer = Trainer(

model=model,

args=training_args,

train_dataset=train_dataset,

eval_dataset=val_dataset,

)

# 开始训练

trainer.train()

from fastapi import FastAPI

from pydantic import BaseModel

import torch

from transformers import pipeline

app = FastAPI()

# 加载模型

classifier = pipeline("text-classification", model="./results/checkpoint-1500")

class TextRequest(BaseModel):

text: str

@app.post("/predict")

def predict(request: TextRequest):

result = classifier(request.text)

return {"label": result[0]["label"], "score": result[0]["score"]}