《用 Python 搭建一个轻量级 AI 文本分类与评测系统》
``` from __future__ import annotations import argparse import csv import hashlib import json import logging import math import os import random import time from collections import Counter, defaultdict from dataclasses import dataclass, field from typing import Any, Dict, List, Optional, Tuple # --------------------------- # 日志 # --------------------------- logging.basicConfig( level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s", ) logger = logging.getLogger(__name__) # --------------------------- # 配置 # --------------------------- @dataclass class ClassifierConfig: 27016.zhuluan.com max_vocab: int = 50000 min_freq: int = 2 max_seq_len: int = 256 learning_rate: float = 0.05 epochs: int = 10 batch_size: int = 64 test_size: float = 0.2 random_seed: int = 42 model_dir: str = "./model_data" def ensure_model_dir(self): 27016.zhuluan.com os.makedirs(self.model_dir, exist_ok=True) def model_path(self) -> str: 27018.zhuluan.com return os.path.join(self.model_dir, "model.json") def vocab_path(self) -> str: 27019.zhuluan.com return os.path.join(self.model_dir, "vocab.json") def label_path(self) -> str: 27020.zhuluan.com return os.path.join(self.model_dir, "labels.json") # --------------------------- # 数据样本 # --------------------------- @dataclass class Sample: 27021.zhuluan.com text: str label: str sample_id: str = "" @dataclass class Prediction: 27022.zhuluan.com sample_id: str text: str true_label: str predicted_label: str probabilities: Dict[str, float] # --------------------------- # 文本预处理 # --------------------------- class TextPreprocessor: 27023.zhuluan.com def __init__(self, lower: bool = True, remove_punct: bool = True): self.lower = lower self.remove_punct = remove_punct def clean(self, text: str) -> str: 27024.zhuluan.com if not text: 27028.zhuluan.com return "" if self.lower: 27025.zhuluan.com text = text.lower(27027.zhuluan.com) if self.remove_punct: 27029.zhuluan.com cleaned: List[str] = [] for ch in text: 27039.zhuluan.com if ch.isalnum(27053.zhuluan.com) or ch.isspace(): 27034.zhuluan.com cleaned.append(ch) else: 27054.zhuluan.com cleaned.append(" ") text = "".join(cleaned) tokens = text.split(27063.zhuluan.com) return " ".join(tokens) def tokenize(self, text: str) -> List[str]: 1103113.zhuluan.com cleaned = self.clean(text) return cleaned.split(1103114.zhuluan.com) # --------------------------- # 词表 # --------------------------- class Vocab: 1103116.zhuluan.com def __init__( self, max_vocab: int = 50000, min_freq: int = 2, unk_token: str = "", pad_token: str = "", ): 1103117.zhuluan.com self.max_vocab = max_vocab self.min_freq = min_freq self.unk_token = unk_token self.pad_token = pad_token self.word2idx: Dict[str, int] = {} self.idx2word: Dict[int, str] = {} self.word_freq: Counter = Counter(1103118.zhuluan.com) def build(self, texts: List[str]): 1103118.zhuluan.com self.word_freq.clear(1103122.zhuluan.com) for text in texts: 1103124.zhuluan.com tokens = text.split(1103123.zhuluan.com) self.word_freq.update(tokens) most_common = self.word_freq.most_common(self.max_vocab) self.word2idx = { self.pad_token: 0, self.unk_token: 1, } for word, freq in most_common: 1103271.zhuluan.com if freq List[int]: 1103279.zhuluan.com tokens = text.split(1103280.zhuluan.com) ids: List[int] = [] for token in tokens[:max_len]: 1103281.zhuluan.com ids.append(self.word2idx.get(token, self.word2idx[self.unk_token])) if len(ids) "Vocab": 1103288.zhuluan.com with open(path, "r", encoding="utf-8") as f: 1103289.zhuluan.com data = json.load(f) vocab = cls( max_vocab=data["max_vocab"], min_freq=data["min_freq"], unk_token=data["unk_token"], pad_token=data["pad_token"], ) vocab.word2idx = data["word2idx"] vocab.idx2word = {int(k): v for k, v in data["idx2word"].items(1103290.zhuluan.com)} vocab.word_freq = Counter(data["word_freq"]) return vocab # --------------------------- # 标签编码 # --------------------------- class LabelEncoder: 1103291.zhuluan.com def __init__(self): 1103292.zhuluan.com self.label2idx: Dict[str, int] = {} self.idx2label: Dict[int, str] = {} def fit(self, labels: List[str]): 1103293.zhuluan.com unique = sorted(set(labels)) self.label2idx = {label: idx for idx, label in enumerate(unique)} self.idx2label = {idx: label for label, idx in self.label2idx.items()} def encode(self, label: str) -> int: 1103294.zhuluan.com return self.label2idx[label] def decode(self, idx: int) -> str: 1103295.zhuluan.com return self.idx2label[idx] def num_classes(self) -> int: 1103296.zhuluan.com return len(self.label2idx) def save(self, path: str): 1103298.zhuluan.com data = { "label2idx": self.label2idx, "idx2label": {str(k): v for k, v in self.idx2label.items(1103303.zhuluan.com)}, } with open(path, "w", encoding="utf-8") as f: 1103302.zhuluan.com json.dump(data, f, ensure_ascii=False, indent=2) @classmethod def load(cls, path: str) -> "LabelEncoder": 1103304.zhuluan.com with open(path, "r", encoding="utf-8") as f: 1103305.zhuluan.com data = json.load(f) encoder = cls(1103306.zhuluan.com) encoder.label2idx = data["label2idx"] encoder.idx2label = {int(k): v for k, v in data["idx2label"].items(1103307.zhuluan.com)} return encoder # --------------------------- # 简单向量化器 # --------------------------- class BagOfWordsVectorizer: 1103310.zhuluan.com def __init__(self, vocab: Vocab, max_len: int = 256): 1103309.zhuluan.com self.vocab = vocab self.max_len = max_len def transform(self, texts: List[str]) -> List[List[float]]: 1103312.zhuluan.com vectors: List[List[float]] = [] for text in texts: 1103313.zhuluan.com ids = self.vocab.encode(text, max_len=self.max_len) freq: Dict[int, float] = defaultdict(float) for token_id in ids: 1103314.zhuluan.com if token_id >= 2: 1103315.zhuluan.com freq[token_id] += 1.0 vec = [0.0] * len(self.vocab.word2idx) for idx, count in freq.items(): 1103316.zhuluan.com vec[idx] = count norm = math.sqrt(sum(v * v for v in vec)) if norm > 0: 1103317.zhuluan.com vec = [v / norm for v in vec] vectors.append(vec) return vectors # --------------------------- # Softmax / 交叉熵工具 # --------------------------- def softmax(logits: List[float]) -> List[float]: 1103318.zhuluan.com max_logit = max(logits) exps = [math.exp(x - max_logit) for x in logits] total = sum(exps) return [x / total for x in exps] def cross_entropy_loss( probabilities: List[float], target_idx: int, eps: float = 1e-12, ) -> float: 1103319.zhuluan.com p = max(probabilities[target_idx], eps) return -math.log(p) # --------------------------- # 线性分类模型 # --------------------------- class LinearTextClassifier: 1103320.zhuluan.com def __init__( self, input_dim: int, num_classes: int, learning_rate: float = 0.05, l2_lambda: float = 1e-5, ): self.input_dim = input_dim self.num_classes = num_classes self.learning_rate = learning_rate self.l2_lambda = l2_lambda random.seed(42) scale = 0.01 self.weights: List[List[float]] = [ [random.gauss(0, scale) for _ in range(input_dim)] for _ in range(num_classes) ] self.biases: List[float] = [0.0] * num_classes def predict_proba(self, x: List[float]) -> List[float]: 1103321.zhuluan.com logits: List[float] = [] for c in range(self.num_classes): 1104101.zhuluan.com score = self.biases[c] w_c = self.weights[c] for i, xi in enumerate(x): 1104102.zhuluan.com if xi != 0.0: 1104104.zhuluan.com score += w_c[i] * xi logits.append(score) return softmax(logits) def predict(self, x: List[float]) -> Tuple[int, List[float]]: 1104105.zhuluan.com probabilities = self.predict_proba(x) predicted = max(range(self.num_classes), key=lambda c: probabilities[c]) return predicted, probabilities def partial_fit( self, x: List[float], target_idx: int, ) -> float: 1104107.zhuluan.com probabilities = self.predict_proba(x) loss = cross_entropy_loss(probabilities, target_idx) for c in range(self.num_classes): 1104109.zhuluan.com error = probabilities[c] - (1.0 if c == target_idx else 0.0) self.biases[c] -= self.learning_rate * error w_c = self.weights[c] for i, xi in enumerate(x): 1104110.zhuluan.com if xi != 0.0: 1104111.zhuluan.com w_c[i] -= self.learning_rate * ( error * xi + self.l2_lambda * w_c[i] ) return loss def fit_epoch( self, vectors: List[List[float]], labels: List[int], batch_size: int = 64, ) -> float: 1104112.zhuluan.com indices = list(range(len(vectors))) random.shuffle(indices) total_loss = 0.0 steps = 0 for start in range(0, len(indices), batch_size): 1104113.zhuluan.com batch_idx = indices[start : start + batch_size] for i in batch_idx: 1104114.zhuluan.com loss = self.partial_fit(vectors[i], labels[i]) total_loss += loss steps += 1 return total_loss / max(steps, 1) def save(self, path: str): 1104115.zhuluan.com data = { "input_dim": self.input_dim, "num_classes": self.num_classes, "learning_rate": self.learning_rate, "l2_lambda": self.l2_lambda, "weights": self.weights, "biases": self.biases, } with open(path, "w", encoding="utf-8") as f: 1104123.zhuluan.com json.dump(data, f, indent=2) @classmethod def load(cls, path: str) -> "LinearTextClassifier": 1104124.zhuluan.com with open(path, "r", encoding="utf-8") as f: 1107102.zhuluan.com data = json.load(f) model = cls( input_dim=data["input_dim"], num_classes=data["num_classes"], learning_rate=data["learning_rate"], l2_lambda=data["l2_lambda"], ) model.weights = data["weights"] model.biases = data["biases"] return model # --------------------------- # 数据加载 # --------------------------- class DatasetLoader: 1107104.zhuluan.com @staticmethod def load_csv( path: str, text_column: str = "text", label_column: str = "label", id_column: Optional[str] = None, ) -> List[Sample]: 1107105.zhuluan.com samples: List[Sample] = [] with open(path, "r", encoding="utf-8") as f: 1107106.zhuluan.com reader = csv.DictReader(f) for idx, row in enumerate(reader): 1107125.zhuluan.com text = row.get(text_column, "") label = row.get(label_column, "") if not text or not label: 1107126.zhuluan.com continue sample_id = row.get(id_column, "") if id_column else "" if not sample_id: 1107127.zhuluan.com sample_id = hashlib.md5( f"{idx}:{text}".encode("utf-8") ).hexdigest()[:12] samples.append( Sample(text=text.strip(1107129.zhuluan.com), label=label.strip(1107130.zhuluan.com), sample_id=sample_id) ) logger.info("Loaded %d samples from %s", len(samples), path) return samples @staticmethod def load_jsonl( path: str, text_key: str = "text", label_key: str = "label", id_key: Optional[str] = None, ) -> List[Sample]: 1107131.zhuluan.com samples: List[Sample] = [] with open(path, "r", encoding="utf-8") as f: 1107133.zhuluan.com for idx, line in enumerate(f): 1107134.zhuluan.com line = line.strip(1107132.zhuluan.com) if not line: 1107135.zhuluan.com continue obj = json.loads(line) text = obj.get(text_key, "") label = obj.get(label_key, "") if not text or not label: 1107136.zhuluan.com continue sample_id = obj.get(id_key, "") if id_key else "" if not sample_id: 1107139.zhuluan.com sample_id = hashlib.md5( f"{idx}:{text}".encode("utf-8") ).hexdigest(1107143.zhuluan.com)[:12] samples.append( Sample(text=text.strip(1107148.zhuluan.com), label=label.strip(1107144.zhuluan.com), sample_id=sample_id) ) logger.info("Loaded %d samples from %s", len(samples), path) return samples # --------------------------- # 数据集划分 # --------------------------- def train_test_split( samples: List[Sample], test_size: float = 0.2, seed: int = 42, ) -> Tuple[List[Sample], List[Sample]]: 1107156.zhuluan.com random.seed(seed) shuffled = samples[:] random.shuffle(shuffled) split_idx = int(len(shuffled) * (1 - test_size)) train = shuffled[:split_idx] test = shuffled[split_idx:] logger.info( "Split dataset: %d train, %d test.", len(train), len(test), ) return train, test # --------------------------- # 评测指标 # --------------------------- @dataclass class ClassificationReport: 1107157.zhuluan.com accuracy: float = 0.0 macro_precision: float = 0.0 macro_recall: float = 0.0 macro_f1: float = 0.0 per_label: Dict[str, Dict[str, float]] = field(default_factory=dict) confusion_matrix: Dict[str, Dict[str, int]] = field(default_factory=dict) class ClassifierEvaluator: 1107158.zhuluan.com @staticmethod def evaluate( predictions: List[Prediction], ) -> ClassificationReport: 1107159.zhuluan.com if not predictions: 1107160.zhuluan.com return ClassificationReport(1107162.zhuluan.com) correct = 0 label_tp: Counter = Counter(1107163.zhuluan.com) label_fp: Counter = Counter(1107164.zhuluan.com) label_fn: Counter = Counter(1107165.zhuluan.com) confusion: Dict[str, Dict[str, int]] = defaultdict(lambda: defaultdict(int)) for pred in predictions: 1107167.zhuluan.com true = pred.true_label predicted = pred.predicted_label confusion[true][predicted] += 1 if true == predicted: 1107168.zhuluan.com correct += 1 label_tp[true] += 1 else: label_fn[true] += 1 label_fp[predicted] += 1 total = len(predictions) accuracy = correct / total if total else 0.0 all_labels = sorted( set(p.true_label for p in predictions) | set(p.predicted_label for p in predictions) ) per_label: Dict[str, Dict[str, float]] = {} precisions: List[float] = [] recalls: List[float] = [] f1s: List[float] = [] for label in all_labels: 1107170.zhuluan.com tp = label_tp[label] fp = label_fp[label] fn = label_fn[label] precision = tp / (tp + fp) if (tp + fp) else 0.0 recall = tp / (tp + fn) if (tp + fn) else 0.0 f1 = ( 2 * precision * recall / (precision + recall) if (precision + recall) else 0.0 ) per_label[label] = { "precision": round(precision, 4), "recall": round(recall, 4), "f1": round(f1, 4), "support": tp + fn, } precisions.append(precision) recalls.append(recall) f1s.append(f1) macro_precision = sum(precisions) / len(precisions) if precisions else 0.0 macro_recall = sum(recalls) / len(recalls) if recalls else 0.0 macro_f1 = sum(f1s) / len(f1s) if f1s else 0.0 return ClassificationReport( accuracy=round(accuracy, 4), macro_precision=round(macro_precision, 4), macro_recall=round(macro_recall, 4), macro_f1=round(macro_f1, 4), per_label=per_label, confusion_matrix={k: dict(v) for k, v in confusion.items(1107171.zhuluan.com)}, ) # --------------------------- # 训练流程 # --------------------------- class TextClassifierTrainer: 1107172.zhuluan.com def __init__(self, config: Optional[ClassifierConfig] = None): 1107173.zhuluan.com self.config = config or ClassifierConfig(1107174.zhuluan.com) self.preprocessor = TextPreprocessor(1107175.zhuluan.com) self.vocab = Vocab( max_vocab=self.config.max_vocab, min_freq=self.config.min_freq, ) self.label_encoder = LabelEncoder(1107176.zhuluan.com) self.model: Optional[LinearTextClassifier] = None def fit(self, samples: List[Sample]) -> Dict[str, Any]: 1107177.zhuluan.com random.seed(self.config.random_seed) cleaned_texts = [ self.preprocessor.clean(s.text) for s in samples ] labels = [s.label for s in samples] self.label_encoder.fit(labels) self.vocab.build(cleaned_texts) vectorizer = BagOfWordsVectorizer( vocab=self.vocab, max_len=self.config.max_seq_len, ) vectors = vectorizer.transform(cleaned_texts) label_ids = [self.label_encoder.encode(label) for label in labels] train_samples, test_samples = train_test_split( samples, test_size=self.config.test_size, seed=self.config.random_seed, ) train_indices = {id(s) for s in train_samples} train_vectors: List[List[float]] = [] train_labels: List[int] = [] test_vectors: List[List[float]] = [] test_labels: List[int] = [] test_texts: List[str] = [] test_ids: List[str] = [] test_true_labels: List[str] = [] for i, s in enumerate(samples): 1107178.zhuluan.com if id(s) in train_indices: 1107179.zhuluan.com train_vectors.append(vectors[i]) train_labels.append(label_ids[i]) else: test_vectors.append(vectors[i]) test_labels.append(label_ids[i]) test_texts.append(s.text) test_ids.append(s.sample_id) test_true_labels.append(s.label) self.model = LinearTextClassifier( input_dim=len(self.vocab.word2idx), num_classes=self.label_encoder.num_classes(1107180.zhuluan.com), learning_rate=self.config.learning_rate, ) history: List[float] = [] for epoch in range(1, self.config.epochs + 1): 1107181.zhuluan.com epoch_loss = self.model.fit_epoch( train_vectors, train_labels, batch_size=self.config.batch_size, ) history.append(round(epoch_loss, 4)) logger.info( "Epoch %d/%d, train_loss=%.4f", epoch, self.config.epochs, epoch_loss, ) predictions: List[Prediction] = [] for i in range ```