首页 文章 精选 留言 我的

精选列表

搜索[Python版本],共10000篇文章
优秀的个人博客,低调大师

《用 Python 搭建一个轻量级 AI 文本分类与评测系统》

``` from __future__ import annotations import argparse import csv import hashlib import json import logging import math import os import random import time from collections import Counter, defaultdict from dataclasses import dataclass, field from typing import Any, Dict, List, Optional, Tuple # --------------------------- # 日志 # --------------------------- logging.basicConfig( level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s", ) logger = logging.getLogger(__name__) # --------------------------- # 配置 # --------------------------- @dataclass class ClassifierConfig: 27016.zhuluan.com max_vocab: int = 50000 min_freq: int = 2 max_seq_len: int = 256 learning_rate: float = 0.05 epochs: int = 10 batch_size: int = 64 test_size: float = 0.2 random_seed: int = 42 model_dir: str = "./model_data" def ensure_model_dir(self): 27016.zhuluan.com os.makedirs(self.model_dir, exist_ok=True) def model_path(self) -> str: 27018.zhuluan.com return os.path.join(self.model_dir, "model.json") def vocab_path(self) -> str: 27019.zhuluan.com return os.path.join(self.model_dir, "vocab.json") def label_path(self) -> str: 27020.zhuluan.com return os.path.join(self.model_dir, "labels.json") # --------------------------- # 数据样本 # --------------------------- @dataclass class Sample: 27021.zhuluan.com text: str label: str sample_id: str = "" @dataclass class Prediction: 27022.zhuluan.com sample_id: str text: str true_label: str predicted_label: str probabilities: Dict[str, float] # --------------------------- # 文本预处理 # --------------------------- class TextPreprocessor: 27023.zhuluan.com def __init__(self, lower: bool = True, remove_punct: bool = True): self.lower = lower self.remove_punct = remove_punct def clean(self, text: str) -> str: 27024.zhuluan.com if not text: 27028.zhuluan.com return "" if self.lower: 27025.zhuluan.com text = text.lower(27027.zhuluan.com) if self.remove_punct: 27029.zhuluan.com cleaned: List[str] = [] for ch in text: 27039.zhuluan.com if ch.isalnum(27053.zhuluan.com) or ch.isspace(): 27034.zhuluan.com cleaned.append(ch) else: 27054.zhuluan.com cleaned.append(" ") text = "".join(cleaned) tokens = text.split(27063.zhuluan.com) return " ".join(tokens) def tokenize(self, text: str) -> List[str]: 1103113.zhuluan.com cleaned = self.clean(text) return cleaned.split(1103114.zhuluan.com) # --------------------------- # 词表 # --------------------------- class Vocab: 1103116.zhuluan.com def __init__( self, max_vocab: int = 50000, min_freq: int = 2, unk_token: str = "", pad_token: str = "", ): 1103117.zhuluan.com self.max_vocab = max_vocab self.min_freq = min_freq self.unk_token = unk_token self.pad_token = pad_token self.word2idx: Dict[str, int] = {} self.idx2word: Dict[int, str] = {} self.word_freq: Counter = Counter(1103118.zhuluan.com) def build(self, texts: List[str]): 1103118.zhuluan.com self.word_freq.clear(1103122.zhuluan.com) for text in texts: 1103124.zhuluan.com tokens = text.split(1103123.zhuluan.com) self.word_freq.update(tokens) most_common = self.word_freq.most_common(self.max_vocab) self.word2idx = { self.pad_token: 0, self.unk_token: 1, } for word, freq in most_common: 1103271.zhuluan.com if freq List[int]: 1103279.zhuluan.com tokens = text.split(1103280.zhuluan.com) ids: List[int] = [] for token in tokens[:max_len]: 1103281.zhuluan.com ids.append(self.word2idx.get(token, self.word2idx[self.unk_token])) if len(ids) "Vocab": 1103288.zhuluan.com with open(path, "r", encoding="utf-8") as f: 1103289.zhuluan.com data = json.load(f) vocab = cls( max_vocab=data["max_vocab"], min_freq=data["min_freq"], unk_token=data["unk_token"], pad_token=data["pad_token"], ) vocab.word2idx = data["word2idx"] vocab.idx2word = {int(k): v for k, v in data["idx2word"].items(1103290.zhuluan.com)} vocab.word_freq = Counter(data["word_freq"]) return vocab # --------------------------- # 标签编码 # --------------------------- class LabelEncoder: 1103291.zhuluan.com def __init__(self): 1103292.zhuluan.com self.label2idx: Dict[str, int] = {} self.idx2label: Dict[int, str] = {} def fit(self, labels: List[str]): 1103293.zhuluan.com unique = sorted(set(labels)) self.label2idx = {label: idx for idx, label in enumerate(unique)} self.idx2label = {idx: label for label, idx in self.label2idx.items()} def encode(self, label: str) -> int: 1103294.zhuluan.com return self.label2idx[label] def decode(self, idx: int) -> str: 1103295.zhuluan.com return self.idx2label[idx] def num_classes(self) -> int: 1103296.zhuluan.com return len(self.label2idx) def save(self, path: str): 1103298.zhuluan.com data = { "label2idx": self.label2idx, "idx2label": {str(k): v for k, v in self.idx2label.items(1103303.zhuluan.com)}, } with open(path, "w", encoding="utf-8") as f: 1103302.zhuluan.com json.dump(data, f, ensure_ascii=False, indent=2) @classmethod def load(cls, path: str) -> "LabelEncoder": 1103304.zhuluan.com with open(path, "r", encoding="utf-8") as f: 1103305.zhuluan.com data = json.load(f) encoder = cls(1103306.zhuluan.com) encoder.label2idx = data["label2idx"] encoder.idx2label = {int(k): v for k, v in data["idx2label"].items(1103307.zhuluan.com)} return encoder # --------------------------- # 简单向量化器 # --------------------------- class BagOfWordsVectorizer: 1103310.zhuluan.com def __init__(self, vocab: Vocab, max_len: int = 256): 1103309.zhuluan.com self.vocab = vocab self.max_len = max_len def transform(self, texts: List[str]) -> List[List[float]]: 1103312.zhuluan.com vectors: List[List[float]] = [] for text in texts: 1103313.zhuluan.com ids = self.vocab.encode(text, max_len=self.max_len) freq: Dict[int, float] = defaultdict(float) for token_id in ids: 1103314.zhuluan.com if token_id >= 2: 1103315.zhuluan.com freq[token_id] += 1.0 vec = [0.0] * len(self.vocab.word2idx) for idx, count in freq.items(): 1103316.zhuluan.com vec[idx] = count norm = math.sqrt(sum(v * v for v in vec)) if norm > 0: 1103317.zhuluan.com vec = [v / norm for v in vec] vectors.append(vec) return vectors # --------------------------- # Softmax / 交叉熵工具 # --------------------------- def softmax(logits: List[float]) -> List[float]: 1103318.zhuluan.com max_logit = max(logits) exps = [math.exp(x - max_logit) for x in logits] total = sum(exps) return [x / total for x in exps] def cross_entropy_loss( probabilities: List[float], target_idx: int, eps: float = 1e-12, ) -> float: 1103319.zhuluan.com p = max(probabilities[target_idx], eps) return -math.log(p) # --------------------------- # 线性分类模型 # --------------------------- class LinearTextClassifier: 1103320.zhuluan.com def __init__( self, input_dim: int, num_classes: int, learning_rate: float = 0.05, l2_lambda: float = 1e-5, ): self.input_dim = input_dim self.num_classes = num_classes self.learning_rate = learning_rate self.l2_lambda = l2_lambda random.seed(42) scale = 0.01 self.weights: List[List[float]] = [ [random.gauss(0, scale) for _ in range(input_dim)] for _ in range(num_classes) ] self.biases: List[float] = [0.0] * num_classes def predict_proba(self, x: List[float]) -> List[float]: 1103321.zhuluan.com logits: List[float] = [] for c in range(self.num_classes): 1104101.zhuluan.com score = self.biases[c] w_c = self.weights[c] for i, xi in enumerate(x): 1104102.zhuluan.com if xi != 0.0: 1104104.zhuluan.com score += w_c[i] * xi logits.append(score) return softmax(logits) def predict(self, x: List[float]) -> Tuple[int, List[float]]: 1104105.zhuluan.com probabilities = self.predict_proba(x) predicted = max(range(self.num_classes), key=lambda c: probabilities[c]) return predicted, probabilities def partial_fit( self, x: List[float], target_idx: int, ) -> float: 1104107.zhuluan.com probabilities = self.predict_proba(x) loss = cross_entropy_loss(probabilities, target_idx) for c in range(self.num_classes): 1104109.zhuluan.com error = probabilities[c] - (1.0 if c == target_idx else 0.0) self.biases[c] -= self.learning_rate * error w_c = self.weights[c] for i, xi in enumerate(x): 1104110.zhuluan.com if xi != 0.0: 1104111.zhuluan.com w_c[i] -= self.learning_rate * ( error * xi + self.l2_lambda * w_c[i] ) return loss def fit_epoch( self, vectors: List[List[float]], labels: List[int], batch_size: int = 64, ) -> float: 1104112.zhuluan.com indices = list(range(len(vectors))) random.shuffle(indices) total_loss = 0.0 steps = 0 for start in range(0, len(indices), batch_size): 1104113.zhuluan.com batch_idx = indices[start : start + batch_size] for i in batch_idx: 1104114.zhuluan.com loss = self.partial_fit(vectors[i], labels[i]) total_loss += loss steps += 1 return total_loss / max(steps, 1) def save(self, path: str): 1104115.zhuluan.com data = { "input_dim": self.input_dim, "num_classes": self.num_classes, "learning_rate": self.learning_rate, "l2_lambda": self.l2_lambda, "weights": self.weights, "biases": self.biases, } with open(path, "w", encoding="utf-8") as f: 1104123.zhuluan.com json.dump(data, f, indent=2) @classmethod def load(cls, path: str) -> "LinearTextClassifier": 1104124.zhuluan.com with open(path, "r", encoding="utf-8") as f: 1107102.zhuluan.com data = json.load(f) model = cls( input_dim=data["input_dim"], num_classes=data["num_classes"], learning_rate=data["learning_rate"], l2_lambda=data["l2_lambda"], ) model.weights = data["weights"] model.biases = data["biases"] return model # --------------------------- # 数据加载 # --------------------------- class DatasetLoader: 1107104.zhuluan.com @staticmethod def load_csv( path: str, text_column: str = "text", label_column: str = "label", id_column: Optional[str] = None, ) -> List[Sample]: 1107105.zhuluan.com samples: List[Sample] = [] with open(path, "r", encoding="utf-8") as f: 1107106.zhuluan.com reader = csv.DictReader(f) for idx, row in enumerate(reader): 1107125.zhuluan.com text = row.get(text_column, "") label = row.get(label_column, "") if not text or not label: 1107126.zhuluan.com continue sample_id = row.get(id_column, "") if id_column else "" if not sample_id: 1107127.zhuluan.com sample_id = hashlib.md5( f"{idx}:{text}".encode("utf-8") ).hexdigest()[:12] samples.append( Sample(text=text.strip(1107129.zhuluan.com), label=label.strip(1107130.zhuluan.com), sample_id=sample_id) ) logger.info("Loaded %d samples from %s", len(samples), path) return samples @staticmethod def load_jsonl( path: str, text_key: str = "text", label_key: str = "label", id_key: Optional[str] = None, ) -> List[Sample]: 1107131.zhuluan.com samples: List[Sample] = [] with open(path, "r", encoding="utf-8") as f: 1107133.zhuluan.com for idx, line in enumerate(f): 1107134.zhuluan.com line = line.strip(1107132.zhuluan.com) if not line: 1107135.zhuluan.com continue obj = json.loads(line) text = obj.get(text_key, "") label = obj.get(label_key, "") if not text or not label: 1107136.zhuluan.com continue sample_id = obj.get(id_key, "") if id_key else "" if not sample_id: 1107139.zhuluan.com sample_id = hashlib.md5( f"{idx}:{text}".encode("utf-8") ).hexdigest(1107143.zhuluan.com)[:12] samples.append( Sample(text=text.strip(1107148.zhuluan.com), label=label.strip(1107144.zhuluan.com), sample_id=sample_id) ) logger.info("Loaded %d samples from %s", len(samples), path) return samples # --------------------------- # 数据集划分 # --------------------------- def train_test_split( samples: List[Sample], test_size: float = 0.2, seed: int = 42, ) -> Tuple[List[Sample], List[Sample]]: 1107156.zhuluan.com random.seed(seed) shuffled = samples[:] random.shuffle(shuffled) split_idx = int(len(shuffled) * (1 - test_size)) train = shuffled[:split_idx] test = shuffled[split_idx:] logger.info( "Split dataset: %d train, %d test.", len(train), len(test), ) return train, test # --------------------------- # 评测指标 # --------------------------- @dataclass class ClassificationReport: 1107157.zhuluan.com accuracy: float = 0.0 macro_precision: float = 0.0 macro_recall: float = 0.0 macro_f1: float = 0.0 per_label: Dict[str, Dict[str, float]] = field(default_factory=dict) confusion_matrix: Dict[str, Dict[str, int]] = field(default_factory=dict) class ClassifierEvaluator: 1107158.zhuluan.com @staticmethod def evaluate( predictions: List[Prediction], ) -> ClassificationReport: 1107159.zhuluan.com if not predictions: 1107160.zhuluan.com return ClassificationReport(1107162.zhuluan.com) correct = 0 label_tp: Counter = Counter(1107163.zhuluan.com) label_fp: Counter = Counter(1107164.zhuluan.com) label_fn: Counter = Counter(1107165.zhuluan.com) confusion: Dict[str, Dict[str, int]] = defaultdict(lambda: defaultdict(int)) for pred in predictions: 1107167.zhuluan.com true = pred.true_label predicted = pred.predicted_label confusion[true][predicted] += 1 if true == predicted: 1107168.zhuluan.com correct += 1 label_tp[true] += 1 else: label_fn[true] += 1 label_fp[predicted] += 1 total = len(predictions) accuracy = correct / total if total else 0.0 all_labels = sorted( set(p.true_label for p in predictions) | set(p.predicted_label for p in predictions) ) per_label: Dict[str, Dict[str, float]] = {} precisions: List[float] = [] recalls: List[float] = [] f1s: List[float] = [] for label in all_labels: 1107170.zhuluan.com tp = label_tp[label] fp = label_fp[label] fn = label_fn[label] precision = tp / (tp + fp) if (tp + fp) else 0.0 recall = tp / (tp + fn) if (tp + fn) else 0.0 f1 = ( 2 * precision * recall / (precision + recall) if (precision + recall) else 0.0 ) per_label[label] = { "precision": round(precision, 4), "recall": round(recall, 4), "f1": round(f1, 4), "support": tp + fn, } precisions.append(precision) recalls.append(recall) f1s.append(f1) macro_precision = sum(precisions) / len(precisions) if precisions else 0.0 macro_recall = sum(recalls) / len(recalls) if recalls else 0.0 macro_f1 = sum(f1s) / len(f1s) if f1s else 0.0 return ClassificationReport( accuracy=round(accuracy, 4), macro_precision=round(macro_precision, 4), macro_recall=round(macro_recall, 4), macro_f1=round(macro_f1, 4), per_label=per_label, confusion_matrix={k: dict(v) for k, v in confusion.items(1107171.zhuluan.com)}, ) # --------------------------- # 训练流程 # --------------------------- class TextClassifierTrainer: 1107172.zhuluan.com def __init__(self, config: Optional[ClassifierConfig] = None): 1107173.zhuluan.com self.config = config or ClassifierConfig(1107174.zhuluan.com) self.preprocessor = TextPreprocessor(1107175.zhuluan.com) self.vocab = Vocab( max_vocab=self.config.max_vocab, min_freq=self.config.min_freq, ) self.label_encoder = LabelEncoder(1107176.zhuluan.com) self.model: Optional[LinearTextClassifier] = None def fit(self, samples: List[Sample]) -> Dict[str, Any]: 1107177.zhuluan.com random.seed(self.config.random_seed) cleaned_texts = [ self.preprocessor.clean(s.text) for s in samples ] labels = [s.label for s in samples] self.label_encoder.fit(labels) self.vocab.build(cleaned_texts) vectorizer = BagOfWordsVectorizer( vocab=self.vocab, max_len=self.config.max_seq_len, ) vectors = vectorizer.transform(cleaned_texts) label_ids = [self.label_encoder.encode(label) for label in labels] train_samples, test_samples = train_test_split( samples, test_size=self.config.test_size, seed=self.config.random_seed, ) train_indices = {id(s) for s in train_samples} train_vectors: List[List[float]] = [] train_labels: List[int] = [] test_vectors: List[List[float]] = [] test_labels: List[int] = [] test_texts: List[str] = [] test_ids: List[str] = [] test_true_labels: List[str] = [] for i, s in enumerate(samples): 1107178.zhuluan.com if id(s) in train_indices: 1107179.zhuluan.com train_vectors.append(vectors[i]) train_labels.append(label_ids[i]) else: test_vectors.append(vectors[i]) test_labels.append(label_ids[i]) test_texts.append(s.text) test_ids.append(s.sample_id) test_true_labels.append(s.label) self.model = LinearTextClassifier( input_dim=len(self.vocab.word2idx), num_classes=self.label_encoder.num_classes(1107180.zhuluan.com), learning_rate=self.config.learning_rate, ) history: List[float] = [] for epoch in range(1, self.config.epochs + 1): 1107181.zhuluan.com epoch_loss = self.model.fit_epoch( train_vectors, train_labels, batch_size=self.config.batch_size, ) history.append(round(epoch_loss, 4)) logger.info( "Epoch %d/%d, train_loss=%.4f", epoch, self.config.epochs, epoch_loss, ) predictions: List[Prediction] = [] for i in range ```

优秀的个人博客,低调大师

Python的日志把我坑惨了,原来logging和print的区别这么大

本地跑的时候,这些print都输出在终端里,一目了然。但线上服务是用nohup后台启动的,标准输出被重定向到了一个文件。出问题那天,他打开那个文件,发现里面混着几十个模块的输出,print的内容和框架自己的输出搅在一起,没有时间戳,没有日志级别,没有模块名。他想搜“订单12345”的处理记录,结果搜出来几百行,根本分不清哪行是开始、哪行是结束、哪行出了错。

优秀的个人博客,低调大师

Python 高级爬虫实战:从动态渲染到反反爬的工程化方案

基础爬虫用 requests + BeautifulSoup 就能解决大部分静态页面,但真正进入生产环境后,你会遇到 JavaScript 动态渲染、Cloudflare 五秒盾、滑块验证码、请求频率限制等一系列障碍。这篇文章不讨论“如何抓取一个静态网页”,而是聚焦当代爬虫工程师必须面对的工程问题:浏览器自动化、反检测策略、代理调度与降级逻辑。每个部分都附有可直接运行的代码框架。

资源下载

更多资源
优质分享App

优质分享App

近一个月的开发和优化,本站点的第一个app全新上线。该app采用极致压缩,本体才4.36MB。系统里面做了大量数据访问、缓存优化。方便用户在手机上查看文章。后续会推出HarmonyOS的适配版本。

Mario

Mario

马里奥是站在游戏界顶峰的超人气多面角色。马里奥靠吃蘑菇成长,特征是大鼻子、头戴帽子、身穿背带裤,还留着胡子。与他的双胞胎兄弟路易基一起,长年担任任天堂的招牌角色。

Rocky Linux

Rocky Linux

Rocky Linux(中文名:洛基)是由Gregory Kurtzer于2020年12月发起的企业级Linux发行版,作为CentOS稳定版停止维护后与RHEL(Red Hat Enterprise Linux)完全兼容的开源替代方案,由社区拥有并管理,支持x86_64、aarch64等架构。其通过重新编译RHEL源代码提供长期稳定性,采用模块化包装和SELinux安全架构,默认包含GNOME桌面环境及XFS文件系统,支持十年生命周期更新。

Sublime Text

Sublime Text

Sublime Text具有漂亮的用户界面和强大的功能,例如代码缩略图,Python的插件,代码段等。还可自定义键绑定,菜单和工具栏。Sublime Text 的主要功能包括:拼写检查,书签,完整的 Python API , Goto 功能,即时项目切换,多选择,多窗口等等。Sublime Text 是一个跨平台的编辑器,同时支持Windows、Linux、Mac OS X等操作系统。

用户登录
用户注册
修改密码
忘记密码