Python でのテキスト分類の例

import os import pandas as pd # 读取文件 def read_data(folder): files = os.listdir(folder) data = {'text': [], 'sentiment': []} for file in files: with open(os.path.join(folder, file), 'r') as f: data['text'].append(f.read()) data['sentiment'].append(file.split('.')[0]) return pd.DataFrame.from_dict(data) # 读取数据集 train_folder = 'aclImdb/train' test_folder = 'aclImdb/test' train_data = read_data(train_folder) test_data = read_data(test_folder)

# 统计文本长度 train_data['text_len'] = train_data['text'].apply(len) test_data['text_len'] = test_data['text'].apply(len) # 统计情感标签比例 train_sentiment_pct = train_data.groupby('sentiment').size() / len(train_data) test_sentiment_pct = test_data.groupby('sentiment').size() / len(test_data) print('Train Sentiment Distribution: {} '.format(train_sentiment_pct)) print('Test Sentiment Distribution: {} '.format(test_sentiment_pct))

from sklearn.feature_extraction.text import CountVectorizer # 创建CountVectorizer对象 vectorizer = CountVectorizer(stop_words='english') # 将文本转换为向量 train_features = vectorizer.fit_transform(train_data['text']) test_features = vectorizer.transform(test_data['text']) # 打印特征维度 print('Train Feature Dimension: {}'.format(train_features.shape)) print('Test Feature Dimension: {}'.format(test_features.shape))

from sklearn.linear_model import LogisticRegression from sklearn.naive_bayes import MultinomialNB from sklearn.svm import SVC from sklearn.ensemble import RandomForestClassifier from sklearn.metrics import f1_score, accuracy_score # 训练和评估函数 def train_and_evalute(classifier, train_features, train_labels, test_features, test_labels): # 训练分类器 classifier.fit(train_features, train_labels) # 在训练集和测试集上计算F1分数和准确率 train_predictions = classifier.predict(train_features) test_predictions = classifier.predict(test_features) train_f1 = f1_score(train_labels, train_predictions, pos_label='pos') test_f1 = f1_score(test_labels, test_predictions, pos_label='pos') train_accuracy = accuracy_score(train_labels, train_predictions) test_accuracy = accuracy_score(test_labels, test_predictions) # 打印评估结果 print('Train F1 Score: {0:.3f}'.format(train_f1)) print('Test F1 Score: {0:.3f}'.format(test_f1)) print('Train Accuracy: {0:.3f}'.format(train_accuracy)) print('Test Accuracy: {0:.3f}'.format(test_accuracy)) # 训练和评估各个分类器 classifiers = [ ('Logistic Regression', LogisticRegression(max_iter=1000)), ('Multinomial Naive Bayes', MultinomialNB()), ('Support Vector Machine', SVC(kernel='linear')), ('Random Forest', RandomForestClassifier(n_estimators=100)) ] for classifier_name, classifier in classifiers: print(' {}'.format(classifier_name)) train_and_evalute(classifier, train_features, train_data['sentiment'], test_features, test_data['sentiment'])