数据集

数据集使用的是开源英文文本60000条。详细内容见：
完整示例（包含数据集）

代码

训练：

import pandas as pd
from sklearn.feature_extraction.text import  TfidfVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.metrics import accuracy_score
import joblib


# 1. 数据加载和预处理
train_data = pd.read_csv('data/sentiment-train.csv')

X_train = train_data['text']
y_train = train_data['sentiment']


# 2. 特征提取
vectorizer = TfidfVectorizer()  # 使用TF-IDF特征提取
X_train = vectorizer.fit_transform(X_train)

# 3. 构建情感分类模型
clf = MultinomialNB()
clf.fit(X_train, y_train)


# 5. 保存模型
model_filename = "runs/model.pkl"
tfidf_vectorizer = "runs/tfidf_vectorizer.pkl"
joblib.dump(clf, model_filename)
joblib.dump(vectorizer, tfidf_vectorizer)
print("模型已保存为", model_filename)
print("tfidf向量已保存", tfidf_vectorizer)


# 6. 推理
new_text = ["a nice day"]
new_text_features = vectorizer.transform(new_text)

# print('new_text_features',  new_text_features)

# dense_matrix = vectorizer.transform(new_text).toarray()
# print('dense_matrix',dense_matrix)

# # file_name = "array.txt"
# # with open(file_name,'w') as file:
# #   for item in  dense_matrix:
# #     file.write(str(item))
# # print(f"数组已写入到文件 {file_name}")
# import numpy as np 
# np.savetxt("dense_matrix3.txt",dense_matrix,fmt='%f')
import json
predicted_sentiments = clf.predict(new_text_features)
print('predicted_sentiments', json.dumps({"value":predicted_sentiments.tolist()}))

评估：

import joblib
import pandas as pd
from sklearn.metrics import accuracy_score

def load_model():
    loaded_model = joblib.load('runs/model.pkl')
    vectorizer = joblib.load('runs/tfidf_vectorizer.pkl')
    return loaded_model,vectorizer

test_data = pd.read_csv('data/sentiment-test.csv')

X_test = test_data['text']
y_test = test_data['sentiment']

loaded_model ,vectorizer = load_model()

X_test = vectorizer.transform(X_test)

y_pred = loaded_model.predict(X_test)
accuracy = accuracy_score(y_test, y_pred)

print("准确度:", accuracy)

总结

以上为使用sklearn实现的文本分类模型，其中数据集下载后存在data目录下。原始文件