21丨朴素贝叶斯分类（下）：如何对文档进行分类？

#!/usr/bin/env python # -*- coding:utf8 -*- # __author__ = '北方姆Q' # __datetime__ = 2019/2/14 14:04 import os import jieba from sklearn.feature_extraction.text import TfidfVectorizer from sklearn.naive_bayes import MultinomialNB from sklearn import metrics LABEL_MAP = {'体育': 0, '女性': 1, '文学': 2, '校园': 3} # 加载停用词 with open('./text classification/stop/stopword.txt', 'rb') as f: STOP_WORDS = [line.strip() for line in f.readlines()] def load_data(base_path): """ :param base_path: 基础路径 :return: 分词列表，标签列表 """ documents = [] labels = [] for root, dirs, files in os.walk(base_path): # 循环所有文件并进行分词打标 for file in files: label = root.split('\\')[-1] # 因为windows上路径符号自动转成\了，所以要转义下 labels.append(label) filename = os.path.join(root, file) with open(filename, 'rb') as f: # 因为字符集问题因此直接用二进制方式读取 content = f.read() word_list = list(jieba.cut(content)) words = [wl for wl in word_list] documents.append(' '.join(words)) return documents, labels def train_fun(td, tl, testd, testl): """ 构造模型并计算测试集准确率，字数限制变量名简写 :param td: 训练集数据 :param tl: 训练集标签 :param testd: 测试集数据 :param testl: 测试集标签 :return: 测试集准确率 """ # 计算矩阵 tt = TfidfVectorizer(stop_words=STOP_WORDS, max_df=0.5) tf = tt.fit_transform(td) # 训练模型 clf = MultinomialNB(alpha=0.001).fit(tf, tl) # 模型预测 test_tf = TfidfVectorizer(stop_words=STOP_WORDS, max_df=0.5, vocabulary=tt.vocabulary_) test_features = test_tf.fit_transform(testd) predicted_labels = clf.predict(test_features) # 获取结果 x = metrics.accuracy_score(testl, predicted_labels) return x # text classification与代码同目录下 train_documents, train_labels = load_data('./text classification/train') test_documents, test_labels = load_data('./text classification/test') x = train_fun(train_documents, train_labels, test_documents, test_labels) print(x)

需要完整代码，不然看不明白！

老师，能不能在答疑的时候给这道题的完整代码看看

看过很多朴素贝叶斯原理和分类的讲解文章，很少能像前辈这样既有理论，又有实战的讲解，让大家既了解了理论知识，又有相应实际的操作经验可学，真的好棒，这个专栏，必须多多点赞，为老师加油！！！

https://github.com/yourSprite/AnalysisExcercise/tree/master/%E6%9C%B4%E7%B4%A0%E8%B4%9D%E5%8F%B6%E6%96%AF%E6%96%87%E6%9C%AC%E5%88%86%E7%B1%BB

老师，弱弱的说一句，代码感觉能看明白，但是不明白的是模型是如何使用的， 比如上一节和本节，都是只知道了准确率，但是对于有新的要处理的数据，如何做，怎么做好总是感觉差一点点东西。

#!/usr/bin/env python # coding: utf-8 import os import jieba import pandas as pd from sklearn.feature_extraction.text import TfidfVectorizer from sklearn.naive_bayes import MultinomialNB # 1. 加载数据 # 加载停用词表 l_stopWords = set() with open('./text_classification/text_classification/stop/stopword.txt', 'r') as l_f: for l_line in l_f: l_stopWords.add(l_line.strip()) l_labelMap = {'体育': 0, '女性': 1, '文学': 2, '校园': 3} # 加载训练数据和测试数据 def LoadData(filepath): l_documents = [] l_labels = [] for root, dirs, files in os.walk(filepath): for l_file in files: l_label = root.split('/')[-1] l_filename = os.path.join(root, l_file) with open(l_filename, 'r') as l_f: l_content = l_f.read() l_wordlist = list(jieba.cut(l_content)) l_words = [item for item in l_wordlist if item not in l_stopWords] l_documents.append(' '.join(l_words)) l_labels.append(l_labelMap[l_label]) return l_documents, l_labels l_trainDocuments, l_trainLabels = LoadData('./text_classification/text_classification/train') l_testDocuments, l_testLabels = LoadData('./text_classification/text_classification/test') # # 2. 计算权重矩阵 l_tfidfVec = TfidfVectorizer(max_df=0.5) l_tfidfMatrix = l_tfidfVec.fit_transform(l_trainDocuments) # for item in l_tfidfVec.get_feature_names(): # print item # print l_tfidfVec.get_feature_names() # print l_tfidfVec.vocabulary_ print l_tfidfMatrix.toarray().shape # # 3. 朴素贝叶斯模型 # ## 3.1 模型训练 l_clf = MultinomialNB(alpha=0.001) l_clf.fit(l_tfidfMatrix, l_trainLabels) # ## 3.2 模型预测 l_testTfidf = TfidfVectorizer(max_df=0.5, vocabulary=l_tfidfVec.vocabulary_) l_testFeature = l_testTfidf.fit_transform(l_testDocuments) l_hats = l_clf.predict(l_testFeature) # ## 3.3 模型评估 from sklearn.metrics import accuracy_score print accuracy_score(l_hats, l_testLabels)

老师，我想请教一下，计算单词权重时，为什么train_features用的fit_transform方法，而test_feature用的是transform

最后面的代码太乱，很多都不知道从哪里来的，无法顺着看下去~~~

# -*- coding:utf8 -*- # 系统：mac # 1. 加载数据 # 加载停用词表 l_stopWords = [line.strip() for line in open('./text_classification-master/text classification/stop/stopword.txt', 'r', encoding='utf-8').readlines()] l_labelMap = {'体育': 0, '女性': 1, '文学': 2, '校园': 3} # 加载训练数据和测试数据 def LoadData(filepath): l_documents = [] l_labels = [] for root, dirs, files in os.walk(filepath): for l_file in files: if l_file=='.DS_Store': continue l_label = root.split('/')[-1] l_filename = os.path.join(root, l_file) with open(l_filename, 'r',encoding='gbk') as l_f: try: l_content = l_f.read() except Exception as err: print(err) print(l_filename) continue generator = jieba.cut(l_content) words = ' '.join(generator) l_wordlist=words.split(' ') l_words = [item for item in l_wordlist if item not in l_stopWords] l_documents.append(' '.join(l_words)) l_labels.append(l_labelMap[l_label]) return l_documents, l_labels l_trainDocuments, l_trainLabels = LoadData('./text_classification-master/text classification/train') l_testDocuments, l_testLabels = LoadData('./text_classification-master/text classification/test') # # 2. 计算权重矩阵 l_tfidfVec = TfidfVectorizer(max_df=0.5) l_tfidfMatrix = l_tfidfVec.fit_transform(l_trainDocuments) print (l_tfidfMatrix.toarray().shape) # # 3. 朴素贝叶斯模型 # ## 3.1 模型训练 l_clf = MultinomialNB(alpha=0.001) l_clf.fit(l_tfidfMatrix, l_trainLabels) # ## 3.2 模型预测 l_testTfidf = TfidfVectorizer(max_df=0.5, vocabulary=l_tfidfVec.vocabulary_) l_testFeature = l_testTfidf.fit_transform(l_testDocuments) l_hats = l_clf.predict(l_testFeature) # ## 3.3 模型评估 from sklearn.metrics import accuracy_score print (accuracy_score(l_hats, l_testLabels))