python基于百度,哈工大等停用表进行的中文分词

import os
import pandas as pd
import jieba# 加载停用词
def load_stopwords(filenames):stopwords = set()for filename in filenames:with open(filename, 'r', encoding='utf-8') as f:for line in f:stopwords.add(line.strip())return stopwords# 中文分词并去除停用词
def segment_and_remove_stopwords(text, stopwords):words = jieba.cut(text)filtered_words = [word for word in words if word not in stopwords and len(word) > 1]return ' '.join(filtered_words)# 处理评论数据
def process_comments(df, comment_column, stopwords):df['connected_words'] = df[comment_column].apply(lambda x: segment_and_remove_stopwords(x, stopwords))return df# 主函数
def main(input_file_path, output_file_path, comment_column, stopwords_files=[]):# 加载停用词stopwords = load_stopwords(stopwords_files)# 读取CSV文件df = pd.read_csv(input_file_path, encoding='utf-8')# 处理评论数据processed_df = process_comments(df, comment_column, stopwords)# 保存处理后的数据到新的CSV文件processed_df.to_csv(output_file_path, index=False, encoding='utf-8-sig')print(f"数据预处理完成，已保存到 {output_file_path}")if __name__ == '__main__':input_file_path = r"D:\pycharm\爬虫案列\24.汽车之家\_0_10.csv"  # 你的CSV文件路径output_file_path = 'comments_processed.csv'  # 输出文件的路径comment_column = '空间'  # 假设评论数据在'comment'列中# 停用词文件列表，确保这些文件在你的工作目录中stopwords_files = [r"stopwords-master\baidu_stopwords.txt",r"stopwords-master\cn_stopwords.txt",r"stopwords-master\hit_stopwords.txt",r"stopwords-master\scu_stopwords.txt",# ... 其他停用词文件]# 确保所有停用词文件都存在for filename in stopwords_files:if not os.path.exists(filename):print(f"Stopwords file {filename} not found.")exit(1)# 调用主函数处理评论数据main(input_file_path, output_file_path, comment_column, stopwords_files)

停用词表可以去看一下博主的上传的资源 , 可以免费获取的