qnloft
/
MediaCrawler
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

过滤空白字符

This commit is contained in:
AuYeung 2024-08-06 15:24:23 +08:00
parent 1c2237a66f
commit fc16ab7c5d
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
tools/words.py
View File

@ -27,7 +27,7 @@ class AsyncWordCloudGenerator:
async def generate_word_frequency_and_cloud(self, data, save_words_prefix): async def generate_word_frequency_and_cloud(self, data, save_words_prefix):
all_text = ' '.join(item['content'] for item in data) all_text = ' '.join(item['content'] for item in data)
words = [word for word in jieba.lcut(all_text) if word not in self.stop_words] words = [word for word in jieba.lcut(all_text) if word not in self.stop_words and len(word.strip()) > 0]
word_freq = Counter(words) word_freq = Counter(words)
# Save word frequency to file # Save word frequency to file