qnloft
/
MediaCrawler
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'main' into feature/baidu_tieba_20240805

This commit is contained in:
Relakkes 2024-08-08 14:21:59 +08:00
parent 3f42368c02 7e9a759d94
commit 62ac454639
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
tools/words.py
View File

@ -27,7 +27,7 @@ class AsyncWordCloudGenerator:
async def generate_word_frequency_and_cloud(self, data, save_words_prefix): async def generate_word_frequency_and_cloud(self, data, save_words_prefix):
all_text = ' '.join(item['content'] for item in data) all_text = ' '.join(item['content'] for item in data)
words = [word for word in jieba.lcut(all_text) if word not in self.stop_words] words = [word for word in jieba.lcut(all_text) if word not in self.stop_words and len(word.strip()) > 0]
word_freq = Counter(words) word_freq = Counter(words)
# Save word frequency to file # Save word frequency to file