import re

def clean_text(text):
    # 去除HTML標(biāo)簽
    text = re.sub(r'<[^>]+>', '', text)
    # 去除特殊字符和數(shù)字
    text = re.sub(r'[^a-zA-Z\s]', '', text)
    # 轉(zhuǎn)換為小寫
    text = text.lower()
    # 去除多余空格
    text = ' '.join(text.split())
    return text

sample_text = "<p>This is a sample text! 123</p>"
print(clean_text(sample_text))  # 輸出: this is a sample text

# 英文分詞
import nltk
nltk.download('punkt')  # 第一次使用需要下載數(shù)據(jù)

from nltk.tokenize import word_tokenize
text = "Natural language processing is fascinating."
tokens = word_tokenize(text)
print(tokens)  # 輸出: ['Natural', 'language', 'processing', 'is', 'fascinating', '.']

# 中文分詞
import jieba
text_chinese = "自然語(yǔ)言處理非常有趣"
tokens_chinese = jieba.lcut(text_chinese)
print(tokens_chinese)  # 輸出: ['自然語(yǔ)言', '處理', '非常', '有趣']

from nltk.corpus import stopwords
nltk.download('stopwords')  # 第一次使用需要下載數(shù)據(jù)

stop_words = set(stopwords.words('english'))
filtered_tokens = [word for word in tokens if word.lower() not in stop_words]
print(filtered_tokens)  # 輸出: ['Natural', 'language', 'processing', 'fascinating', '.']

# 中文停用詞示例
stopwords_chinese = {"的", "是", "在", "非常"}
filtered_chinese = [word for word in tokens_chinese if word not in stopwords_chinese]
print(filtered_chinese)  # 輸出: ['自然語(yǔ)言', '處理', '有趣']

from collections import Counter
from wordcloud import WordCloud
import matplotlib.pyplot as plt

# 詞頻統(tǒng)計(jì)
word_counts = Counter(filtered_tokens)
print(word_counts.most_common(3))  # 輸出: [('Natural', 1), ('language', 1), ('processing', 1)]

# 生成詞云
text_for_wordcloud = " ".join(filtered_tokens)
wordcloud = WordCloud(width=800, height=400).generate(text_for_wordcloud)

plt.figure(figsize=(10, 5))
plt.imshow(wordcloud, interpolation='bilinear')
plt.axis("off")
plt.show()

from textblob import TextBlob
nltk.download('averaged_perceptron_tagger')  # 第一次使用需要下載數(shù)據(jù)

feedback = "I love this product. It's amazing!"
analysis = TextBlob(feedback)
print(f"情感極性: {analysis.sentiment.polarity}")  # 范圍從-1到1
print(f"主觀性: {analysis.sentiment.subjectivity}")  # 范圍從0到1

# 中文情感分析示例（需要先翻譯或使用中文專用庫(kù)）
chinese_feedback = "這個(gè)產(chǎn)品太糟糕了，我非常失望"
# 實(shí)際應(yīng)用中應(yīng)使用SnowNLP等中文庫(kù)

from sklearn.feature_extraction.text import TfidfVectorizer

documents = [
    "Python is a popular programming language",
    "Java is another programming language",
    "Python and Java are both object-oriented"
]

vectorizer = TfidfVectorizer()
X = vectorizer.fit_transform(documents)
print(vectorizer.get_feature_names_out())  # 輸出特征詞
print(X.shape)  # 文檔-詞矩陣形狀