pip install nltk
# 下載NLTK數(shù)據(jù)包（首次使用時需運(yùn)行）
import nltk
nltk.download('punkt')      # 分詞模型
nltk.download('averaged_perceptron_tagger')  # 詞性標(biāo)注模型
nltk.download('wordnet')    # 詞匯數(shù)據(jù)庫
nltk.download('stopwords')  # 停用詞

from nltk.tokenize import sent_tokenize
text = "Hello world! This is NLTK. Let's learn NLP."
sentences = sent_tokenize(text)  # ['Hello world!', 'This is NLTK.', "Let's learn NLP."]

from nltk.tokenize import word_tokenize
words = word_tokenize("Hello, world!")  # ['Hello', ',', 'world', '!']

from nltk import pos_tag
tokens = word_tokenize("I love NLP.")
tags = pos_tag(tokens)  # [('I', 'PRP'), ('love', 'VBP'), ('NLP', 'NNP'), ('.', '.')]

from nltk.stem import PorterStemmer
stemmer = PorterStemmer()
stemmed = stemmer.stem("running")  # 'run'

from nltk.stem import WordNetLemmatizer
lemmatizer = WordNetLemmatizer()
lemma = lemmatizer.lemmatize("better", pos='a')  # 'good'（需指定詞性）

from nltk import RegexpParser
grammar = r"NP: {<DT>?<JJ>*<NN>}"  # 定義名詞短語規(guī)則
parser = RegexpParser(grammar)
tree = parser.parse(tags)  # 生成語法樹
tree.draw()  # 可視化樹結(jié)構(gòu)

from nltk import ne_chunk
text = "Apple is headquartered in Cupertino."
tags = pos_tag(word_tokenize(text))
entities = ne_chunk(tags)
# 輸出： (GPE Apple/NNP) is/VBZ headquartered/VBN in/IN (GPE Cupertino/NNP)

from nltk.corpus import stopwords
stop_words = set(stopwords.words('english'))
filtered_words = [w for w in word_tokenize(text) if w.lower() not in stop_words]

from nltk import edit_distance
distance = edit_distance("apple", "appel")  # 2

from nltk.sentiment import SentimentIntensityAnalyzer
sia = SentimentIntensityAnalyzer()
score = sia.polarity_scores("I love this movie!")  # {'compound': 0.8316, 'pos': 0.624, ...}

from nltk.corpus import gutenberg
print(gutenberg.fileids())  # 查看內(nèi)置語料庫
emma = gutenberg.words('austen-emma.txt')  # 加載文本

from nltk.text import TextCollection
corpus = TextCollection([text1, text2, text3])
tfidf = corpus.tf_idf(word, text)

from nltk.util import ngrams
bigrams = list(ngrams(tokens, 2))  # 生成二元組

# 示例：使用 jieba 分詞
import jieba
words = jieba.lcut("自然語言處理很有趣")  # ['自然語言', '處理', '很', '有趣']

from nltk.corpus import movie_reviews
import random
# 加載數(shù)據(jù)（正面和負(fù)面評論）
documents = [(list(movie_reviews.words(fileid)), category)
             for category in movie_reviews.categories()
             for fileid in movie_reviews.fileids(category)]
random.shuffle(documents)  # 打亂順序
# 提取所有單詞并構(gòu)建特征集
all_words = [word.lower() for word in movie_reviews.words()]
all_words = nltk.FreqDist(all_words)
word_features = list(all_words.keys())[:3000]  # 選擇前3000個高頻詞作為特征
# 定義特征提取函數(shù)
def document_features(document):
    document_words = set(document)
    features = {}
    for word in word_features:
        features[f'contains({word})'] = (word in document_words)
    return features
featuresets = [(document_features(doc), category) for (doc, category) in documents]
train_set, test_set = featuresets[100:], featuresets[:100]  # 劃分訓(xùn)練集和測試集

classifier = nltk.NaiveBayesClassifier.train(train_set)
# 評估模型
accuracy = nltk.classify.accuracy(classifier, test_set)
print(f"Accuracy: {accuracy:.2f}")  # 輸出約 0.7-0.8
# 查看重要特征
classifier.show_most_informative_features(10)
# 示例輸出：
# Most Informative Features
#     contains(outstanding) = True              pos : neg    =     12.4 : 1.0
#       contains(seagal) = True              neg : pos    =     10.6 : 1.0

from nltk.corpus import PlaintextCorpusReader
corpus_root = './my_corpus'  # 本地文件夾路徑
file_pattern = r'.*\.txt'    # 匹配所有txt文件
my_corpus = PlaintextCorpusReader(corpus_root, file_pattern)
# 訪問語料庫內(nèi)容
print(my_corpus.fileids())          # 查看文件列表
print(my_corpus.words('doc1.txt'))  # 獲取特定文檔的單詞

from nltk.probability import FreqDist
import matplotlib.pyplot as plt
custom_text = nltk.Text(my_corpus.words())
fdist = FreqDist(custom_text)
# 繪制高頻詞分布
plt.figure(figsize=(12,5))
fdist.plot(30, cumulative=False)
plt.show()
# 查找特定詞的上下文
custom_text.concordance("人工智能", width=100, lines=10)

from functools import lru_cache
from nltk.stem import WordNetLemmatizer
lemmatizer = WordNetLemmatizer()
@lru_cache(maxsize=10000)  # 緩存最近10000次調(diào)用
def cached_lemmatize(word, pos='n'):
    return lemmatizer.lemmatize(word, pos)
# 使用緩存版本處理大規(guī)模文本
lemmas = [cached_lemmatize(word) for word in huge_word_list]

from joblib import Parallel, delayed
from nltk.tokenize import word_tokenize
# 并行分詞加速
texts = [...]  # 大規(guī)模文本列表
results = Parallel(n_jobs=4)(delayed(word_tokenize)(text) for text in texts)

from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
from gensim import models, corpora
# 預(yù)處理
stop_words = stopwords.words('english')
lemmatizer = WordNetLemmatizer()
processed_docs = [
    [lemmatizer.lemmatize(word) for word in doc.lower().split() 
     if word not in stop_words and word.isalpha()]
    for doc in text_corpus
]
# 創(chuàng)建詞典和文檔-詞矩陣
dictionary = corpora.Dictionary(processed_docs)
doc_term_matrix = [dictionary.doc2bow(doc) for doc in processed_docs]
# 訓(xùn)練LDA模型
lda_model = models.LdaModel(
    doc_term_matrix,
    num_topics=5,
    id2word=dictionary,
    passes=10
)
# 查看主題
print(lda_model.print_topics())

import networkx as nx
from nltk import bigrams
# 生成共現(xiàn)網(wǎng)絡(luò)
cooc_network = nx.Graph()
for doc in documents:
    doc_bigrams = list(bigrams(doc))
    for (w1, w2) in doc_bigrams:
        if cooc_network.has_edge(w1, w2):
            cooc_network[w1][w2]['weight'] += 1
        else:
            cooc_network.add_edge(w1, w2, weight=1)
# 可視化重要連接
plt.figure(figsize=(15,10))
pos = nx.spring_layout(cooc_network)
nx.draw_networkx_nodes(cooc_network, pos, node_size=50)
nx.draw_networkx_edges(cooc_network, pos, alpha=0.2)
nx.draw_networkx_labels(cooc_network, pos, font_size=8)
plt.show()

# 指定下載鏡像源
import nltk
nltk.download('punkt', download_dir='/path/to/nltk_data', 
             quiet=True, halt_on_error=False)

# 使用生成器處理大文件
def stream_docs(path):
    with open(path, 'r', encoding='utf-8') as f:
        for line in f:
            yield line.strip()
# 分塊處理
for chunk in nltk.chunk(stream_docs('big_file.txt'), 10000):
    process(chunk)

from nltk import data
data.path.append('/path/to/unicode/corpora')  # 添加自定義編碼語料路徑

import pandas as pd
from nltk.sentiment import SentimentIntensityAnalyzer
df = pd.read_csv('reviews.csv')
sia = SentimentIntensityAnalyzer()
# 為每條評論添加情感分?jǐn)?shù)
df['sentiment'] = df['text'].apply(
    lambda x: sia.polarity_scores(x)['compound']
)
# 分析結(jié)果分布
df['sentiment'].hist(bins=20)

from sklearn.pipeline import Pipeline
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.naive_bayes import MultinomialNB
from nltk.tokenize import TreebankWordTokenizer
# 自定義分詞器
nltk_tokenizer = TreebankWordTokenizer().tokenize
pipeline = Pipeline([
    ('vect', CountVectorizer(tokenizer=nltk_tokenizer)),
    ('tfidf', TfidfTransformer()),
    ('clf', MultinomialNB()),
])
pipeline.fit(X_train, y_train)

from nltk.sentiment.util import mark_negation
from nltk import FreqDist
# 自定義情感詞典
positive_words = {'excellent', 'brilliant', 'superb'}
negative_words = {'terrible', 'awful', 'horrible'}
def custom_sentiment_analyzer(text):
    tokens = mark_negation(word_tokenize(text.lower()))  # 處理否定詞
    score = 0
    for word in tokens:
        if word in positive_words:
            score += 1
        elif word in negative_words:
            score -= 1
        elif word.endswith("_NEG"):  # 處理否定修飾
            base_word = word[:-4]
            if base_word in positive_words:
                score -= 1
            elif base_word in negative_words:
                score += 1
    return score
# 測試示例
text = "The service was not excellent but the food was superb."
print(custom_sentiment_analyzer(text))  # 輸出：0 (因?yàn)?excellent_NEG"扣分，但"superb"加分)

from sklearn.svm import SVC
from nltk.classify.scikitlearn import SklearnClassifier
from nltk.sentiment import SentimentAnalyzer
# 使用scikit-learn的SVM算法
sentiment_analyzer = SentimentAnalyzer()
svm_classifier = SklearnClassifier(SVC(kernel='linear'))
# 添加自定義特征
all_words = [word.lower() for word in movie_reviews.words()]
unigram_feats = sentiment_analyzer.unigram_word_feats(all_words, min_freq=10)
sentiment_analyzer.add_feat_extractor(
    nltk.sentiment.util.extract_unigram_feats, unigrams=unigram_feats[:2000]
)
# 轉(zhuǎn)換特征格式
training_set = sentiment_analyzer.apply_features(movie_reviews.sents(categories='pos')[:500] + \
               sentiment_analyzer.apply_features(movie_reviews.sents(categories='neg')[:500]
# 訓(xùn)練并評估模型
svm_classifier.train(training_set)
accuracy = nltk.classify.accuracy(svm_classifier, training_set)
print(f"SVM分類器準(zhǔn)確率: {accuracy:.2%}")

import pandas as pd
from nltk.sentiment import SentimentIntensityAnalyzer
# 加載帶時間戳的新聞數(shù)據(jù)
news_data = [
    ("2023-01-01", "Company A launched revolutionary new product"),
    ("2023-02-15", "Company A faces regulatory investigation"),
    ("2023-03-30", "Company A reports record profits")
]
df = pd.DataFrame(news_data, columns=['date', 'text'])
df['date'] = pd.to_datetime(df['date'])
# 計算每日情感分?jǐn)?shù)
sia = SentimentIntensityAnalyzer()
df['sentiment'] = df['text'].apply(lambda x: sia.polarity_scores(x)['compound'])
# 可視化趨勢
df.set_index('date')['sentiment'].plot(
    title='公司A新聞情感趨勢分析',
    ylabel='情感分?jǐn)?shù)',
    figsize=(10,6),
    grid=True
)

from nltk.tokenize import RegexpTokenizer
# 自定義多語言分詞器
multilingual_tokenizer = RegexpTokenizer(r'''\w+@\w+\.\w+ |  # 保留電子郵件
                                           [A-Za-z]+(?:'\w+)? |  # 英文單詞
                                           [\u4e00-\u9fff]+ |  # 中文字符
                                           \d+''')  # 數(shù)字
text = "Hello 你好！Contact me at example@email.com 或撥打400-123456"
tokens = multilingual_tokenizer.tokenize(text)
# 輸出：['Hello', '你好', 'Contact', 'me', 'at', 'example@email.com', '或', '撥打', '400', '123456']

from gensim.models import KeyedVectors
from nltk.corpus import wordnet as wn
# 加載預(yù)訓(xùn)練跨語言詞向量（需提前下載）
# 示例使用Facebook的MUSE詞向量
zh_model = KeyedVectors.load_word2vec_format('wiki.multi.zh.vec')
en_model = KeyedVectors.load_word2vec_format('wiki.multi.en.vec')
def cross_lingual_similarity(word_en, word_zh):
    try:
        return en_model.similarity(word_en, zh_model[word_zh])
    except KeyError:
        return None
print(f"Apple 與 蘋果 的相似度: {cross_lingual_similarity('apple', '蘋果'):.2f}")
# 輸出：約0.65-0.75

from nltk.metrics import ConfusionMatrix, precision, recall, f_measure
ref_set = ['pos', 'neg', 'pos', 'pos']
test_set = ['pos', 'pos', 'neg', 'pos']
# 創(chuàng)建混淆矩陣
cm = ConfusionMatrix(ref_set, test_set)
print(cm)
# 計算指標(biāo)
print(f"Precision: {precision(set(ref_set), set(test_set)):.2f}")
print(f"Recall: {recall(set(ref_set), set(test_set)):.2f}")
print(f"F1-Score: {f_measure(set(ref_set), set(test_set)):.2f}")

from nltk.translate.bleu_score import sentence_bleu
reference = [['this', 'is', 'a', 'test']]
candidate = ['this', 'is', 'a', 'test']
print(f"BLEU-4 Score: {sentence_bleu(reference, candidate):.2f}")
# 輸出：1.0
candidate = ['this', 'is', 'test']
print(f"BLEU-4 Score: {sentence_bleu(reference, candidate):.2f}")
# 輸出：約0.59

from tweepy import Stream
from nltk import FreqDist
import json
class TweetAnalyzer(Stream):
    def __init__(self, consumer_key, consumer_secret):
        super().__init__(consumer_key, consumer_secret)
        self.keywords_fd = FreqDist()
    def on_data(self, data):
        tweet = json.loads(data)
        text = tweet.get('text', '')
        tokens = [word.lower() for word in word_tokenize(text) 
                 if word.isalpha() and len(word) > 2]
        for word in tokens:
            self.keywords_fd[word] += 1
        return True
# 使用示例（需申請Twitter API密鑰）
analyzer = TweetAnalyzer('YOUR_KEY', 'YOUR_SECRET')
analyzer.filter(track=['python', 'AI'], languages=['en'])

from dash import Dash, dcc, html
import plotly.express as px
from collections import deque
# 創(chuàng)建實(shí)時更新隊(duì)列
sentiment_history = deque(maxlen=100)
timestamps = deque(maxlen=100)
app = Dash(__name__)
app.layout = html.Div([
    dcc.Graph(id='live-graph'),
    dcc.Interval(id='interval', interval=5000)
])
@app.callback(Output('live-graph', 'figure'),
              Input('interval', 'n_intervals'))
def update_graph(n):
    # 這里添加實(shí)時獲取數(shù)據(jù)的邏輯
    return px.line(x=list(timestamps), 
                  y=list(sentiment_history),
                  title="實(shí)時情感趨勢")
if __name__ == '__main__':
    app.run_server(debug=True)

from nltk.tag import UnigramTagger
from nltk.corpus import treebank
# 訓(xùn)練自定義標(biāo)注器
train_sents = treebank.tagged_sents()[:3000]
tagger = UnigramTagger(train_sents)
# 查看內(nèi)部概率分布
word = 'run'
prob_dist = tagger._model[word]
print(f"{word} 的標(biāo)注概率分布:")
for tag, prob in prob_dist.items():
    print(f"{tag}: {prob:.2%}")
# 輸出示例：
# VB: 45.32%
# NN: 32.15%
# ...其他詞性概率

from nltk.parse import RecursiveDescentParser
from nltk.grammar import CFG
# 定義簡單語法
grammar = CFG.fromstring("""
    S -> NP VP
    VP -> V NP | V NP PP
    PP -> P NP
    NP -> Det N | Det N PP
    Det -> 'a' | 'the'
    N -> 'man' | 'park' | 'dog'
    V -> 'saw' | 'walked'
    P -> 'in' | 'with'
""")
# 創(chuàng)建解析器
parser = RecursiveDescentParser(grammar)
sentence = "the man saw a dog in the park".split()
for tree in parser.parse(sentence):
    tree.pretty_print()

from IPython.display import display
import ipywidgets as widgets
# 創(chuàng)建交互式詞性標(biāo)注器
text_input = widgets.Textarea(value='Enter text here')
output = widgets.Output()
def tag_text(b):
    with output:
        output.clear_output()
        text = text_input.value
        tokens = word_tokenize(text)
        tags = pos_tag(tokens)
        print("標(biāo)注結(jié)果:")
        for word, tag in tags:
            print(f"{word:15}{tag}")
button = widgets.Button(description="標(biāo)注文本")
button.on_click(tag_text)
display(widgets.VBox([text_input, button, output]))

from nltk import ngrams
from nltk.corpus import brown
# 構(gòu)建語言模型
brown_ngrams = list(ngrams(brown.words(), 3))
freq_dist = FreqDist(brown_ngrams)
def detect_errors(sentence):
    tokens = word_tokenize(sentence)
    trigrams = list(ngrams(tokens, 3))
    for i, trigram in enumerate(trigrams):
        if freq_dist[trigram] < 5:  # 出現(xiàn)頻率過低的組合
            print(f"潛在錯誤位置 {i+1}-{i+3}: {' '.join(trigram)}")
detect_errors("He don't knows the answer.")
# 輸出：潛在錯誤位置 2-4: don't knows the

from transformers import pipeline
from nltk import word_tokenize
# 結(jié)合HuggingFace模型
class AdvancedNLTKAnalyzer:
    def __init__(self):
        self.sentiment = pipeline('sentiment-analysis')
        self.ner = pipeline('ner')
    def enhanced_analysis(self, text):
        return {
            'sentiment': self.sentiment(text),
            'entities': self.ner(text),
            'tokens': word_tokenize(text)
        }
# 使用示例
analyzer = AdvancedNLTKAnalyzer()
result = analyzer.enhanced_analysis("Apple Inc. is looking to buy U.K. startup for $1 billion")
print(result['entities'])  # 識別組織、地點(diǎn)、貨幣等實(shí)體

from numba import jit
from nltk import edit_distance
# 使用GPU加速編輯距離計算
@jit(nopython=True, parallel=True)
def gpu_edit_distance(s1, s2):
    # 實(shí)現(xiàn)動態(tài)規(guī)劃算法
    m, n = len(s1), len(s2)
    dp = [[0]*(n+1) for _ in range(m+1)]
    for i in range(m+1): dp[i][0] = i
    for j in range(n+1): dp[0][j] = j
    for i in range(1, m+1):
        for j in range(1, n+1):
            cost = 0 if s1[i-1] == s2[j-1] else 1
            dp[i][j] = min(dp[i-1][j]+1, 
                          dp[i][j-1]+1, 
                          dp[i-1][j-1]+cost)
    return dp[m][n]
print(gpu_edit_distance("kitten", "sitting"))  # 輸出：3