📖 第3章：文本表示方法¶

good_representation = {
    "语义相关性": "语义相近的文本在向量空间中距离近",
    "维度合理": "既不过高（维度灾难）也不过低（信息丢失）",
    "可计算性": "支持加减乘除等向量运算",
    "泛化能力": "能处理未见过的词语或表达",
    "上下文感知": "同一个词在不同语境下有不同表示",
}

import numpy as np

class OneHotEncoder:
    """One-Hot编码器"""

    def __init__(self):
        self.word2idx = {}
        self.idx2word = {}
        self.vocab_size = 0

    def fit(self, corpus):
        """构建词汇表"""
        vocab = set()
        for doc in corpus:
            words = doc.split() if isinstance(doc, str) else doc  # isinstance检查类型
            vocab.update(words)

        self.word2idx = {word: idx for idx, word in enumerate(sorted(vocab))}  # enumerate同时获取索引和元素
        self.idx2word = {idx: word for word, idx in self.word2idx.items()}
        self.vocab_size = len(vocab)

    def encode_word(self, word):
        """单个词编码"""
        vec = np.zeros(self.vocab_size)
        if word in self.word2idx:
            vec[self.word2idx[word]] = 1
        return vec

    def encode_document(self, words):
        """文档编码（方法：求和/平均）"""
        if isinstance(words, str):
            words = words.split()
        return sum(self.encode_word(w) for w in words)

# 示例
corpus = ["我 喜欢 自然语言处理", "自然语言处理 很 有趣", "深度学习 推动 自然语言处理"]
encoder = OneHotEncoder()
encoder.fit(corpus)

print(f"词汇表大小: {encoder.vocab_size}")
print(f"词汇表: {encoder.word2idx}")
print()

# 编码单个词
for word in ["自然语言处理", "喜欢", "深度学习"]:
    vec = encoder.encode_word(word)
    print(f"'{word}': {vec}")

from collections import Counter

class BagOfWords:
    """词袋模型"""

    def __init__(self):
        self.vocabulary = {}
        self.vocab_size = 0

    def fit(self, corpus):
        vocab = set()
        for doc in corpus:
            words = doc if isinstance(doc, list) else doc.split()
            vocab.update(words)
        self.vocabulary = {word: idx for idx, word in enumerate(sorted(vocab))}
        self.vocab_size = len(self.vocabulary)

    def transform(self, document):
        """将文档转为词频向量"""
        words = document if isinstance(document, list) else document.split()
        vec = np.zeros(self.vocab_size)
        word_counts = Counter(words)  # Counter统计元素出现次数
        for word, count in word_counts.items():
            if word in self.vocabulary:
                vec[self.vocabulary[word]] = count
        return vec

    def fit_transform(self, corpus):
        self.fit(corpus)
        return np.array([self.transform(doc) for doc in corpus])  # np.array创建NumPy数组

# 示例
corpus = [
    "我 喜欢 机器学习 和 自然语言处理",
    "自然语言处理 是 机器学习 的 一个 分支",
    "深度学习 推动 了 自然语言处理 的 发展",
]

bow = BagOfWords()
vectors = bow.fit_transform(corpus)

print(f"词汇表: {bow.vocabulary}")
print(f"\nBoW矩阵 (shape={vectors.shape}):")
for i, (doc, vec) in enumerate(zip(corpus, vectors)):  # zip按位置配对
    print(f"  文档{i}: {vec.astype(int)}")

# 词袋模型丢失了词序信息
doc1 = "猫 追 狗"
doc2 = "狗 追 猫"

# 在BoW中，这两个文档的表示完全相同！
bow_demo = BagOfWords()
bow_demo.fit([doc1, doc2])
v1 = bow_demo.transform(doc1)
v2 = bow_demo.transform(doc2)
print(f"'{doc1}' → {v1}")
print(f"'{doc2}' → {v2}")
print(f"两个向量是否相同: {np.array_equal(v1, v2)}")
# True! 但这两句话意思完全不同

import math
from collections import Counter
import numpy as np

class TFIDF:
    """TF-IDF实现"""

    def __init__(self):
        self.vocabulary = {}
        self.idf = {}
        self.vocab_size = 0

    def fit(self, corpus):
        """
        corpus: List[List[str]] - 分词后的文档列表
        """
        # 建立词汇表
        vocab = set()
        for doc in corpus:
            vocab.update(doc)
        self.vocabulary = {word: idx for idx, word in enumerate(sorted(vocab))}
        self.vocab_size = len(self.vocabulary)

        # 计算IDF
        N = len(corpus)
        doc_freq = Counter()
        for doc in corpus:
            unique_words = set(doc)
            for word in unique_words:
                doc_freq[word] += 1

        self.idf = {}
        for word in self.vocabulary:
            self.idf[word] = math.log(N / (1 + doc_freq.get(word, 0)))

    def transform(self, document):
        """将文档转为TF-IDF向量"""
        vec = np.zeros(self.vocab_size)
        word_counts = Counter(document)
        total_words = len(document)

        for word, count in word_counts.items():
            if word in self.vocabulary:
                tf = count / total_words
                idf = self.idf.get(word, 0)
                vec[self.vocabulary[word]] = tf * idf

        return vec

    def fit_transform(self, corpus):
        self.fit(corpus)
        return np.array([self.transform(doc) for doc in corpus])

    def get_top_keywords(self, document, top_k=10):
        """获取文档的Top-K关键词"""
        vec = self.transform(document)
        idx2word = {idx: word for word, idx in self.vocabulary.items()}

        top_indices = vec.argsort()[::-1][:top_k]
        keywords = [(idx2word[idx], vec[idx]) for idx in top_indices if vec[idx] > 0]
        return keywords

# 示例
corpus = [
    ["自然语言处理", "是", "人工智能", "的", "重要", "分支"],
    ["机器学习", "和", "深度学习", "推动", "了", "人工智能", "的", "发展"],
    ["自然语言处理", "利用", "深度学习", "技术", "处理", "文本", "数据"],
    ["计算机视觉", "是", "人工智能", "的", "另一个", "重要", "方向"],
    ["自然语言处理", "包括", "文本分类", "命名实体识别", "机器翻译", "等", "任务"],
]

tfidf = TFIDF()
vectors = tfidf.fit_transform(corpus)

print("TF-IDF关键词提取结果：")
for i, doc in enumerate(corpus):
    keywords = tfidf.get_top_keywords(doc, top_k=5)
    doc_str = "".join(doc)[:20]  # 切片操作，取前n个元素
    kw_str = ", ".join([f"{w}({s:.3f})" for w, s in keywords])
    print(f"  文档{i} '{doc_str}...': {kw_str}")

# 计算文档相似度
print("\n文档相似度矩阵：")
for i in range(len(corpus)):
    for j in range(len(corpus)):
        sim = np.dot(vectors[i], vectors[j]) / (np.linalg.norm(vectors[i]) * np.linalg.norm(vectors[j]) + 1e-8)  # np.dot矩阵/向量点乘  # np.linalg线性代数运算
        print(f"{sim:.3f}", end=" ")
    print()

from sklearn.feature_extraction.text import TfidfVectorizer
import numpy as np

# 中文需要先分词，用空格连接
corpus = [
    "自然语言处理 是 人工智能 的 重要 分支",
    "机器学习 和 深度学习 推动 了 人工智能 的 发展",
    "自然语言处理 利用 深度学习 技术 处理 文本 数据",
    "计算机视觉 是 人工智能 的 另一个 重要 方向",
]

# 使用sklearn的TF-IDF
vectorizer = TfidfVectorizer()
tfidf_matrix = vectorizer.fit_transform(corpus)

print(f"特征名称: {vectorizer.get_feature_names_out()}")
print(f"TF-IDF矩阵形状: {tfidf_matrix.shape}")
print(f"\nTF-IDF稠密矩阵:")
print(np.round(tfidf_matrix.toarray(), 3))

# 文档相似度
from sklearn.metrics.pairwise import cosine_similarity
similarity = cosine_similarity(tfidf_matrix)
print(f"\n余弦相似度矩阵:")
print(np.round(similarity, 3))

from collections import Counter, defaultdict

class NGramModel:
    """N-gram语言模型"""

    def __init__(self, n=2):
        self.n = n
        self.ngram_counts = Counter()
        self.context_counts = Counter()
        self.vocabulary = set()

    def train(self, corpus):
        """训练N-gram模型"""
        for sentence in corpus:
            # 添加开始和结束标记
            tokens = ['<s>'] * (self.n - 1) + sentence + ['</s>']
            self.vocabulary.update(sentence)

            for i in range(len(tokens) - self.n + 1):
                ngram = tuple(tokens[i:i + self.n])
                context = tuple(tokens[i:i + self.n - 1])
                self.ngram_counts[ngram] += 1
                self.context_counts[context] += 1

    def probability(self, word, context):
        """计算条件概率 P(word|context)"""
        context = tuple(context)
        ngram = context + (word,)

        context_count = self.context_counts.get(context, 0)
        ngram_count = self.ngram_counts.get(ngram, 0)

        if context_count == 0:
            return 1 / (len(self.vocabulary) + 1)  # 均匀分布

        # 加一平滑（Laplace smoothing）
        return (ngram_count + 1) / (context_count + len(self.vocabulary) + 1)

    def generate(self, start_context=None, max_length=20):
        """基于N-gram模型生成文本"""
        import random

        if start_context is None:
            context = ['<s>'] * (self.n - 1)
        else:
            context = list(start_context)

        result = list(start_context) if start_context else []

        for _ in range(max_length):
            ctx = tuple(context[-(self.n-1):])

            # 获取所有可能的下一个词及其概率
            candidates = {}
            for ngram, count in self.ngram_counts.items():
                if ngram[:-1] == ctx:
                    candidates[ngram[-1]] = count  # [-1]负索引取最后元素

            if not candidates or '</s>' in candidates and len(candidates) == 1:
                break

            # 按频率采样
            words = list(candidates.keys())
            weights = [candidates[w] for w in words]
            total = sum(weights)
            weights = [w/total for w in weights]

            next_word = random.choices(words, weights=weights)[0]

            if next_word == '</s>':
                break

            result.append(next_word)
            context.append(next_word)

        return result

    def perplexity(self, sentence):
        """计算句子的困惑度"""
        tokens = ['<s>'] * (self.n - 1) + sentence + ['</s>']
        log_prob = 0
        N = len(sentence) + 1  # +1 for </s>

        for i in range(self.n - 1, len(tokens)):
            context = tokens[i - self.n + 1:i]
            word = tokens[i]
            prob = self.probability(word, context)
            log_prob += math.log2(prob + 1e-10)

        return 2 ** (-log_prob / N)

# 训练N-gram模型
corpus = [
    ["自然", "语言", "处理", "是", "人工智能", "的", "重要", "分支"],
    ["深度", "学习", "推动", "了", "自然", "语言", "处理", "的", "发展"],
    ["机器", "学习", "是", "人工智能", "的", "基础"],
    ["自然", "语言", "处理", "技术", "应用", "广泛"],
    ["人工智能", "包括", "自然", "语言", "处理", "和", "计算机视觉"],
]

import math

# Bigram模型
bigram = NGramModel(n=2)
bigram.train(corpus)

print("Bigram统计（部分）:")
for ngram, count in sorted(bigram.ngram_counts.items(), key=lambda x: -x[1])[:10]:  # lambda匿名函数
    print(f"  {ngram}: {count}")

# 概率查询
print(f"\nP(处理|语言) = {bigram.probability('处理', ['语言']):.4f}")
print(f"P(智能|人工) = {bigram.probability('智能', ['人工']):.4f}")

# 困惑度
test_sent = ["自然", "语言", "处理", "技术"]
ppl = bigram.perplexity(test_sent)
print(f"\n句子困惑度: {''.join(test_sent)} → PPL = {ppl:.2f}")

# 文本生成
print("\n生成文本:")
for _ in range(3):
    generated = bigram.generate(start_context=["自然"])
    print(f"  {''.join(generated)}")

import numpy as np
from collections import Counter
import random

class SimpleWord2Vec:
    """Word2Vec Skip-gram with Negative Sampling 简化实现"""

    def __init__(self, embedding_dim=50, window_size=2, num_negative=5, learning_rate=0.025):
        self.embedding_dim = embedding_dim
        self.window_size = window_size
        self.num_negative = num_negative
        self.lr = learning_rate
        self.word2idx = {}
        self.idx2word = {}
        self.vocab_size = 0
        self.W = None  # 输入矩阵（词向量）
        self.W_prime = None  # 输出矩阵

    def build_vocab(self, corpus, min_count=1):
        """构建词汇表"""
        word_counts = Counter()
        for sentence in corpus:
            word_counts.update(sentence)

        # 过滤低频词
        vocab = [w for w, c in word_counts.items() if c >= min_count]
        self.word2idx = {w: i for i, w in enumerate(vocab)}
        self.idx2word = {i: w for w, i in self.word2idx.items()}
        self.vocab_size = len(vocab)

        # 计算采样概率（用于负采样）
        total = sum(word_counts[w] for w in vocab)
        self.sample_probs = np.array([
            (word_counts[self.idx2word[i]] / total) ** 0.75
            for i in range(self.vocab_size)
        ])
        self.sample_probs /= self.sample_probs.sum()

        # 初始化权重矩阵
        self.W = np.random.uniform(-0.5/self.embedding_dim, 0.5/self.embedding_dim,
                                    (self.vocab_size, self.embedding_dim))
        self.W_prime = np.zeros((self.vocab_size, self.embedding_dim))

    def sigmoid(self, x):
        """稳定的sigmoid计算"""
        return np.where(x >= 0,
                       1 / (1 + np.exp(-x)),
                       np.exp(x) / (1 + np.exp(x)))

    def generate_training_data(self, corpus):
        """生成训练数据（中心词, 上下文词）对"""
        pairs = []
        for sentence in corpus:
            indices = [self.word2idx[w] for w in sentence if w in self.word2idx]
            for i, center in enumerate(indices):
                # 动态窗口大小
                actual_window = random.randint(1, self.window_size)
                context_range = range(max(0, i - actual_window),
                                     min(len(indices), i + actual_window + 1))
                for j in context_range:
                    if i != j:
                        pairs.append((center, indices[j]))
        return pairs

    def train_pair(self, center_idx, context_idx):
        """训练一个(中心词, 上下文词)对"""
        # 正样本
        center_vec = self.W[center_idx]
        context_vec = self.W_prime[context_idx]

        score = np.dot(center_vec, context_vec)
        pred = self.sigmoid(score)

        # 正样本梯度
        grad = (pred - 1) * self.lr
        center_grad = grad * context_vec
        self.W_prime[context_idx] -= grad * center_vec

        # 负采样
        neg_indices = np.random.choice(self.vocab_size, self.num_negative,
                                        p=self.sample_probs)
        for neg_idx in neg_indices:
            if neg_idx == context_idx:
                continue
            neg_vec = self.W_prime[neg_idx]
            score = np.dot(center_vec, neg_vec)
            pred = self.sigmoid(score)

            grad = pred * self.lr
            center_grad += grad * neg_vec
            self.W_prime[neg_idx] -= grad * center_vec

        # 更新中心词向量
        self.W[center_idx] -= center_grad

    def train(self, corpus, epochs=5):
        """训练模型"""
        self.build_vocab(corpus)
        print(f"词汇表大小: {self.vocab_size}")

        for epoch in range(epochs):
            pairs = self.generate_training_data(corpus)
            random.shuffle(pairs)

            total_loss = 0
            for center, context in pairs:
                self.train_pair(center, context)

            if (epoch + 1) % 1 == 0:
                print(f"Epoch {epoch+1}/{epochs} completed. Pairs: {len(pairs)}")

    def get_vector(self, word):
        """获取词向量"""
        if word in self.word2idx:
            return self.W[self.word2idx[word]]
        return None

    def most_similar(self, word, top_k=5):
        """查找最相似的词"""
        if word not in self.word2idx:
            return []

        vec = self.get_vector(word)
        # 计算余弦相似度
        norms = np.linalg.norm(self.W, axis=1)
        similarities = np.dot(self.W, vec) / (norms * np.linalg.norm(vec) + 1e-8)

        # 排序
        top_indices = similarities.argsort()[::-1][1:top_k+1]  # 排除自身
        return [(self.idx2word[i], similarities[i]) for i in top_indices]

    def analogy(self, a, b, c, top_k=3):
        """词类比: a之于b，相当于c之于？ (b - a + c)"""
        if any(w not in self.word2idx for w in [a, b, c]):  # any()任一为True则返回True
            return []

        vec = self.get_vector(b) - self.get_vector(a) + self.get_vector(c)
        norms = np.linalg.norm(self.W, axis=1)
        similarities = np.dot(self.W, vec) / (norms * np.linalg.norm(vec) + 1e-8)

        # 排除输入词
        exclude = {self.word2idx[w] for w in [a, b, c]}
        results = []
        for idx in similarities.argsort()[::-1]:
            if idx not in exclude:
                results.append((self.idx2word[idx], similarities[idx]))
                if len(results) >= top_k:
                    break
        return results

# 训练示例
corpus = [
    ["国王", "住在", "皇宫", "里"],
    ["女王", "住在", "皇宫", "里"],
    ["王子", "是", "国王", "的", "儿子"],
    ["公主", "是", "女王", "的", "女儿"],
    ["男人", "和", "女人", "是", "不同", "的"],
    ["国王", "是", "男人", "的", "统治者"],
    ["女王", "是", "女人", "的", "统治者"],
    ["猫", "是", "可爱", "的", "动物"],
    ["狗", "是", "忠诚", "的", "动物"],
    ["猫", "和", "狗", "都是", "宠物"],
] * 100  # 重复以增加数据量

w2v = SimpleWord2Vec(embedding_dim=30, window_size=2, num_negative=3)
w2v.train(corpus, epochs=10)

# 测试
print("\n最相似的词:")
for word in ["国王", "猫", "动物"]:
    similar = w2v.most_similar(word, top_k=3)
    print(f"  '{word}': {similar}")

# pip install gensim

from gensim.models import Word2Vec

# 准备训练数据
sentences = [
    ["自然语言", "处理", "是", "人工智能", "的", "重要", "分支"],
    ["深度学习", "推动", "了", "自然语言", "处理", "的", "发展"],
    ["机器学习", "是", "人工智能", "的", "基础"],
    ["词向量", "是", "自然语言", "处理", "的", "基础", "技术"],
    ["BERT", "使用", "Transformer", "架构"],
    ["Transformer", "改变", "了", "自然语言", "处理"],
    ["GPT", "是", "一种", "生成式", "预训练", "模型"],
    ["预训练", "模型", "在", "自然语言", "处理", "中", "表现", "出色"],
] * 50

# 训练Word2Vec
model = Word2Vec(
    sentences=sentences,
    vector_size=100,    # 向量维度
    window=5,           # 窗口大小
    min_count=2,        # 最小词频
    workers=4,          # 线程数
    sg=1,               # 1=Skip-gram, 0=CBOW
    epochs=20,          # 训练轮数
    negative=5,         # 负采样数
)

# 获取词向量
vector = model.wv["自然语言"]
print(f"'自然语言' 向量维度: {vector.shape}")
print(f"'自然语言' 向量前10位: {vector[:10]}")

# 相似词查找
print("\n最相似的词:")
try:  # try/except捕获异常
    similar = model.wv.most_similar("自然语言", topn=5)
    for word, score in similar:
        print(f"  {word}: {score:.4f}")
except KeyError as e:
    print(f"  词不在词汇表中: {e}")

# 保存和加载模型
model.save("word2vec.model")
# loaded_model = Word2Vec.load("word2vec.model")

# 只保存词向量（更紧凑）
model.wv.save("word2vec.wv")
# from gensim.models import KeyedVectors
# wv = KeyedVectors.load("word2vec.wv")

class SimpleGloVe:
    """GloVe简化实现：基于共现矩阵"""

    def __init__(self, embedding_dim=50, window_size=5, x_max=100, alpha=0.75):
        self.embedding_dim = embedding_dim
        self.window_size = window_size
        self.x_max = x_max
        self.alpha = alpha

    def build_cooccurrence_matrix(self, corpus):
        """构建共现矩阵"""
        # 建立词汇表
        vocab = set()
        for sentence in corpus:
            vocab.update(sentence)
        self.word2idx = {w: i for i, w in enumerate(sorted(vocab))}
        self.idx2word = {i: w for w, i in self.word2idx.items()}
        self.vocab_size = len(vocab)

        # 构建共现矩阵
        cooccurrence = np.zeros((self.vocab_size, self.vocab_size))

        for sentence in corpus:
            indices = [self.word2idx[w] for w in sentence if w in self.word2idx]
            for i, idx_i in enumerate(indices):
                for j in range(max(0, i - self.window_size),
                              min(len(indices), i + self.window_size + 1)):
                    if i != j:
                        idx_j = indices[j]
                        # 距离越远，权重越小
                        distance = abs(i - j)
                        cooccurrence[idx_i][idx_j] += 1.0 / distance

        return cooccurrence

    def weight_function(self, x):
        """GloVe权重函数"""
        if x < self.x_max:
            return (x / self.x_max) ** self.alpha
        return 1.0

    def train(self, corpus, epochs=50, lr=0.05):
        """训练GloVe模型"""
        cooc = self.build_cooccurrence_matrix(corpus)
        print(f"词汇表大小: {self.vocab_size}")

        # 初始化
        W = np.random.randn(self.vocab_size, self.embedding_dim) * 0.1
        W_tilde = np.random.randn(self.vocab_size, self.embedding_dim) * 0.1
        b = np.zeros(self.vocab_size)
        b_tilde = np.zeros(self.vocab_size)

        # 获取非零共现对
        nonzero_pairs = []
        for i in range(self.vocab_size):
            for j in range(self.vocab_size):
                if cooc[i][j] > 0:
                    nonzero_pairs.append((i, j, cooc[i][j]))

        print(f"非零共现对数: {len(nonzero_pairs)}")

        # 训练
        for epoch in range(epochs):
            total_loss = 0
            random.shuffle(nonzero_pairs)

            for i, j, x_ij in nonzero_pairs:
                weight = self.weight_function(x_ij)

                # 计算误差
                diff = np.dot(W[i], W_tilde[j]) + b[i] + b_tilde[j] - np.log(x_ij + 1)
                loss = weight * diff ** 2
                total_loss += loss

                # 梯度更新
                grad = weight * diff * lr
                W[i] -= grad * W_tilde[j]
                W_tilde[j] -= grad * W[i]
                b[i] -= grad
                b_tilde[j] -= grad

            if (epoch + 1) % 10 == 0:
                avg_loss = total_loss / len(nonzero_pairs)
                print(f"Epoch {epoch+1}/{epochs}, Avg Loss: {avg_loss:.4f}")

        # 最终词向量 = W + W_tilde（论文建议）
        self.embeddings = W + W_tilde

    def get_vector(self, word):
        if word in self.word2idx:
            return self.embeddings[self.word2idx[word]]
        return None

# 示例（小规模演示）
corpus = [
    ["自然", "语言", "处理", "是", "人工智能"],
    ["深度", "学习", "推动", "自然", "语言", "处理"],
    ["机器", "学习", "是", "人工智能", "基础"],
] * 30

glove = SimpleGloVe(embedding_dim=20)
glove.train(corpus, epochs=30, lr=0.05)

class SimpleFastText:
    """FastText核心思想演示：基于子词的词向量"""

    def __init__(self, min_n=3, max_n=6):
        self.min_n = min_n
        self.max_n = max_n

    def get_ngrams(self, word):
        """获取词的所有n-gram"""
        # 添加边界标记
        word = f"<{word}>"
        ngrams = []
        for n in range(self.min_n, self.max_n + 1):
            for i in range(len(word) - n + 1):
                ngrams.append(word[i:i+n])
        return ngrams

ft = SimpleFastText(min_n=3, max_n=6)

# 展示子词分解
words = ["where", "there", "here", "apple"]
for word in words:
    ngrams = ft.get_ngrams(word)
    print(f"'{word}' → {ngrams[:8]}...")  # 只显示前8个
    print()

# Word2Vec/GloVe给每个词一个固定向量 → 无法处理一词多义

examples = {
    "苹果": [
        "我吃了一个苹果",        # 水果
        "苹果公司发布了新产品",   # 公司
    ],
    "bank": [
        "I went to the bank to deposit money",  # 银行
        "The river bank was covered with flowers",  # 河岸
    ],
}

for word, sents in examples.items():
    print(f"'{word}' 的不同含义:")
    for sent in sents:
        print(f"  - {sent}")
    print(f"  → 静态词向量中，两个'{word}'的向量完全相同！")
    print()

import numpy as np
from collections import Counter
from sklearn.metrics import accuracy_score, f1_score
from sklearn.model_selection import train_test_split

# ==================
# 准备数据
# ==================
# 简单的情感分类数据集
data = [
    ("这部电影太好看了强烈推荐", 1),
    ("非常喜欢剧情很精彩", 1),
    ("拍得真好演技在线", 1),
    ("值得一看好片", 1),
    ("超级好看推荐大家看", 1),
    ("很感动的一部电影", 1),
    ("导演功力深厚", 1),
    ("画面精美剧情紧凑", 1),
    ("太难看了浪费时间", 0),
    ("剧情太差不推荐", 0),
    ("很无聊看了一半就关了", 0),
    ("垃圾电影浪费钱", 0),
    ("太差了别看", 0),
    ("无聊透顶完全看不下去", 0),
    ("失望剧情老套", 0),
    ("烂片一部毫无诚意", 0),
]

texts = [t for t, _ in data]
labels = np.array([l for _, l in data])

# 分词
import jieba
tokenized_texts = [jieba.lcut(t) for t in texts]

# ==================
# 方法1: BoW + 简单分类器
# ==================

class SimpleLogisticRegression:
    """简单逻辑回归用于对比"""
    def __init__(self, n_features, lr=0.1, epochs=100):
        self.W = np.zeros(n_features)
        self.b = 0
        self.lr = lr
        self.epochs = epochs

    def sigmoid(self, x):
        return 1 / (1 + np.exp(-np.clip(x, -500, 500)))

    def fit(self, X, y):
        for _ in range(self.epochs):
            pred = self.sigmoid(X @ self.W + self.b)
            error = pred - y
            self.W -= self.lr * (X.T @ error) / len(y)
            self.b -= self.lr * error.mean()

    def predict(self, X):
        return (self.sigmoid(X @ self.W + self.b) > 0.5).astype(int)

# BoW特征
bow = BagOfWords()
bow_features = bow.fit_transform([' '.join(t) for t in tokenized_texts])

# TF-IDF特征
tfidf_model = TFIDF()
tfidf_features = tfidf_model.fit_transform(tokenized_texts)

# 训练和评估
print("="*50)
print("不同文本表示方法的分类效果对比")
print("="*50)

for method_name, features in [("BoW", bow_features), ("TF-IDF", tfidf_features)]:
    clf = SimpleLogisticRegression(features.shape[1], lr=0.5, epochs=200)
    clf.fit(features, labels)
    predictions = clf.predict(features)
    acc = accuracy_score(labels, predictions)
    print(f"\n{method_name}:")
    print(f"  准确率: {acc:.4f}")
    print(f"  特征维度: {features.shape[1]}")

print("\n" + "="*50)
print("注：以上是在训练集上的结果，实际应用应使用交叉验证")

import numpy as np

def visualize_vectors_text(words, vectors, title="词向量2D投影"):
    """
    简化版词向量可视化（文本输出）
    使用PCA降维到2D
    """
    # 简单的PCA实现
    vectors = np.array(vectors)
    mean = vectors.mean(axis=0)
    centered = vectors - mean

    # SVD计算主成分
    U, S, Vt = np.linalg.svd(centered, full_matrices=False)
    projected = centered @ Vt[:2].T

    print(f"\n{title}")
    print("-" * 40)
    print(f"{'词语':<12} {'PC1':<10} {'PC2':<10}")
    print("-" * 40)
    for word, (x, y) in zip(words, projected):
        print(f"{word:<12} {x:>8.4f}  {y:>8.4f}")

    # 计算词间距离
    print(f"\n词间余弦相似度矩阵:")
    for i, w1 in enumerate(words):
        similarities = []
        for j, w2 in enumerate(words):
            v1, v2 = vectors[i], vectors[j]
            sim = np.dot(v1, v2) / (np.linalg.norm(v1) * np.linalg.norm(v2) + 1e-8)
            similarities.append(f"{sim:.2f}")
        print(f"  {w1:<10}: {' '.join(similarities)}")

# 使用Gensim训练的词向量进行可视化
try:
    words_to_show = ["自然语言", "处理", "人工智能", "深度学习", "机器学习"]
    vectors_to_show = [model.wv[w] for w in words_to_show if w in model.wv]
    valid_words = [w for w in words_to_show if w in model.wv]

    if valid_words:
        visualize_vectors_text(valid_words, vectors_to_show)
except NameError:
    print("请先运行前面的Gensim Word2Vec训练代码")

# 加载预训练的中文词向量
# 常用的预训练词向量：
# - Tencent AI Lab: https://ai.tencent.com/ailab/nlp/en/embedding.html
# - Chinese Word Vectors: https://github.com/Embedding/Chinese-Word-Vectors

def load_pretrained_vectors(filepath, limit=None):
    """
    加载预训练词向量（Word2Vec文本格式）
    """
    word2vec = {}
    with open(filepath, 'r', encoding='utf-8', errors='ignore') as f:  # with自动管理文件关闭
        # 第一行通常是 vocab_size embedding_dim
        header = f.readline().strip().split()
        vocab_size, embed_dim = int(header[0]), int(header[1])
        print(f"词汇表大小: {vocab_size}, 向量维度: {embed_dim}")

        count = 0
        for line in f:
            parts = line.strip().split()
            word = parts[0]
            vector = np.array([float(x) for x in parts[1:]])
            word2vec[word] = vector
            count += 1
            if limit and count >= limit:
                break

    print(f"加载了 {len(word2vec)} 个词向量")
    return word2vec

# 使用示例（需要有预训练文件）
# vectors = load_pretrained_vectors("sgns.baidubaike.bigram-char.txt", limit=50000)
# print(vectors.get("自然语言处理"))