Encoder bloğu

import numpy as np

def softmax(z, eksen=-1):
    e = np.exp(z - z.max(axis=eksen, keepdims=True))
    return e / e.sum(axis=eksen, keepdims=True)

metinler = ["film çok iyi", "çok kötü bir film", "iyi", "oyuncular iyi ama senaryo kötü", "kötü"]
sozluk = {"<pad>": 0}
for c in metinler:
    for k in c.split():
        sozluk.setdefault(k, len(sozluk))

def kodla(c, n=5):
    ids = [sozluk[k] for k in c.split()][:n]
    return ids + [0] * (n - len(ids))

X = np.array([kodla(c) for c in metinler])
pad = X == 0

def dikkat(Q, K, V, pad):
    d = Q.shape[-1]
    skor = Q @ K.transpose(0, 2, 1) / np.sqrt(d)
    skor = np.where(pad[:, None, :], -1e9, skor)
    A = softmax(skor)
    return A @ V, A

def layer_norm(x, eps=1e-5):
    return (x - x.mean(-1, keepdims=True)) / np.sqrt(x.var(-1, keepdims=True) + eps)

def konum_kodu(n, d):
    aci = np.arange(n)[:, None] / 10000 ** (np.arange(0, d, 2) / d)
    PE = np.zeros((n, d))
    PE[:, 0::2], PE[:, 1::2] = np.sin(aci), np.cos(aci)
    return PE

rng = np.random.default_rng(0)
d = 8
E = rng.normal(size=(len(sozluk), d)) * 0.5
Wq, Wk, Wv = rng.normal(size=(3, d, d)) * 0.5
W1 = rng.normal(size=(d, 4 * d)) * 0.3
W2 = rng.normal(size=(4 * d, d)) * 0.3

def encoder(X, pad):
    H = E[X] + konum_kodu(X.shape[1], d)
    Y, A = dikkat(H @ Wq, H @ Wk, H @ Wv, pad)
    H = layer_norm(H + Y)
    return layer_norm(H + np.maximum(0, H @ W1) @ W2)

def cumle_vektoru(H, pad):
    m = (~pad)[:, :, None]
    return (H * m).sum(axis=1) / m.sum(axis=1)

H = encoder(X, pad)
C = cumle_vektoru(H, pad)
print(H.shape, C.shape)
E[0] += 10
C2 = cumle_vektoru(encoder(X, pad), pad)
print(np.allclose(H[0, 2], H[2, 0]), np.allclose(C, C2))