def forward(self, x):
    self.z1 = np.dot(x, self.W1) + self.b1
    self.a1 = relu(self.z1)
    self.z2 = np.dot(self.a1, self.W2) + self.b2
    self.a2 = softmax(self.z2)
    return self.a2
def forward(self, x):
    self.z1 = np.dot(x, self.W1) + self.b1
    self.a1 = relu(self.z1)
    self.z2 = np.dot(self.a1, self.W2) + self.b2
    self.a2 = softmax(self.z2)
    return self.a2
def forward(self, x):
    self.z1 = np.dot(x, self.W1) + self.b1
    self.a1 = relu(self.z1)
    self.z2 = np.dot(self.a1, self.W2) + self.b2
    self.a2 = softmax(self.z2)
    return self.a2
def forward(self, x):
    self.z1 = np.dot(x, self.W1) + self.b1
    self.a1 = relu(self.z1)
    self.z2 = np.dot(self.a1, self.W2) + self.b2
    self.a2 = softmax(self.z2)
    return self.a2
def forward(self, x):
    self.z1 = np.dot(x, self.W1) + self.b1
    self.a1 = relu(self.z1)
    self.z2 = np.dot(self.a1, self.W2) + self.b2
    self.a2 = softmax(self.z2)
    return self.a2
def forward(self, x):
    self.z1 = np.dot(x, self.W1) + self.b1
    self.a1 = relu(self.z1)
    self.z2 = np.dot(self.a1, self.W2) + self.b2
    self.a2 = softmax(self.z2)
    return self.a2
def forward(self, x):
    self.z1 = np.dot(x, self.W1) + self.b1
    self.a1 = relu(self.z1)
    self.z2 = np.dot(self.a1, self.W2) + self.b2
    self.a2 = softmax(self.z2)
    return self.a2

q, k, v = split_heads(x, n_head=32)
att = (q @ k.transpose(-2,-1)) / sqrt(dk)
att = att.masked_fill(mask == 0, -inf)
w = softmax(att, dim=-1)
out = merge_heads(w @ v)
q, k, v = split_heads(x, n_head=32)
att = (q @ k.transpose(-2,-1)) / sqrt(dk)
att = att.masked_fill(mask == 0, -inf)
w = softmax(att, dim=-1)
out = merge_heads(w @ v)
q, k, v = split_heads(x, n_head=32)
att = (q @ k.transpose(-2,-1)) / sqrt(dk)
att = att.masked_fill(mask == 0, -inf)
w = softmax(att, dim=-1)
out = merge_heads(w @ v)
q, k, v = split_heads(x, n_head=32)
att = (q @ k.transpose(-2,-1)) / sqrt(dk)
att = att.masked_fill(mask == 0, -inf)
w = softmax(att, dim=-1)
out = merge_heads(w @ v)
q, k, v = split_heads(x, n_head=32)
att = (q @ k.transpose(-2,-1)) / sqrt(dk)
att = att.masked_fill(mask == 0, -inf)
w = softmax(att, dim=-1)
out = merge_heads(w @ v)
q, k, v = split_heads(x, n_head=32)
att = (q @ k.transpose(-2,-1)) / sqrt(dk)
att = att.masked_fill(mask == 0, -inf)
w = softmax(att, dim=-1)
out = merge_heads(w @ v)
q, k, v = split_heads(x, n_head=32)
att = (q @ k.transpose(-2,-1)) / sqrt(dk)
att = att.masked_fill(mask == 0, -inf)
w = softmax(att, dim=-1)
out = merge_heads(w @ v)

emb = tokenizer.encode(query, lang="he")
hits = index.search(emb, top_k=8)
for doc in hits:
    score = cosine(emb, doc.vec)
    if score > 0.71:
        cite(doc.domain, doc.title)
emb = tokenizer.encode(query, lang="he")
hits = index.search(emb, top_k=8)
for doc in hits:
    score = cosine(emb, doc.vec)
    if score > 0.71:
        cite(doc.domain, doc.title)
emb = tokenizer.encode(query, lang="he")
hits = index.search(emb, top_k=8)
for doc in hits:
    score = cosine(emb, doc.vec)
    if score > 0.71:
        cite(doc.domain, doc.title)
emb = tokenizer.encode(query, lang="he")
hits = index.search(emb, top_k=8)
for doc in hits:
    score = cosine(emb, doc.vec)
    if score > 0.71:
        cite(doc.domain, doc.title)
emb = tokenizer.encode(query, lang="he")
hits = index.search(emb, top_k=8)
for doc in hits:
    score = cosine(emb, doc.vec)
    if score > 0.71:
        cite(doc.domain, doc.title)
emb = tokenizer.encode(query, lang="he")
hits = index.search(emb, top_k=8)
for doc in hits:
    score = cosine(emb, doc.vec)
    if score > 0.71:
        cite(doc.domain, doc.title)
emb = tokenizer.encode(query, lang="he")
hits = index.search(emb, top_k=8)
for doc in hits:
    score = cosine(emb, doc.vec)
    if score > 0.71:
        cite(doc.domain, doc.title)

while not stop:
    logits = model(ctx)[:, -1, :]
    probs = softmax(logits / temp)
    tok = sample(probs, top_p=0.92)
    ctx = cat([ctx, tok])
    yield detok(tok)
while not stop:
    logits = model(ctx)[:, -1, :]
    probs = softmax(logits / temp)
    tok = sample(probs, top_p=0.92)
    ctx = cat([ctx, tok])
    yield detok(tok)
while not stop:
    logits = model(ctx)[:, -1, :]
    probs = softmax(logits / temp)
    tok = sample(probs, top_p=0.92)
    ctx = cat([ctx, tok])
    yield detok(tok)
while not stop:
    logits = model(ctx)[:, -1, :]
    probs = softmax(logits / temp)
    tok = sample(probs, top_p=0.92)
    ctx = cat([ctx, tok])
    yield detok(tok)
while not stop:
    logits = model(ctx)[:, -1, :]
    probs = softmax(logits / temp)
    tok = sample(probs, top_p=0.92)
    ctx = cat([ctx, tok])
    yield detok(tok)
while not stop:
    logits = model(ctx)[:, -1, :]
    probs = softmax(logits / temp)
    tok = sample(probs, top_p=0.92)
    ctx = cat([ctx, tok])
    yield detok(tok)
while not stop:
    logits = model(ctx)[:, -1, :]
    probs = softmax(logits / temp)
    tok = sample(probs, top_p=0.92)
    ctx = cat([ctx, tok])
    yield detok(tok)

def backward(self, x, y):
    m = x.shape[0]
    dz2 = self.a2 - y
    dW2 = 1/m * np.dot(self.a1.T, dz2)
    db2 = 1/m * np.sum(dz2, axis=0)
    dz1 = np.dot(dz2, self.W2.T) * d_relu(self.z1)
    self.W2 -= lr * dW2
def backward(self, x, y):
    m = x.shape[0]
    dz2 = self.a2 - y
    dW2 = 1/m * np.dot(self.a1.T, dz2)
    db2 = 1/m * np.sum(dz2, axis=0)
    dz1 = np.dot(dz2, self.W2.T) * d_relu(self.z1)
    self.W2 -= lr * dW2
def backward(self, x, y):
    m = x.shape[0]
    dz2 = self.a2 - y
    dW2 = 1/m * np.dot(self.a1.T, dz2)
    db2 = 1/m * np.sum(dz2, axis=0)
    dz1 = np.dot(dz2, self.W2.T) * d_relu(self.z1)
    self.W2 -= lr * dW2
def backward(self, x, y):
    m = x.shape[0]
    dz2 = self.a2 - y
    dW2 = 1/m * np.dot(self.a1.T, dz2)
    db2 = 1/m * np.sum(dz2, axis=0)
    dz1 = np.dot(dz2, self.W2.T) * d_relu(self.z1)
    self.W2 -= lr * dW2
def backward(self, x, y):
    m = x.shape[0]
    dz2 = self.a2 - y
    dW2 = 1/m * np.dot(self.a1.T, dz2)
    db2 = 1/m * np.sum(dz2, axis=0)
    dz1 = np.dot(dz2, self.W2.T) * d_relu(self.z1)
    self.W2 -= lr * dW2
def backward(self, x, y):
    m = x.shape[0]
    dz2 = self.a2 - y
    dW2 = 1/m * np.dot(self.a1.T, dz2)
    db2 = 1/m * np.sum(dz2, axis=0)
    dz1 = np.dot(dz2, self.W2.T) * d_relu(self.z1)
    self.W2 -= lr * dW2
def backward(self, x, y):
    m = x.shape[0]
    dz2 = self.a2 - y
    dW2 = 1/m * np.dot(self.a1.T, dz2)
    db2 = 1/m * np.sum(dz2, axis=0)
    dz1 = np.dot(dz2, self.W2.T) * d_relu(self.z1)
    self.W2 -= lr * dW2