System.put_env("XLA_BUILD", "true")
Mix.install([
{:nx, "~> 0.11.0"},
{:exla, "~> 0.11.0"},
{:jason, "~> 1.4"}
])
System.put_env("HSA_OVERRIDE_GFX_VERSION", "11.0.0")
System.put_env("XLA_TARGET", "rocm")
# ⚡ 1. Disable the buggy AMD Fusion pass
System.put_env("TF_ROCM_FUSION_ENABLE", "0")
# ⚡ 2. Disable XLA Autotuning completely
System.put_env("XLA_FLAGS", "--xla_gpu_autotune_level=0 --xla_gpu_enable_reassociation_for_converted_ar=false --xla_gpu_graph_enable_concurrent_region=false")
System.put_env("ROCM_DISABLE_CU_MEMSET", "1")
Application.put_env(:exla, :clients,
rocm: [
platform: :rocm,
memory_fraction: 0.75, # ← force total claimed under 8GB
preallocate: false,
num_replicas: 1
]
)
System.put_env("XLA_PYTHON_CLIENT_MEM_FRACTION", "0.33")
Application.put_env(:exla, :default_client, :rocm)
gpu = {EXLA.Backend, client: :rocm}
Nx.global_default_backend(gpu)
Nx.Defn.default_options(compiler: EXLA)
platforms = EXLA.Client.get_supported_platforms()
if platforms[:rocm] do
IO.puts("✅ AMD RX 7600 online!")
else
IO.puts("❌ GPU missing — check HSA_OVERRIDE_GFX_VERSION")
enddefmodule DLM.Positional do
def build(len, dim) do
i = Nx.iota({len, 1})
d = Nx.iota({1, dim})
dim_f = Nx.tensor(dim, type: :f32) # 🔥 Force float math
Nx.sin(Nx.divide(i, Nx.pow(10000.0, Nx.divide(d, dim_f))))
end
end{:module, DLM.Positional, <<70, 79, 82, 49, 0, 0, 8, ...>>, {:build, 2}}
defmodule DLM.Tree do
@moduledoc """
Recursively maps a function over nested tuples of tensors.
Works perfectly both in standard Elixir AND inside Nx `defn`.
"""
def map(tuple, func) when is_tuple(tuple) do
tuple
|> Tuple.to_list()
|> Enum.map(&map(&1, func))
|> List.to_tuple()
end
# The base case: When it hits a tensor (or a compile-time Expr), apply the function
def map(tensor, func) do
func.(tensor)
end
end{:module, DLM.Tree, <<70, 79, 82, 49, 0, 0, 8, ...>>, {:map, 2}}
defmodule DLM.DataStreamer do
def stream(path, seq_len, batch_size) do
lines = path |> File.read!() |> String.split("\n", trim: true) |> Enum.shuffle()
lines
|> Enum.join("\n")
|> to_charlist()
|> Stream.chunk_every(seq_len + 1, seq_len, :discard)
|> Stream.map(fn chunk ->
ins = chunk |> Enum.take(seq_len) |> Nx.tensor(type: :s32)
tgs = chunk |> Enum.drop(1) |> Nx.tensor(type: :s32)
{ins, tgs}
end)
|> Stream.chunk_every(batch_size, batch_size, :discard)
|> Stream.map(fn batch ->
{ins_list, tgs_list} = Enum.unzip(batch)
{Nx.stack(ins_list), Nx.stack(tgs_list)}
end)
end
@doc """
Takes a massive 1D tensor of dataset tokens and creates an infinite stream
of {batch_size, seq_len} matrices for the Physics Engine.
"""
def build_infinite_stream(dataset_1d, batch_size, seq_len) do
chunk_size = batch_size * seq_len
total_tokens = Nx.size(dataset_1d)
# Calculate how many full batches we can make
total_batches = div(total_tokens, chunk_size)
0..(total_batches - 1)
|> Stream.cycle() # ⚡ Loops infinitely!
|> Stream.map(fn batch_idx ->
start_idx = batch_idx * chunk_size
# Slice out the flat chunk and reshape it for the batch
dataset_1d
|> Nx.slice_along_axis(start_idx, chunk_size, axis: 0)
|> Nx.reshape({batch_size, seq_len})
end)
end
end{:module, DLM.DataStreamer, <<70, 79, 82, 49, 0, 0, 19, ...>>, {:build_infinite_stream, 3}}
defmodule DLM.Loss do
import Nx.Defn
defn cross_entropy(logits, targets) do
vocab_size = Nx.axis_size(logits, -1)
# Create a 1-hot vector for the correct answer
one_hot = Nx.equal(Nx.iota({vocab_size}), Nx.new_axis(targets, -1))
# Stable Log-Softmax
max_logits = Nx.reduce_max(logits, axes: [-1], keep_axes: true)
safe_logits = Nx.subtract(logits, max_logits)
log_probs = Nx.subtract(safe_logits, Nx.log(Nx.sum(Nx.exp(safe_logits), axes: [-1], keep_axes: true)))
# Cross Entropy calculation
Nx.mean(Nx.negate(Nx.sum(Nx.multiply(one_hot, log_probs), axes: [-1])))
end
# ⚡ Universal L2 Normalizer (Works on 2D, 3D, N-D tensors)
defn l2_normalize(tensor) do
# Calculate Euclidean norm across the last axis natively
norm = Nx.sqrt(Nx.sum(Nx.pow(tensor, 2), axes: [-1], keep_axes: true))
# Divide by norm with a tiny epsilon to prevent divide-by-zero
Nx.divide(tensor, Nx.add(norm, 1.0e-6))
end
# ⚡ 1. Angular Alignment (Cosine Distance)
defn cosine_distance(pred, target) do
p_norm = l2_normalize(pred)
t_norm = l2_normalize(target)
# Dot product across the 512 dimension
cos_sim = Nx.sum(Nx.multiply(p_norm, t_norm), axes: [-1])
# Minimize distance (1.0 - similarity)
Nx.mean(Nx.subtract(1.0, cos_sim))
end
# ⚡ 2. Repulsion (Memory-Safe Batched Soft-NN for Rank 2 Tensors)
defn soft_nn_loss(pred, target, temperature \\ 0.1) do
# pred and target are {batch_size, dim}
p_l2 = l2_normalize(pred)
t_l2 = l2_normalize(target)
# 1. Batched Dot Product: {batch, dim} x {dim, batch} -> {batch, batch}
# This creates a similarity matrix comparing every pred to every target in the batch
sim_tensor = Nx.dot(p_l2, Nx.transpose(t_l2))
sim_tensor = Nx.divide(sim_tensor, temperature)
# 2. Create Identity Targets -> {batch, batch}
batch_size = Nx.axis_size(pred, 0)
# ⚡ THE FIX: Pass a 2D tuple {32, 32} instead of a 1D tuple {32}
labels = Nx.eye({batch_size, batch_size}, type: :f32)
# 3. Manual Dense Log-Softmax
# Step A: Max trick for numerical stability
max_sim = Nx.reduce_max(sim_tensor, axes: [-1], keep_axes: true)
shifted_sim = Nx.subtract(sim_tensor, max_sim)
# Step B: Log(Sum(Exp))ozen_w2,
log_sum_exp = Nx.log(Nx.sum(Nx.exp(shifted_sim), axes: [-1], keep_axes: true))
log_softmax = Nx.subtract(shifted_sim, log_sum_exp)
# Step C: -Sum(Targets * Log_Softmax)
Nx.multiply(labels, log_softmax)
|> Nx.sum(axes: [-1])
|> Nx.negate()
|> Nx.mean()
end
end{:module, DLM.Loss, <<70, 79, 82, 49, 0, 0, 27, ...>>, true}
defmodule DLM.SinkhornOT do
import Nx.Defn
defn cost_matrix(pred, target) do
p_exp = Nx.new_axis(pred, 1)
t_exp = Nx.new_axis(target, 0)
diff = Nx.subtract(p_exp, t_exp)
Nx.sum(Nx.pow(diff, 2), axes: [-1])
end
# ⚡ FIX 1: Bump the default epsilon up to 10.0 to handle your 512D spatial scale
defn compute(pred, target, epsilon \\ 10.0, max_iters \\ 5) do
c = cost_matrix(pred, target)
k = Nx.exp(Nx.divide(Nx.negate(c), epsilon))
batch_size = Nx.axis_size(pred, 0)
mass = 1.0 / batch_size
mu = Nx.broadcast(mass, {batch_size})
nu = Nx.broadcast(mass, {batch_size})
{_final_k, u_final, v_final, _mu, _nu, _i} =
while {k_mat = k,
u = Nx.broadcast(1.0, {batch_size}),
_v = Nx.broadcast(1.0, {batch_size}),
m = mu, n = nu, i = 0},
Nx.less(i, max_iters) do
# ⚡ FIX 2: Add 1.0e-8 to the denominator to mathematically prevent NaN crashes
denom_v = Nx.add(Nx.dot(Nx.transpose(k_mat), u), 1.0e-8)
v_new = Nx.divide(n, denom_v)
denom_u = Nx.add(Nx.dot(k_mat, v_new), 1.0e-8)
u_new = Nx.divide(m, denom_u)
{k_mat, u_new, v_new, m, n, i + 1}
end
u_exp = Nx.new_axis(u_final, 1)
v_exp = Nx.new_axis(v_final, 0)
p = Nx.multiply(Nx.multiply(u_exp, k), v_exp)
Nx.sum(Nx.multiply(p, c))
end
end{:module, DLM.SinkhornOT, <<70, 79, 82, 49, 0, 0, 25, ...>>, true}
defmodule DLM.Main_SSM do
import Nx.Defn
defn swish(x) do
x * Nx.sigmoid(x)
end
defn rms_norm(x, epsilon \\ 1.0e-6) do
variance = Nx.mean(Nx.pow(x, 2), axes: [-1], keep_axes: true)
x * Nx.rsqrt(variance + epsilon)
end
defn softplus(x) do
# ⚡ UNBREAKABLE SOFTPLUS: Guaranteed stable in forward AND backward passes
Nx.max(x, 0.0) + Nx.log1p(Nx.exp(-Nx.abs(x)))
end
defn apply_svd_weight(x, u, v) do
x |> Nx.dot(u) |> softplus() |> Nx.dot(v)
end
# ⚡ The core physics step (Used heavily by the new Master Loop)
defn ssm_recurrence(x_t, x_p1, x_p2, h_prev, a, bu, bv, cu, cv, du, dv, mu1, mv1, mu2, mv2) do
# 1. Local Mixers
mix1 = apply_svd_weight(x_p2, mu1, mv1)
mix2 = apply_svd_weight(x_p1, mu2, mv2)
x_mixed = Nx.add(x_t, Nx.add(mix1, mix2))
# 2. Input Force
b_proj = apply_svd_weight(x_mixed, bu, bv)
delta = Nx.tensor(0.1, type: :f32)
# 3. Friction
a_bar = Nx.exp(Nx.multiply(delta, -Nx.abs(a)))
b_bar = Nx.multiply(delta, b_proj)
# 4. Momentum Update
h_new = Nx.tanh(Nx.add(Nx.multiply(a_bar, h_prev), b_bar))
# 5. Output Velocity
out_proj = apply_svd_weight(h_new, cu, cv)
# 6. Skip Connection
skip = apply_svd_weight(x_t, du, dv)
y_t = Nx.add(out_proj, skip)
{y_t, h_new}
end
end{:module, DLM.Main_SSM, <<70, 79, 82, 49, 0, 0, 31, ...>>, true}
defmodule DLM.MuonHybrid do
import Nx.Defn
defn newton_schulz(g, iters \\ 5) do
frobenius_norm = Nx.sqrt(Nx.sum(Nx.pow(g, 2))) + 1.0e-7
x = g / frobenius_norm
{final_x, _} = while {curr_x = x, i = 0}, Nx.less(i, iters) do
x_t_x = Nx.dot(Nx.transpose(curr_x), curr_x)
next_x = 1.5 * curr_x - 0.5 * Nx.dot(curr_x, x_t_x)
{next_x, i + 1}
end
final_x
end
defn muon_update(grad, lr) do
Nx.subtract(0.0, Nx.multiply(lr, newton_schulz(grad)))
end
defn adam_update(param, grad, m, v, step, lr) do
safe_step = step + 1.0
beta1 = 0.9
beta2 = 0.999
eps = 1.0e-8
new_m = beta1 * m + (1.0 - beta1) * grad
new_v = beta2 * v + (1.0 - beta2) * Nx.pow(grad, 2)
m_hat = new_m / (1.0 - Nx.pow(beta1, safe_step))
v_hat = new_v / (1.0 - Nx.pow(beta2, safe_step))
param_update = param - lr * m_hat / (Nx.sqrt(v_hat) + eps)
{param_update, new_m, new_v}
end
defn step(params, grads, ms, vs, step_count, lr) do
# ⚡ Unpack 19 elements
{p_a, p_bu, p_bv, p_cu, p_cv, p_du, p_dv, p_mu1, p_mv1, p_mu2, p_mv2,
p_gw, p_gb, p_spell_w, p_spell_b, p_valve_w, p_valve_b, p_out_w, p_out_b} = params
{g_a, g_bu, g_bv, g_cu, g_cv, g_du, g_dv, g_mu1, g_mv1, g_mu2, g_mv2,
g_gw, g_gb, g_spell_w, g_spell_b, g_valve_w, g_valve_b, g_out_w, g_out_b} = grads
{m_a, m_bu, m_bv, m_cu, m_cv, m_du, m_dv, m_mu1, m_mv1, m_mu2, m_mv2,
m_gw, m_gb, m_spell_w, m_spell_b, m_valve_w, m_valve_b, m_out_w, m_out_b} = ms
{v_a, v_bu, v_bv, v_cu, v_cv, v_du, v_dv, v_mu1, v_mv1, v_mu2, v_mv2,
v_gw, v_gb, v_spell_w, v_spell_b, v_valve_w, v_valve_b, v_out_w, v_out_b} = vs
# ⚡ Adam for scalars, biases, and the 1D gate weight
{up_a, nm_a, nv_a} = adam_update(p_a, g_a, m_a, v_a, step_count, lr)
{up_gw, nm_gw, nv_gw} = adam_update(p_gw, g_gw, m_gw, v_gw, step_count, lr)
{up_gb, nm_gb, nv_gb} = adam_update(p_gb, g_gb, m_gb, v_gb, step_count, lr)
{up_spell_b, nm_spell_b, nv_spell_b} = adam_update(p_spell_b, g_spell_b, m_spell_b, v_spell_b, step_count, lr)
{up_valve_b, nm_valve_b, nv_valve_b} = adam_update(p_valve_b, g_valve_b, m_valve_b, v_valve_b, step_count, lr)
{up_out_b, nm_out_b, nv_out_b} = adam_update(p_out_b, g_out_b, m_out_b, v_out_b, step_count, lr)
# ⚡ Muon for SVD and Dense 2D Matrices
up_bu = p_bu + muon_update(g_bu, lr)
up_bv = p_bv + muon_update(g_bv, lr)
up_cu = p_cu + muon_update(g_cu, lr)
up_cv = p_cv + muon_update(g_cv, lr)
up_du = p_du + muon_update(g_du, lr)
up_dv = p_dv + muon_update(g_dv, lr)
up_mu1 = p_mu1 + muon_update(g_mu1, lr)
up_mv1 = p_mv1 + muon_update(g_mv1, lr)
up_mu2 = p_mu2 + muon_update(g_mu2, lr)
up_mv2 = p_mv2 + muon_update(g_mv2, lr)
up_spell_w = p_spell_w + muon_update(g_spell_w, lr)
up_valve_w = p_valve_w + muon_update(g_valve_w, lr)
up_out_w = p_out_w + muon_update(g_out_w, lr)
# Repack
new_params = {up_a, up_bu, up_bv, up_cu, up_cv, up_du, up_dv, up_mu1, up_mv1, up_mu2, up_mv2,
up_gw, up_gb, up_spell_w, up_spell_b, up_valve_w, up_valve_b, up_out_w, up_out_b}
new_ms = {nm_a, m_bu, m_bv, m_cu, m_cv, m_du, m_dv, m_mu1, m_mv1, m_mu2, m_mv2,
nm_gw, nm_gb, m_spell_w, nm_spell_b, m_valve_w, nm_valve_b, m_out_w, nm_out_b}
new_vs = {nv_a, v_bu, v_bv, v_cu, v_cv, v_du, v_dv, v_mu1, v_mv1, v_mu2, v_mv2,
nv_gw, nv_gb, v_spell_w, nv_spell_b, v_valve_w, nv_valve_b, v_out_w, nv_out_b}
{new_params, new_ms, new_vs}
end
end{:module, DLM.MuonHybrid, <<70, 79, 82, 49, 0, 0, 45, ...>>, true}
defmodule DLM.Main_SSM_Genesis do
@vocab_size 128
@dim 512
@rank 256
def init_params(seed \\ 42) do
key = Nx.Random.key(seed)
{a_diag, key} = Nx.Random.normal(key, -0.1, 0.01, shape: {@dim})
{b_u, key} = Nx.Random.normal(key, 0.0, 0.02, shape: {@dim, @rank})
{b_v, key} = Nx.Random.normal(key, 0.0, 0.02, shape: {@rank, @dim})
{c_u, key} = Nx.Random.normal(key, 0.0, 0.02, shape: {@dim, @rank})
{c_v, key} = Nx.Random.normal(key, 0.0, 0.02, shape: {@rank, @dim})
{d_u, key} = Nx.Random.normal(key, 0.0, 0.02, shape: {@dim, @rank})
{d_v, key} = Nx.Random.normal(key, 0.0, 0.02, shape: {@rank, @dim})
{mx_u1, key} = Nx.Random.normal(key, 0.0, 0.02, shape: {@dim, @rank})
{mx_v1, key} = Nx.Random.normal(key, 0.0, 0.02, shape: {@rank, @dim})
{mx_u2, key} = Nx.Random.normal(key, 0.0, 0.02, shape: {@dim, @rank})
{mx_v2, key} = Nx.Random.normal(key, 0.0, 0.02, shape: {@rank, @dim})
# 1. The Boundary Gate
{gate_w, key} = Nx.Random.normal(key, 0.0, 0.02, shape: {@dim, 1})
gate_b = Nx.tensor([-3.0], type: :f32)
# 2. ⚡ The Spell Feature Generator
{spell_w, key} = Nx.Random.normal(key, 0.0, 0.02, shape: {@dim, @dim})
spell_b = Nx.broadcast(0.0, {@dim})
# 3. ⚡ The Syntax Valve (Swish Gate)
{valve_w, key} = Nx.Random.normal(key, 0.0, 0.02, shape: {@dim, @dim})
valve_b = Nx.broadcast(0.0, {@dim})
# 4. ⚡ Latent Output Projection
{out_w, _key} = Nx.Random.normal(key, 0.0, 0.02, shape: {@dim, @dim})
out_b = Nx.broadcast(0.0, {@dim})
# 19 Parameters Total
{a_diag, b_u, b_v, c_u, c_v, d_u, d_v, mx_u1, mx_v1, mx_u2, mx_v2,
gate_w, gate_b, spell_w, spell_b, valve_w, valve_b, out_w, out_b}
end
def init_optimizer(params) do
zero_like = fn t -> Nx.broadcast(Nx.tensor(0.0, type: :f32), t) end
ms = DLM.Tree.map(params, zero_like)
vs = DLM.Tree.map(params, zero_like)
{ms, vs}
end
endwarning: module attribute @vocab_size was set but never used
└─ hssmp3.livemd#cell:albe4mvo5o252ghx:2: DLM.Main_SSM_Genesis (module)
{:module, DLM.Main_SSM_Genesis, <<70, 79, 82, 49, 0, 0, 18, ...>>, {:init_optimizer, 1}}
defmodule DLM.TinySSM do
import Nx.Defn
defn gelu(x) do
cdf = Nx.multiply(
Nx.tensor(0.7978845608, type: :f32),
Nx.add(x, Nx.multiply(Nx.tensor(0.044715, type: :f32), Nx.pow(x, 3)))
)
Nx.multiply(Nx.multiply(Nx.tensor(0.5, type: :f32), x), Nx.add(Nx.tensor(1.0, type: :f32), Nx.tanh(cdf)))
end
defn rms_norm(x, epsilon \\ 1.0e-6) do
variance = Nx.mean(Nx.pow(x, 2), axes: [-1], keep_axes: true)
x * Nx.rsqrt(variance + epsilon)
end
defn apply_svd(x, u, v) do
hidden = Nx.dot(x, u)
# 🛡️ The Shield: normalizes variance to 1.0 BEFORE the tanh
normed_hidden = rms_norm(hidden)
activated = Nx.tanh(normed_hidden)
Nx.dot(activated, v)
end
defn softplus(x) do
Nx.select(x > 20.0, x, Nx.log1p(Nx.exp(x)))
end
defn forward(x_t, h_prev, params) do
# ⚡ Unpack the 7 parameters
{a, bu, bv, cu, cv, du, dv} = params
b_proj = apply_svd(x_t, bu, bv)
# ⚡ The Dynamic Gate
gate_proj = apply_svd(x_t, du, dv)
# ⚡ Shift the projection by -3.0 so the initial delta is ~0.048
shifted_gate = Nx.subtract(gate_proj, 3.0)
delta = Nx.max(softplus(shifted_gate), 0.01)
# Dynamic Friction & Force
a_bar = Nx.exp(Nx.multiply(delta, -Nx.abs(a)))
b_bar = b_proj
h_new = Nx.tanh(Nx.add(Nx.multiply(a_bar, h_prev), b_bar))
predicted_vector = apply_svd(h_new, cu, cv)
{predicted_vector, h_new}
end
end{:module, DLM.TinySSM, <<70, 79, 82, 49, 0, 0, 23, ...>>, true}
defmodule DLM.Surgery do
def upgrade_to_tier3(old_ckpt_path, new_prefix, gpu) do
IO.puts("🔪 Performing Tensor Surgery on #{old_ckpt_path}...")
ckpt = File.read!(old_ckpt_path) |> :erlang.binary_to_term()
# 1. Unpack the 15 Phase 3 parameters
{a, bu, bv, cu, cv, du, dv, mu1, mv1, mu2, mv2, _old_lmw, _old_lmb, g1_w, g1_b} = ckpt.params
# 2. Re-pack the 13 parameters for the Mid Level (Discarding the old head)
mid_p = {a, bu, bv, cu, cv, du, dv, mu1, mv1, mu2, mv2, g1_w, g1_b}
# 3. Initialize a brand new Top Level (15 parameters, including g2 and the new head)
key = Nx.Random.key(42)
{top_p, _} = DLM.Main_SSM_Genesis.init_phase3(key)
# (Assuming your init function generates the a...mv2, lm_w, lm_b, g2_w, and -3.0 g2_b)
# 4. Initialize fresh optimizer states for the paired {mid, top} tuple
zero_mid = DLM.Tree.map(mid_p, fn t -> Nx.broadcast(0.0, Nx.shape(t)) end)
zero_top = DLM.Tree.map(top_p, fn t -> Nx.broadcast(0.0, Nx.shape(t)) end)
# 5. Save the new 3-Tier Genesis Checkpoint
state = %{
params: {mid_p, top_p},
ms: {zero_mid, zero_top},
vs: {zero_mid, zero_top},
step: 0,
h_base: Nx.broadcast(0.0, {1, 512}),
h_mid: Nx.broadcast(0.0, {1, 512}), # The old h_top is now h_mid
h_top: Nx.broadcast(0.0, {1, 512}) # Brand new grammatical memory
}
File.write!("#{new_prefix}_checkpoint_step_0.bin", :erlang.term_to_binary(state))
IO.puts("✅ Surgery Complete. Ready for Phase 4.")
end
endgpu = {EXLA.Backend, client: :rocm}
DLM.Surgery.upgrade_to_tier3("v34_DLM_Phase_3_Top_Biased_checkpoint_step_80000.bin", "v35_level_2", gpu)defmodule DLM.BLiMP do
import Nx.Defn
# ⚡ 1. The Evaluation Forward Pass (V2 Swish Latent Architecture)
defn eval_loss(tokens, h_base_init, h_top_init, frozen_base_p, frozen_map, top_p) do
seq_len = Nx.axis_size(tokens, 1) - 1
input_tokens = Nx.slice_along_axis(tokens, 0, seq_len, axis: 1)
target_tokens = Nx.slice_along_axis(tokens, 1, seq_len, axis: 1)
# Map inputs to coordinates
clean_coords = Nx.take(frozen_map, Nx.flatten(input_tokens))
|> Nx.reshape({1, seq_len, 512})
in_t = Nx.transpose(clean_coords, axes: [1, 0, 2])
# ⚡ NEW: Map targets to continuous coordinates for Cosine Evaluation
target_coords = Nx.take(frozen_map, Nx.flatten(target_tokens))
|> Nx.reshape({1, seq_len, 512})
targ_t = Nx.transpose(target_coords, axes: [1, 0, 2])
zero_frame = Nx.broadcast(0.0, {1, 512})
result =
while {i = 0, total_loss = 0.0,
h_base = h_base_init, h_top = h_top_init,
xp1 = zero_frame, xp2 = zero_frame,
x_in = in_t, y_targ = targ_t,
b_p = frozen_base_p, t_p = top_p, f_map = frozen_map},
Nx.less(i, seq_len) do
x_curr = x_in[i]
# 1. Base SSM
{_y_base, h_base_new} = DLM.TinySSM.forward(x_curr, h_base, b_p)
# ⚡ Unpack all 19 Params for the Swish Gate
{a, bu, bv, cu, cv, du, dv, mu1, mv1, mu2, mv2,
gate_w, gate_b, spell_w, spell_b, valve_w, valve_b, out_w, out_b} = t_p
# 2. Boundary Gate
gate_logit = Nx.add(Nx.dot(h_base_new, gate_w), gate_b)
g = Nx.sigmoid(gate_logit)
# 3. Top SSM
{_y_top, h_top_candidate} = DLM.Main_SSM.ssm_recurrence(
h_base_new, xp1, xp2, h_top,
a, bu, bv, cu, cv, du, dv, mu1, mv1, mu2, mv2
)
# 4. Continuous Update
h_top_new = Nx.add(Nx.multiply(g, h_top_candidate), Nx.multiply(Nx.subtract(1.0, g), h_top))
h_base_flushed = Nx.multiply(h_base_new, Nx.subtract(1.0, g))
# ⚡ 5. Multiplicative Swish Fusion
f_spell = Nx.add(Nx.dot(h_base_new, spell_w), spell_b)
v_syntax = DLM.Main_SSM.swish(Nx.add(Nx.dot(h_top_new, valve_w), valve_b))
h_fused = Nx.multiply(v_syntax, f_spell)
h_fused_norm = DLM.Main_SSM.rms_norm(h_fused)
# ⚡ 6. Latent Prediction & Cosine Loss
pred_latent = Nx.add(Nx.dot(h_fused_norm, out_w), out_b)
step_loss = DLM.Loss.cosine_distance(pred_latent, y_targ[i])
{i + 1, total_loss + step_loss, h_base_flushed, h_top_new, h_base_new, xp1, x_in, y_targ, b_p, t_p, f_map}
end
# Return average sequence Cosine Loss
Nx.divide(elem(result, 1), seq_len)
end
# ⚡ 2. The Elixir Wrapper
def evaluate_sentence(sentence, frozen_base_p, frozen_map, top_p, gpu) do
# Convert string to {1, seq_len} tensor
tokens = sentence |> to_charlist() |> Nx.tensor(type: :s32) |> Nx.new_axis(0) |> Nx.backend_copy(gpu)
# Initialize blank memory for a single batch
h_base = Nx.broadcast(0.0, {1, 512}) |> Nx.backend_copy(gpu)
h_top = Nx.broadcast(0.0, {1, 512}) |> Nx.backend_copy(gpu)
# Run the compiled XLA graph
loss_tensor = eval_loss(tokens, h_base, h_top, frozen_base_p, frozen_map, top_p)
Nx.to_number(loss_tensor)
end
end{:module, DLM.BLiMP, <<70, 79, 82, 49, 0, 0, 30, ...>>, {:evaluate_sentence, 5}}
gpu = {EXLA.Backend, client: :rocm}
blimp_file = "./data/determiner_noun_agreement_1.jsonl" # 👈 Change this to your specific BLiMP file
# ==============================================================================
# 1. LOAD WEIGHTS
# ==============================================================================
IO.puts("🌌 Loading Phase 1, 2, and 3 Weights...")
%{params: phase1_p} = File.read!("phase1_shape_diffusion_COMPLETE.bin") |> :erlang.binary_to_term()
frozen_map = elem(phase1_p, 0) |> Nx.backend_copy(gpu) |> DLM.Loss.l2_normalize()
frozen_base_p = File.read!("v33_DLM_Phase_2_cos_checkpoint_step_13000.bin") |> :erlang.binary_to_term()
# Load your best Phase 3 Top Level weights (Adjust filename as needed)
top_ckpt = File.read!("v34_DLM_Phase_3_Top_256_lowLR_checkpoint_step_44000.bin") |> :erlang.binary_to_term()
top_p = DLM.Tree.map(top_ckpt.params, &Nx.backend_copy(&1, gpu))
# ==============================================================================
# 2. RUN BLIMP EVALUATION
# ==============================================================================
IO.puts("📖 Reading BLiMP Dataset: #{blimp_file}")
# Parse the JSON Lines
pairs =
File.read!(blimp_file)
|> String.split("\n", trim: true)
|> Enum.map(&Jason.decode!/1)
total_pairs = length(pairs)
IO.puts("⚖️ Evaluating #{total_pairs} minimalist pairs...")
# Tally the results
{passes, total_loss_diff} =
Enum.reduce(pairs, {0, 0.0}, fn pair, {pass_count, diff_acc} ->
good_sentence = pair["sentence_good"]
bad_sentence = pair["sentence_bad"]
# Calculate losses
good_loss = DLM.BLiMP.evaluate_sentence(good_sentence, frozen_base_p, frozen_map, top_p, gpu)
bad_loss = DLM.BLiMP.evaluate_sentence(bad_sentence, frozen_base_p, frozen_map, top_p, gpu)
# Did the model assign a lower loss (higher probability) to the grammatically correct sentence?
is_pass? = good_loss < bad_loss
new_pass_count = if is_pass?, do: pass_count + 1, else: pass_count
{new_pass_count, diff_acc + (bad_loss - good_loss)}
end)
# Calculate final metrics
accuracy = (passes / total_pairs) * 100.0
avg_margin = total_loss_diff / total_pairs
IO.puts(String.duplicate("=", 50))
IO.puts("🏆 BLiMP EVALUATION RESULTS")
IO.puts(String.duplicate("=", 50))
IO.puts("Dataset: #{blimp_file}")
IO.puts("Accuracy: #{Float.round(accuracy, 2)}% (#{passes}/#{total_pairs})")
IO.puts("Margin: #{Float.round(avg_margin, 4)} (Avg loss difference)")
IO.puts(String.duplicate("=", 50))🌌 Loading Phase 1, 2, and 3 Weights...
📖 Reading BLiMP Dataset: ./data/determiner_noun_agreement_1.jsonl
⚖️ Evaluating 1000 minimalist pairs...
==================================================
🏆 BLiMP EVALUATION RESULTS
==================================================
Dataset: ./data/determiner_noun_agreement_1.jsonl
Accuracy: 51.2% (512/1000)
Margin: 0.0001 (Avg loss difference)
==================================================
defmodule DLM.Generator do
import Nx.Defn
# ⚡ 1. The Single Forward Step
defn step(char_idx, h_base, h_top, xp1, xp2, frozen_base_p, top_p, f_map) do
# Unpack params
{a, bu, bv, cu, cv, du, dv, mu1, mv1, mu2, mv2, lm_w, lm_b, gate_w, gate_b} = top_p
# Get the continuous 512D coordinate for this discrete character
coord = Nx.take(f_map, Nx.reshape(char_idx, {1})) |> Nx.reshape({1, 512})
# Physics Engine Step
{_y_b, h_b_new} = DLM.TinySSM.forward(coord, h_base, frozen_base_p)
# The Boundary Gate
gate_logit = Nx.add(Nx.dot(h_b_new, gate_w), gate_b)
g = Nx.sigmoid(gate_logit)
# Top Level Logic Step
{y_t, h_t_cand} = DLM.Main_SSM.ssm_recurrence(
h_b_new, xp1, xp2, h_top,
a, bu, bv, cu, cv, du, dv, mu1, mv1, mu2, mv2
)
# Continuous Interpolation & Memory Flush
h_t_new = Nx.add(Nx.multiply(g, h_t_cand), Nx.multiply(Nx.subtract(1.0, g), h_top))
h_b_flushed = Nx.multiply(h_b_new, Nx.subtract(1.0, g))
# Output Head
y_final = DLM.Main_SSM.rms_norm(y_t)
logits = Nx.add(Nx.dot(y_final, lm_w), lm_b) |> Nx.squeeze()
# Return logits, the updated states, the shift register (xp1/xp2), and the gate value
{logits, h_b_flushed, h_t_new, h_b_new, xp1, Nx.squeeze(g)}
end
# ⚡ 2. The Gumbel Sampler
defn sample(logits, key, temperature) do
scaled = Nx.divide(logits, Nx.add(temperature, 1.0e-5))
{u, next_key} = Nx.Random.uniform(key, 0.0, 1.0, shape: Nx.shape(scaled))
# Gumbel noise distribution
gumbel = Nx.negate(Nx.log(Nx.negate(Nx.log(Nx.add(u, 1.0e-7)))))
noisy = Nx.add(scaled, gumbel)
{Nx.argmax(noisy), next_key}
end
# ⚡ 3. The Generation Loop (Runs in standard Elixir)
def generate(prompt, num_tokens, temp, f_base_p, t_p, f_map, gpu) do
prompt_tokens = to_charlist(prompt)
key = Nx.Random.key(System.system_time(:second)) |> Nx.backend_copy(gpu)
# Initialize blank states
h_b = Nx.broadcast(0.0, {1, 512}) |> Nx.backend_copy(gpu)
h_t = Nx.broadcast(0.0, {1, 512}) |> Nx.backend_copy(gpu)
xp1 = Nx.broadcast(0.0, {1, 512}) |> Nx.backend_copy(gpu)
xp2 = Nx.broadcast(0.0, {1, 512}) |> Nx.backend_copy(gpu)
# Warmup Phase (Feed the prompt in without sampling)
IO.puts("🔥 Warming up semantic context...")
{warm_hb, warm_ht, warm_xp1, warm_xp2} = Enum.reduce(prompt_tokens, {h_b, h_t, xp1, xp2},
fn char, {cb, ct, cx1, cx2} ->
char_tensor = Nx.tensor(char, type: :s32) |> Nx.backend_copy(gpu)
{_logits, nb, nt, nx1, nx2, _g} = step(char_tensor, cb, ct, cx1, cx2, f_base_p, t_p, f_map)
{nb, nt, nx1, nx2}
end
)
# Generation Phase
IO.puts("\n🚀 Generating...\n")
IO.write(prompt)
last_char = List.last(prompt_tokens)
Enum.reduce(1..num_tokens, {last_char, warm_hb, warm_ht, warm_xp1, warm_xp2, key},
fn _i, {curr_char, cb, ct, cx1, cx2, cur_key} ->
char_tensor = Nx.tensor(curr_char, type: :s32) |> Nx.backend_copy(gpu)
# Step the network
{logits, nb, nt, nx1, nx2, g_tensor} = step(char_tensor, cb, ct, cx1, cx2, f_base_p, t_p, f_map)
# Sample the next character
{next_char_tensor, next_key} = sample(logits, cur_key, Nx.tensor(temp, type: :f32))
next_char = Nx.to_number(next_char_tensor)
# Pull the gate value for visualization
g_val = Nx.to_number(g_tensor)
# If the gate spikes above 0.5, we'll visually mark it with a pipe |
marker = if g_val > 0.5, do: "|", else: ""
IO.write("#{marker}#{List.to_string([next_char])}")
{next_char, nb, nt, nx1, nx2, next_key}
end
)
IO.puts("\n\n✅ Done.")
end
end{:module, DLM.Generator, <<70, 79, 82, 49, 0, 0, 37, ...>>, {:generate, 7}}
gpu = {EXLA.Backend, client: :rocm}
blimp_file = "./data/regular_plural_subject_verb_agreement_1.jsonl" # 👈 Change this to your specific BLiMP file
# ==============================================================================
# 1. LOAD WEIGHTS
# ==============================================================================
IO.puts("🌌 Loading Phase 1, 2, and 3 Weights...")
%{params: phase1_p} = File.read!("phase1_shape_diffusion_COMPLETE.bin") |> :erlang.binary_to_term()
frozen_map = elem(phase1_p, 0) |> Nx.backend_copy(gpu) |> DLM.Loss.l2_normalize()
frozen_base_p = File.read!("v33_DLM_Phase_2_cos_checkpoint_step_13000.bin") |> :erlang.binary_to_term()
# Load your best Phase 3 Top Level weights (Adjust filename as needed)
top_ckpt = File.read!("v34_DLM_Phase_3_Top_256_midLR_checkpoint_step_16000.bin") |> :erlang.binary_to_term()
top_p = DLM.Tree.map(top_ckpt.params, &Nx.backend_copy(&1, gpu))
# ==============================================================================
# 2. RUN BLIMP EVALUATION
# ==============================================================================
IO.puts("📖 Reading BLiMP Dataset: #{blimp_file}")
# Parse the JSON Lines
pairs =
File.read!(blimp_file)
|> String.split("\n", trim: true)
|> Enum.map(&Jason.decode!/1)
total_pairs = length(pairs)
IO.puts("⚖️ Evaluating #{total_pairs} minimalist pairs...")
# Tally the results
{passes, total_loss_diff} =
Enum.reduce(pairs, {0, 0.0}, fn pair, {pass_count, diff_acc} ->
good_sentence = pair["sentence_good"]
bad_sentence = pair["sentence_bad"]
# Calculate losses
good_loss = DLM.BLiMP.evaluate_sentence(good_sentence, frozen_base_p, frozen_map, top_p, gpu)
bad_loss = DLM.BLiMP.evaluate_sentence(bad_sentence, frozen_base_p, frozen_map, top_p, gpu)
# Did the model assign a lower loss (higher probability) to the grammatically correct sentence?
is_pass? = good_loss < bad_loss
new_pass_count = if is_pass?, do: pass_count + 1, else: pass_count
{new_pass_count, diff_acc + (bad_loss - good_loss)}
end)
# Calculate final metrics
accuracy = (passes / total_pairs) * 100.0
avg_margin = total_loss_diff / total_pairs
IO.puts(String.duplicate("=", 50))
IO.puts("🏆 BLiMP EVALUATION RESULTS")
IO.puts(String.duplicate("=", 50))
IO.puts("Dataset: #{blimp_file}")
IO.puts("Accuracy: #{Float.round(accuracy, 2)}% (#{passes}/#{total_pairs})")
IO.puts("Margin: #{Float.round(avg_margin, 4)} (Avg loss difference)")
IO.puts(String.duplicate("=", 50))🌌 Loading Phase 1, 2, and 3 Weights...
📖 Reading BLiMP Dataset: ./data/regular_plural_subject_verb_agreement_1.jsonl
⚖️ Evaluating 1000 minimalist pairs...
** (MatchError) no match of right hand side value: {#Nx.Tensor<
f32[512]
Nx.Defn.Expr
parameter a:15 f32[512]
>, #Nx.Tensor<
f32[512][64]
Nx.Defn.Expr
parameter a:16 f32[512][64]
>, #Nx.Tensor<
f32[64][512]
Nx.Defn.Expr
parameter a:17 f32[64][512]
>, #Nx.Tensor<
f32[512][64]
Nx.Defn.Expr
parameter a:18 f32[512][64]
>, #Nx.Tensor<
f32[64][512]
Nx.Defn.Expr
parameter a:19 f32[64][512]
>, #Nx.Tensor<
f32[512][64]
Nx.Defn.Expr
parameter a:20 f32[512][64]
>, #Nx.Tensor<
f32[64][512]
Nx.Defn.Expr
parameter a:21 f32[64][512]
>, #Nx.Tensor<
f32[512][64]
Nx.Defn.Expr
parameter a:22 f32[512][64]
>, #Nx.Tensor<
f32[64][512]
Nx.Defn.Expr
parameter a:23 f32[64][512]
>, #Nx.Tensor<
f32[512][64]
Nx.Defn.Expr
parameter a:24 f32[512][64]
>, #Nx.Tensor<
f32[64][512]
Nx.Defn.Expr
parameter a:25 f32[64][512]
>, #Nx.Tensor<
f32[512][128]
Nx.Defn.Expr
parameter a:26 f32[512][128]
>, #Nx.Tensor<
f32[128]
Nx.Defn.Expr
parameter a:27 f32[128]
>, #Nx.Tensor<
f32[512][1]
Nx.Defn.Expr
parameter a:28 f32[512][1]
>, #Nx.Tensor<
f32[1]
Nx.Defn.Expr
parameter a:29 f32[1]
>}
#cell:lfbfrue6mc255axz:35: anonymous fn/3 in DLM.BLiMP."__defn:eval_loss__"/6
(nx 0.11.0) lib/nx/defn/expr.ex:530: Nx.Defn.Expr.while_vectorized/7
#cell:lfbfrue6mc255axz:22: DLM.BLiMP."__defn:eval_loss__"/6
(nx 0.11.0) lib/nx/defn/compiler.ex:190: Nx.Defn.Compiler.runtime_fun/3
(exla 0.11.0) lib/exla/defn.ex:377: anonymous fn/4 in EXLA.Defn.compile/8
(exla 0.11.0) lib/exla/defn/locked_cache.ex:36: EXLA.Defn.LockedCache.run/2
(stdlib 6.2.2) timer.erl:595: :timer.tc/2
#cell:iqhd5oaj6cbvkwag:33: (file)
# ==============================================================================
# 1. LOAD WEIGHTS
# ==============================================================================
IO.puts("🌌 Loading Phase 1, 2, and 3 Weights...")
gpu = {EXLA.Backend, client: :rocm}
%{params: phase1_p} = File.read!("phase1_shape_diffusion_COMPLETE.bin") |> :erlang.binary_to_term()
frozen_map = elem(phase1_p, 0) |> Nx.backend_copy(gpu) |> DLM.Loss.l2_normalize()
frozen_base_p = File.read!("v33_DLM_Phase_2_cos_checkpoint_step_13000.bin") |> :erlang.binary_to_term()
# Load your best Phase 3 Top Level weights (Adjust filename as needed)
top_ckpt = File.read!("v34_DLM_Phase_3_Top_Biased_checkpoint_step_16000.bin") |> :erlang.binary_to_term()
top_p = DLM.Tree.map(top_ckpt.params, &Nx.backend_copy(&1, gpu))
prompt = "Hello, my name is "
tokens_to_generate = 128 - String.length(prompt)
DLM.Generator.generate(prompt, tokens_to_generate, 0.65, frozen_base_p, top_p, frozen_map, gpu)🌌 Loading Phase 1, 2, and 3 Weights...
10:42:32.060 [info] XLA service 0x7e08a409ab70 initialized for platform ROCM (this does not guarantee that XLA will be used). Devices:
10:42:32.062 [info] StreamExecutor [0]: AMD Radeon RX 7600, AMDGPU ISA version: gfx1100 (Driver: 6.4.43482; Runtime: 6.4.43482; Toolkit: 6.4.43482; DNN: 0.0.0)
10:42:32.062 [info] Using BFC allocator.
10:42:32.062 [info] XLA backend will use up to 6429868032 bytes on device 0 for BFCAllocator.
10:42:32.062 [info] XLA backend will use up to 2143289344 bytes on device 0 for CollectiveBFCAllocator.
🔥 Warming up semantic context...
** (UndefinedFunctionError) function DLM.TinySSM.forward/3 is undefined (module DLM.TinySSM is not available)
DLM.TinySSM.forward(#Nx.Tensor<
f32[1][512]
Nx.Defn.Expr
parameter a:27 f32[128][512]
parameter b:0 s32
c = reshape b s32[1]
d = take a, c, axis: 0 f32[1][512]
>, #Nx.Tensor<
f32[1][512]
Nx.Defn.Expr
parameter a:1 f32[1][512]
>, {#Nx.Tensor<
f32[512]
Nx.Defn.Expr
parameter a:5 f32[512]
>, #Nx.Tensor<
f32[512][256]
Nx.Defn.Expr
parameter a:6 f32[512][256]
>, #Nx.Tensor<
f32[256][512]
Nx.Defn.Expr
parameter a:7 f32[256][512]
>, #Nx.Tensor<
f32[512][256]
Nx.Defn.Expr
parameter a:8 f32[512][256]
>, #Nx.Tensor<
f32[256][512]
Nx.Defn.Expr
parameter a:9 f32[256][512]
>, #Nx.Tensor<
f32[512][256]
Nx.Defn.Expr
parameter a:10 f32[512][256]
>, #Nx.Tensor<
f32[256][512]
Nx.Defn.Expr
parameter a:11 f32[256][512]
>})
#cell:jowzjq3lsg7lgi63:13: DLM.Generator."__defn:step__"/8
(nx 0.11.0) lib/nx/defn/compiler.ex:190: Nx.Defn.Compiler.runtime_fun/3
(exla 0.11.0) lib/exla/defn.ex:377: anonymous fn/4 in EXLA.Defn.compile/8
(exla 0.11.0) lib/exla/defn/locked_cache.ex:36: EXLA.Defn.LockedCache.run/2
(stdlib 6.2.2) timer.erl:595: :timer.tc/2
#cell:a4gyninsclvsoeku:20: (file)
# defmodule DLM.Probe do
# import Nx.Defn
# # Run the sequence and extract ONLY the gate values
# defn extract_gates(tokens, h_base_init, h_top_init, frozen_base_p, frozen_map, top_p) do
# seq_len = Nx.axis_size(tokens, 1)
# clean_coords = Nx.take(frozen_map, Nx.flatten(tokens)) |> Nx.reshape({1, seq_len, 512})
# x_in = Nx.transpose(clean_coords, axes: [1, 0, 2])
# zero_frame = Nx.broadcast(0.0, {1, 512})
# g_out_init = Nx.broadcast(0.0, {seq_len})
# result =
# while {i = 0, h_b = h_base_init, h_t = h_top_init, xp1 = zero_frame, xp2 = zero_frame,
# gates = g_out_init, x_t = x_in, b_p = frozen_base_p, t_p = top_p},
# Nx.less(i, seq_len) do
# x_curr = x_t[i]
# {_y_base, h_b_new} = DLM.TinySSM.forward(x_curr, h_b, b_p)
# {a, bu, bv, cu, cv, du, dv, mu1, mv1, mu2, mv2, _lmw, _lmb, gate_w} = t_p
# # ⚡ Calculate the Gate
# gate_logit = Nx.dot(h_b_new, gate_w)
# g = Nx.sigmoid(gate_logit) |> Nx.squeeze()
# # Step the top layer to keep the math accurate
# {_y_top, h_t_cand} = DLM.Main_SSM.ssm_recurrence(h_b_new, xp1, xp2, h_t, a, bu, bv, cu, cv, du, dv, mu1, mv1, mu2, mv2)
# h_t_new = Nx.add(Nx.multiply(g, h_t_cand), Nx.multiply(Nx.subtract(1.0, g), h_t))
# h_b_flushed = Nx.multiply(h_b_new, Nx.subtract(1.0, g))
# updated_gates = Nx.put_slice(gates, [i], Nx.new_axis(g, 0))
# {i + 1, h_b_flushed, h_t_new, h_b_new, xp1, updated_gates, x_t, b_p, t_p}
# end
# elem(result, 5)
# end
# def watch_the_heartbeat(sentence, frozen_base_p, frozen_map, top_p, gpu) do
# chars = String.graphemes(sentence)
# tokens = sentence |> to_charlist() |> Nx.tensor(type: :s32) |> Nx.new_axis(0) |> Nx.backend_copy(gpu)
# h_b = Nx.broadcast(0.0, {1, 512}) |> Nx.backend_copy(gpu)
# h_t = Nx.broadcast(0.0, {1, 512}) |> Nx.backend_copy(gpu)
# gates = extract_gates(tokens, h_b, h_t, frozen_base_p, frozen_map, top_p)
# |> Nx.backend_transfer()
# |> Nx.to_flat_list()
# IO.puts("🌊 BOUNDARY GATE RHYTHM")
# IO.puts("-----------------------")
# Enum.zip(chars, gates)
# |> Enum.each(fn {char, g} ->
# # Create a simple visual bar based on the gate value
# bar_length = trunc(g * 20)
# bar = String.duplicate("█", bar_length)
# display_char = if char == " ", do: "[space]", else: char
# IO.puts("#{String.pad_trailing(display_char, 7)} | #{Float.round(g, 3)} | #{bar}")
# end)
# end
# end{:module, DLM.Probe, <<70, 79, 82, 49, 0, 0, 31, ...>>, {:watch_the_heartbeat, 5}}
defmodule DLM.HierarchyBridge do
import Nx.Defn
# Assume we have the base state and the previous top state
defn bridge_step(h_base, h_top_prev, top_params, boundary_weights) do
# 1. ⚡ Calculate the Boundary Gate (g)
# Project 512D -> 1D, then Sigmoid to force it between 0.0 and 1.0
boundary_logit = Nx.dot(h_base, boundary_weights)
g = Nx.sigmoid(boundary_logit)
# 2. ⚡ Calculate the potential next state for the Top Level
# We feed the Base's 512D summary into the Top Level as its "input"
{_predicted_chunk, h_top_candidate} = DLM.TinySSM.forward(h_base, h_top_prev, top_params)
# 3. ⚡ The Continuous Update (Linear Interpolation)
# If g=0 (mid-word), it ignores the input and keeps h_top_prev.
# If g=1 (word end), it fully updates to h_top_candidate.
h_top_new = Nx.add(
Nx.multiply(g, h_top_candidate),
Nx.multiply(Nx.subtract(1.0, g), h_top_prev)
)
# 4. ⚡ The Soft Reset (Crucial)
# If the base layer just passed a completed concept up to the top,
# it needs to flush its own memory to start building the next concept.
h_base_flushed = Nx.multiply(h_base, Nx.subtract(1.0, g))
{h_top_new, h_base_flushed, g}
end
end{:module, DLM.HierarchyBridge, <<70, 79, 82, 49, 0, 0, 13, ...>>, true}
defmodule DLM.Main_SSM_Trainer do
import Nx.Defn
# 1. Maps discrete tokens to their perfect 512D spatial coordinates
defn get_real_coordinates(batch_tokens, frozen_embeds) do
Nx.take(frozen_embeds, Nx.flatten(batch_tokens))
|> Nx.reshape({Nx.axis_size(batch_tokens, 0), Nx.axis_size(batch_tokens, 1), 512})
|> stop_grad()
end
# 2. THE MASTER SEQUENCE LOOP (Now with Gate Tracking!)
defn compute_sequence_loss(input_tokens, target_tokens, h_base_init, h_top_init, frozen_base_p, frozen_map, top_p) do
seq_len = Nx.axis_size(input_tokens, 1)
clean_coords = get_real_coordinates(input_tokens, frozen_map)
inputs_time = Nx.transpose(clean_coords, axes: [1, 0, 2])
# ⚡ NEW: Convert discrete targets into continuous targets BEFORE the loop
target_coords = get_real_coordinates(target_tokens, frozen_map)
targets_time = Nx.transpose(target_coords, axes: [1, 0, 2])
zero_frame = Nx.broadcast(0.0, Nx.shape(h_top_init))
result =
while {i = 0, total_loss = 0.0, total_g = 0.0,
h_base = h_base_init, h_top = h_top_init,
xp1 = zero_frame, xp2 = zero_frame,
in_t = inputs_time, targ_t = targets_time,
b_p = frozen_base_p, t_p = top_p, f_map = frozen_map},
Nx.less(i, seq_len) do
x_curr = in_t[i]
{_y_base, h_base_new} = DLM.TinySSM.forward(x_curr, h_base, b_p)
# ⚡ Unpack 19 params
{a, bu, bv, cu, cv, du, dv, mu1, mv1, mu2, mv2,
gate_w, gate_b, spell_w, spell_b, valve_w, valve_b, out_w, out_b} = t_p
gate_logit = Nx.add(Nx.dot(h_base_new, gate_w), gate_b)
g = Nx.sigmoid(gate_logit)
batch_avg_g = Nx.mean(g)
{_y_top, h_top_candidate} = DLM.Main_SSM.ssm_recurrence(
h_base_new, xp1, xp2, h_top,
a, bu, bv, cu, cv, du, dv, mu1, mv1, mu2, mv2
)
h_top_new = Nx.add(Nx.multiply(g, h_top_candidate), Nx.multiply(Nx.subtract(1.0, g), h_top))
h_base_flushed = Nx.multiply(h_base_new, Nx.subtract(1.0, g))
# ⚡ 4. MULTIPLICATIVE SWISH FUSION
f_spell = Nx.add(Nx.dot(h_base_new, spell_w), spell_b)
v_syntax = DLM.Main_SSM.swish(Nx.add(Nx.dot(h_top_new, valve_w), valve_b))
h_fused = Nx.multiply(v_syntax, f_spell)
h_fused_norm = DLM.Main_SSM.rms_norm(h_fused)
# ⚡ 5. LATENT PREDICTION & CONTRASTIVE LOSS
pred_latent = Nx.add(Nx.dot(h_fused_norm, out_w), out_b)
# ⚡ Swap from purely attractive to push/pull
# step_loss = DLM.Loss.cosine_distance(pred_latent, targ_t[i])
step_loss = DLM.Loss.soft_nn_loss(pred_latent, targ_t[i], 0.1)
{i + 1, total_loss + step_loss, total_g + batch_avg_g, h_base_flushed, h_top_new, h_base_new, xp1, in_t, targ_t, b_p, t_p, f_map}
end
{Nx.divide(elem(result, 1), seq_len), elem(result, 3), elem(result, 4), Nx.divide(elem(result, 2), seq_len)}
end
# 3. GRADIENT TAPE
defn compute_grad_and_step(batch_tokens, h_base_init, h_top_init, frozen_base_p, frozen_map, top_p) do
seq_len = Nx.axis_size(batch_tokens, 1) - 1
input_tokens = Nx.slice_along_axis(batch_tokens, 0, seq_len, axis: 1)
target_tokens = Nx.slice_along_axis(batch_tokens, 1, seq_len, axis: 1)
{loss, raw_grads} = value_and_grad(top_p, fn p ->
{avg_loss, _h_base_final, _h_top_final, avg_g} =
compute_sequence_loss(input_tokens, target_tokens, h_base_init, h_top_init, frozen_base_p, frozen_map, p)
# ⚡ THE "GENTLE PRESSURE" L1 SPARSITY PENALTY
# We no longer force a specific rhythm. We just apply a tiny "energy cost" to opening the gate.
# The Swish fusion will naturally pull the gate open when it needs linguistic features.
energy_cost = Nx.tensor(0.005, type: :f32)
# Penalty = energy_cost * avg_g
gentle_penalty = Nx.multiply(energy_cost, avg_g)
# The optimizer must now balance minimizing Cosine Distance against conserving gate energy.
Nx.add(avg_loss, gentle_penalty)
end)
# Run a clean pass to get states AND the average gate value for telemetry
{_loss, final_h_base, final_h_top, avg_g} =
compute_sequence_loss(input_tokens, target_tokens, h_base_init, h_top_init, frozen_base_p, frozen_map, top_p)
{loss, raw_grads, final_h_base, final_h_top, avg_g}
end
# ⚡ 4. TELEMETRY HELPER (Unpacks all 19 Phase 3 Params)
def grad_norms(grads) do
{g_a, g_bu, g_bv, g_cu, g_cv, g_du, g_dv, g_mu1, g_mv1, g_mu2, g_mv2,
g_gw, g_gb, g_spell_w, g_spell_b, g_valve_w, g_valve_b, g_out_w, g_out_b} = grads
calc = fn g -> g |> Nx.pow(2) |> Nx.sum() |> Nx.sqrt() |> Nx.to_number() |> Float.round(4) end
[
a_diag: calc.(g_a), b_u: calc.(g_bu), b_v: calc.(g_bv),
c_u: calc.(g_cu), c_v: calc.(g_cv), d_u: calc.(g_du), d_v: calc.(g_dv),
mx_u1: calc.(g_mu1), mx_v1: calc.(g_mv1), mx_u2: calc.(g_mu2), mx_v2: calc.(g_mv2),
gate_w: calc.(g_gw), gate_b: calc.(g_gb),
spell_w: calc.(g_spell_w), spell_b: calc.(g_spell_b),
valve_w: calc.(g_valve_w), valve_b: calc.(g_valve_b),
out_w: calc.(g_out_w), out_b: calc.(g_out_b)
]
end
end{:module, DLM.Main_SSM_Trainer, <<70, 79, 82, 49, 0, 0, 51, ...>>, {:grad_norms, 1}}
defmodule DLM.CheckpointManager do
@moduledoc """
Handles resuming and saving states for the Hierarchical DLM.
"""
def get_resume_state(run_prefix) do
checkpoints = Path.wildcard("#{run_prefix}_checkpoint_step_*.bin")
if Enum.empty?(checkpoints) do
if File.exists?("#{run_prefix}_final.bin") do
IO.puts("✨ Found completed run: #{run_prefix}_final.bin")
{"#{run_prefix}_final.bin", :completed}
else
IO.puts("✨ No checkpoints found for '#{run_prefix}'. Starting FRESH.")
{:fresh, 0}
end
else
latest_file = Enum.max_by(checkpoints, fn file ->
case Regex.run(~r/_step_(\d+)\.bin$/, file) do
[_, step_str] -> String.to_integer(step_str)
_ -> -1
end
end)
[_, step_str] = Regex.run(~r/_step_(\d+)\.bin$/, latest_file)
step = String.to_integer(step_str)
IO.puts("🚀 Auto-Resuming '#{run_prefix}' from Step #{step}...")
{latest_file, step}
end
end
# ⚡ NEW: Clean, centralized saving logic
def save_checkpoint(prefix, step, p, m, v, h_base, h_top) do
IO.puts("💾 Auto-Saving Checkpoint Step #{step}...")
state = %{
params: DLM.Tree.map(p, &Nx.backend_copy(&1, Nx.BinaryBackend)),
ms: DLM.Tree.map(m, &Nx.backend_copy(&1, Nx.BinaryBackend)),
vs: DLM.Tree.map(v, &Nx.backend_copy(&1, Nx.BinaryBackend)),
step: step,
h_base: Nx.backend_copy(h_base, Nx.BinaryBackend),
h_top: Nx.backend_copy(h_top, Nx.BinaryBackend)
}
File.write!("#{prefix}_checkpoint_step_#{step}.bin", :erlang.term_to_binary(state))
end
end{:module, DLM.CheckpointManager, <<70, 79, 82, 49, 0, 0, 23, ...>>, {:save_checkpoint, 7}}
run_prefix = "v37_DLM_Phase_3_softnn"
total_steps = 100000
batch_size = 32
seq_len = 128
save_interval = 1000
# ==============================================================================
# 1. LOAD FROZEN FOUNDATIONS (Manifold + Physics Engine)
# ==============================================================================
IO.puts("🌌 Loading Phase 1 Frozen Manifold...")
%{params: phase1_p} = File.read!("phase1_shape_diffusion_COMPLETE.bin") |> :erlang.binary_to_term()
{cpu_embeds, _w1, _w2, _key} = phase1_p
frozen_map = cpu_embeds |> Nx.backend_copy(gpu) |> DLM.Loss.l2_normalize()
IO.puts("⚙️ Loading Phase 2 Base SSM (Physics Engine)...")
base_checkpoint = File.read!("v33_DLM_Phase_2_cos_checkpoint_step_13000.bin") |> :erlang.binary_to_term()
frozen_base_p = DLM.Tree.map(base_checkpoint, &Nx.backend_copy(&1, gpu))
# ==============================================================================
# 2. INITIALIZE / RESUME TOP LEVEL
# ==============================================================================
{load_path, start_step} = DLM.CheckpointManager.get_resume_state(run_prefix)
{starting_p, starting_m, starting_v, start_h_base, start_h_top} =
if load_path == :fresh do
IO.puts("✨ Igniting brand new Phase 3 Genesis weights...")
raw_p = DLM.Main_SSM_Genesis.init_params(42)
{raw_m, raw_v} = DLM.Main_SSM_Genesis.init_optimizer(raw_p)
blank_h = Nx.broadcast(0.0, {batch_size, 512}) |> Nx.backend_copy(gpu)
{raw_p, raw_m, raw_v, blank_h, blank_h}
else
ckpt = File.read!(load_path) |> :erlang.binary_to_term()
blank_h = Nx.broadcast(0.0, {batch_size, 512}) |> Nx.backend_copy(gpu)
{
ckpt.params,
ckpt.ms,
ckpt.vs,
Map.get(ckpt, :h_base, blank_h),
Map.get(ckpt, :h_top, blank_h)
}
end
# Push standard optimizer states to GPU
p = DLM.Tree.map(starting_p, &Nx.backend_copy(&1, gpu))
m = DLM.Tree.map(starting_m, &Nx.backend_copy(&1, gpu))
v = DLM.Tree.map(starting_v, &Nx.backend_copy(&1, gpu))
h_base = Nx.backend_copy(start_h_base, gpu)
h_top = Nx.backend_copy(start_h_top, gpu)
# ==============================================================================
# 3. DATA STREAM
# ==============================================================================
IO.puts("🧠 Loading Curriculum...")
dataset_2d = File.read!("scholar_full_mixed_128.bin") |> :erlang.binary_to_term()
dataset = Nx.flatten(dataset_2d)
data_stream = DLM.DataStreamer.build_infinite_stream(dataset, batch_size, seq_len)
# ==============================================================================
# 4. THE MASTER LOOP
# ==============================================================================
if start_step != :completed do
IO.puts("🚀 Entering Continuous Hierarchy Loop...")
{final_p, _m, _v, _hb, _ht} =
Enum.reduce(Enum.zip(start_step..(total_steps - 1), data_stream), {p, m, v, h_base, h_top},
fn {step, cpu_batch}, {cp, cm, cv, ch_base, ch_top} ->
gpu_batch = Nx.backend_copy(cpu_batch, gpu)
# ⚡ 1. Learning Rate Schedule
base_lr = 1.5e-4
min_lr = 1.5e-5
warmup_steps = 1000
current_lr =
if step < warmup_steps do
base_lr * (step / warmup_steps)
else
progress = min(max((step - warmup_steps) / (total_steps - warmup_steps), 0.0), 1.0)
min_lr + 0.5 * (base_lr - min_lr) * (1.0 + :math.cos(:math.pi() * progress))
end
# ⚡ 2. The Unified Gradient Tape
{loss_tensor, grads, next_h_base, next_h_top, avg_g_tensor} =
DLM.Main_SSM_Trainer.compute_grad_and_step(
gpu_batch, ch_base, ch_top, frozen_base_p, frozen_map, cp
)
loss_val = Nx.to_number(loss_tensor)
g_val = Nx.to_number(avg_g_tensor)
# ⚡ 3. Safety Check
if loss_val == :nan or loss_val > 20.0 do
IO.puts("⚠️ Step #{step}: NaN/Spike detected! Flushing Hidden States.")
clean_h = Nx.broadcast(0.0, {batch_size, 512}) |> Nx.backend_copy(gpu)
{cp, cm, cv, clean_h, clean_h} # Skip update, flush memory
else
# ⚡ 4. Optimizer Step
lr_tensor = Nx.tensor(current_lr, type: :f32) |> Nx.backend_copy(gpu)
step_tensor = Nx.tensor(step, type: :f32) |> Nx.backend_copy(gpu)
{up_p, up_m, up_v} = DLM.MuonHybrid.step(cp, grads, cm, cv, step_tensor, lr_tensor)
# ⚡ 5. Telemetry
if rem(step, 50) == 0 do
# Note: Since we use Cosine Distance now, loss is roughly between 0.0 (perfect) and 1.0 (orthogonal)
mem = :erlang.memory(:total) |> div(1024 * 1024)
# Transfer grads to CPU for inspection to avoid slowing down the GPU tape
cpu_grads = DLM.Tree.map(grads, &Nx.backend_copy(&1, Nx.BinaryBackend))
norms = DLM.Main_SSM_Trainer.grad_norms(cpu_grads)
# Sort to find the hottest gradients
sorted_norms = Enum.sort_by(norms, fn {_k, v} -> v end, :desc)
{max_name, max_norm} = hd(sorted_norms)
# Specifically check on our critical temporal and gating parameters
a_norm = Keyword.get(norms, :a_diag)
gb_norm = Keyword.get(norms, :gate_b)
valve_norm = Keyword.get(norms, :valve_w) # Watch the Swish syntax valve!
IO.puts("➡️ Step #{step} | Soft-NN Loss: #{Float.round(loss_val, 4)} | Gate Openness: #{Float.round(g_val * 100, 2)}% | LR: #{Float.round(current_lr, 6)}")
IO.puts(" 📐 GRADS -> Max: [#{max_name}: #{max_norm}] | Valve: #{valve_norm} | Gate Bias: #{gb_norm} | RAM: #{mem}MB")
end
# ⚡ 6. Auto-Save
if rem(step, save_interval) == 0 and step > 0 do
DLM.CheckpointManager.save_checkpoint(run_prefix, step, up_p, up_m, up_v, next_h_base, next_h_top)
:erlang.garbage_collect(self())
end
{up_p, up_m, up_v, next_h_base, next_h_top}
end
end)
IO.puts("🎉 THE OVERNIGHT BAKE IS COMPLETE!")
DLM.CheckpointManager.save_checkpoint("#{run_prefix}_final", total_steps, final_p, m, v, h_base, h_top)
endwarning: variable "a_norm" is unused (if the variable is not meant to be used, prefix it with an underscore)
└─ hssmp3.livemd#cell:axsluod5fswd3qcb:121
🌌 Loading Phase 1 Frozen Manifold...
14:58:09.010 [info] XLA service 0x7320d402ffc0 initialized for platform ROCM (this does not guarantee that XLA will be used). Devices:
14:58:09.012 [info] StreamExecutor [0]: AMD Radeon RX 7600, AMDGPU ISA version: gfx1100 (Driver: 6.4.43482; Runtime: 6.4.43482; Toolkit: 6.4.43482; DNN: 0.0.0)
14:58:09.012 [info] Using BFC allocator.
14:58:09.012 [info] XLA backend will use up to 6429868032 bytes on device 0 for BFCAllocator.
14:58:09.012 [info] XLA backend will use up to 2143289344 bytes on device 0 for CollectiveBFCAllocator.
⚙️ Loading Phase 2 Base SSM (Physics Engine)...
🚀 Auto-Resuming 'v34_DLM_Phase_3_Top_256_midLR' from Step 27000...
🧠 Loading Curriculum...
🚀 Entering Continuous Hierarchy Loop...
14:58:09.488 [info] Merging Dots in computation: region_0.7
14:58:09.488 [info] Merging Dots in computation: region_8.27.clone
14:58:09.488 [info] Merging Dots in computation: region_28.35.clone.clone
15:00:19.930 [info] Merging Dots in computation: region_1.2
15:00:19.930 [info] Merging Dots in computation: region_4.5
15:00:19.930 [info] Merging Dots in computation: region_7.8
15:00:19.930 [info] Merging Dots in computation: region_10.11
15:00:19.930 [info] Merging Dots in computation: region_13.14
15:00:19.930 [info] Merging Dots in computation: region_16.17
15:00:19.930 [info] Merging Dots in computation: region_19.20
15:00:19.930 [info] Merging Dots in computation: region_22.23
15:00:19.930 [info] Merging Dots in computation: region_25.26
15:00:19.930 [info] Merging Dots in computation: region_28.29
15:00:19.930 [info] Merging Dots in computation: region_31.32
15:00:19.930 [info] Merging Dots in computation: region_34.35
15:00:19.930 [info] Merging Dots in computation: region_37.38
➡️ Step 27000 | Cosine Loss: 0.5245 | Gate Openness: 53.49% | LR: 1.28e-4
📐 GRADS -> Max: [spell_w: 0.4047] | Valve: 0.0241 | Gate Bias: 0.0 | RAM: 230MB
💾 Auto-Saving Checkpoint Step 27000...
➡️ Step 27050 | Cosine Loss: 0.5182 | Gate Openness: 52.91% | LR: 1.28e-4
📐 GRADS -> Max: [spell_w: 0.3148] | Valve: 0.0252 | Gate Bias: 0.0001 | RAM: 230MB
➡️ Step 27100 | Cosine Loss: 0.5294 | Gate Openness: 54.61% | LR: 1.28e-4
📐 GRADS -> Max: [spell_w: 0.3007] | Valve: 0.023 | Gate Bias: 0.0001 | RAM: 231MB
➡️ Step 27150 | Cosine Loss: 0.5182 | Gate Openness: 52.99% | LR: 1.28e-4
📐 GRADS -> Max: [spell_w: 0.276] | Valve: 0.0212 | Gate Bias: 0.0 | RAM: 239MB
➡️ Step 27200 | Cosine Loss: 0.5169 | Gate Openness: 53.0% | LR: 1.28e-4
📐 GRADS -> Max: [spell_w: 0.3075] | Valve: 0.0263 | Gate Bias: 0.0001 | RAM: 248MB
➡️ Step 27250 | Cosine Loss: 0.5155 | Gate Openness: 55.17% | LR: 1.28e-4
📐 GRADS -> Max: [spell_w: 0.1925] | Valve: 0.0227 | Gate Bias: 0.0 | RAM: 256MB
➡️ Step 27300 | Cosine Loss: 0.5218 | Gate Openness: 54.4% | LR: 1.28e-4
📐 GRADS -> Max: [spell_w: 0.4617] | Valve: 0.0213 | Gate Bias: 0.0 | RAM: 240MB
➡️ Step 27350 | Cosine Loss: 0.5256 | Gate Openness: 54.32% | LR: 1.28e-4
📐 GRADS -> Max: [spell_w: 0.2823] | Valve: 0.0223 | Gate Bias: 0.0 | RAM: 230MB
➡️ Step 27400 | Cosine Loss: 0.508 | Gate Openness: 55.15% | LR: 1.28e-4
📐 GRADS -> Max: [spell_w: 0.2668] | Valve: 0.0239 | Gate Bias: 0.0 | RAM: 230MB
➡️ Step 27450 | Cosine Loss: 0.5072 | Gate Openness: 54.58% | LR: 1.28e-4
📐 GRADS -> Max: [spell_w: 0.291] | Valve: 0.022 | Gate Bias: 0.0001 | RAM: 249MB
➡️ Step 27500 | Cosine Loss: 0.5172 | Gate Openness: 54.14% | LR: 1.28e-4
📐 GRADS -> Max: [spell_w: 0.2232] | Valve: 0.025 | Gate Bias: 0.0001 | RAM: 249MB
➡️ Step 27550 | Cosine Loss: 0.505 | Gate Openness: 55.17% | LR: 1.27e-4
📐 GRADS -> Max: [spell_w: 0.3234] | Valve: 0.0236 | Gate Bias: 0.0 | RAM: 231MB
➡️ Step 27600 | Cosine Loss: 0.5184 | Gate Openness: 53.78% | LR: 1.27e-4
📐 GRADS -> Max: [spell_w: 0.2803] | Valve: 0.0222 | Gate Bias: 0.0 | RAM: 231MB
➡️ Step 27650 | Cosine Loss: 0.5212 | Gate Openness: 53.48% | LR: 1.27e-4
📐 GRADS -> Max: [spell_w: 0.2597] | Valve: 0.0229 | Gate Bias: 0.0 | RAM: 250MB
➡️ Step 27700 | Cosine Loss: 0.5223 | Gate Openness: 54.07% | LR: 1.27e-4
📐 GRADS -> Max: [spell_w: 0.2741] | Valve: 0.0274 | Gate Bias: 0.0001 | RAM: 244MB
➡️ Step 27750 | Cosine Loss: 0.5041 | Gate Openness: 53.85% | LR: 1.27e-4
📐 GRADS -> Max: [spell_w: 0.5327] | Valve: 0.0252 | Gate Bias: 0.0 | RAM: 233MB
➡️ Step 27800 | Cosine Loss: 0.5183 | Gate Openness: 53.72% | LR: 1.27e-4
📐 GRADS -> Max: [spell_w: 0.3677] | Valve: 0.0211 | Gate Bias: 0.0 | RAM: 234MB
➡️ Step 27850 | Cosine Loss: 0.4971 | Gate Openness: 54.1% | LR: 1.27e-4
📐 GRADS -> Max: [spell_w: 0.2551] | Valve: 0.0225 | Gate Bias: 0.0001 | RAM: 253MB
➡️ Step 27900 | Cosine Loss: 0.5334 | Gate Openness: 53.72% | LR: 1.27e-4
📐 GRADS -> Max: [spell_w: 0.2377] | Valve: 0.0218 | Gate Bias: 0.0 | RAM: 253MB
➡️ Step 27950 | Cosine Loss: 0.5106 | Gate Openness: 54.1% | LR: 1.27e-4
📐 GRADS -> Max: [spell_w: 0.2328] | Valve: 0.0214 | Gate Bias: 0.0 | RAM: 235MB
➡️ Step 28000 | Cosine Loss: 0.4978 | Gate Openness: 54.6% | LR: 1.27e-4
📐 GRADS -> Max: [spell_w: 0.3468] | Valve: 0.0223 | Gate Bias: 0.0 | RAM: 236MB
💾 Auto-Saving Checkpoint Step 28000...
➡️ Step 28050 | Cosine Loss: 0.5127 | Gate Openness: 54.83% | LR: 1.27e-4
📐 GRADS -> Max: [spell_w: 0.3043] | Valve: 0.0218 | Gate Bias: 0.0 | RAM: 248MB
➡️ Step 28100 | Cosine Loss: 0.5302 | Gate Openness: 54.4% | LR: 1.27e-4
📐 GRADS -> Max: [spell_w: 0.2488] | Valve: 0.0218 | Gate Bias: 0.0 | RAM: 249MB
➡️ Step 28150 | Cosine Loss: 0.5186 | Gate Openness: 54.22% | LR: 1.26e-4
📐 GRADS -> Max: [spell_w: 0.2551] | Valve: 0.0218 | Gate Bias: 0.0 | RAM: 249MB
➡️ Step 28200 | Cosine Loss: 0.5136 | Gate Openness: 55.07% | LR: 1.26e-4
📐 GRADS -> Max: [spell_w: 0.3177] | Valve: 0.0219 | Gate Bias: 0.0 | RAM: 249MB
➡️ Step 28250 | Cosine Loss: 0.5264 | Gate Openness: 52.88% | LR: 1.26e-4
📐 GRADS -> Max: [spell_w: 0.3352] | Valve: 0.0246 | Gate Bias: 0.0001 | RAM: 257MB
➡️ Step 28300 | Cosine Loss: 0.5163 | Gate Openness: 54.97% | LR: 1.26e-4
📐 GRADS -> Max: [spell_w: 0.3535] | Valve: 0.0269 | Gate Bias: 0.0001 | RAM: 266MB
➡️ Step 28350 | Cosine Loss: 0.5078 | Gate Openness: 52.83% | LR: 1.26e-4
📐 GRADS -> Max: [spell_w: 0.3336] | Valve: 0.0249 | Gate Bias: 0.0001 | RAM: 274MB
➡️ Step 28400 | Cosine Loss: 0.5088 | Gate Openness: 54.05% | LR: 1.26e-4
📐 GRADS -> Max: [spell_w: 0.2233] | Valve: 0.0225 | Gate Bias: 0.0 | RAM: 283MB
➡️ Step 28450 | Cosine Loss: 0.5108 | Gate Openness: 55.02% | LR: 1.26e-4
📐 GRADS -> Max: [spell_w: 0.2851] | Valve: 0.0226 | Gate Bias: 0.0 | RAM: 291MB
➡️ Step 28500 | Cosine Loss: 0.5322 | Gate Openness: 52.91% | LR: 1.26e-4
📐 GRADS -> Max: [spell_w: 0.2366] | Valve: 0.0217 | Gate Bias: 0.0001 | RAM: 300MB
➡️ Step 28550 | Cosine Loss: 0.515 | Gate Openness: 54.52% | LR: 1.26e-4
📐 GRADS -> Max: [spell_w: 0.2988] | Valve: 0.0224 | Gate Bias: 0.0 | RAM: 308MB
➡️ Step 28600 | Cosine Loss: 0.5248 | Gate Openness: 55.39% | LR: 1.26e-4
📐 GRADS -> Max: [spell_w: 0.31] | Valve: 0.0213 | Gate Bias: 0.0001 | RAM: 317MB
➡️ Step 28650 | Cosine Loss: 0.5247 | Gate Openness: 54.12% | LR: 1.26e-4
📐 GRADS -> Max: [spell_w: 0.3149] | Valve: 0.023 | Gate Bias: 0.0 | RAM: 325MB
➡️ Step 28700 | Cosine Loss: 0.4996 | Gate Openness: 53.41% | LR: 1.26e-4
📐 GRADS -> Max: [spell_w: 0.3816] | Valve: 0.0228 | Gate Bias: 0.0001 | RAM: 334MB
➡️ Step 28750 | Cosine Loss: 0.5122 | Gate Openness: 54.38% | LR: 1.25e-4
📐 GRADS -> Max: [spell_w: 0.3702] | Valve: 0.0248 | Gate Bias: 0.0001 | RAM: 342MB
➡️ Step 28800 | Cosine Loss: 0.5112 | Gate Openness: 53.08% | LR: 1.25e-4
📐 GRADS -> Max: [spell_w: 0.3293] | Valve: 0.0236 | Gate Bias: 0.0 | RAM: 351MB
➡️ Step 28850 | Cosine Loss: 0.5226 | Gate Openness: 51.22% | LR: 1.25e-4
📐 GRADS -> Max: [spell_w: 0.4798] | Valve: 0.0246 | Gate Bias: 0.0002 | RAM: 360MB
➡️ Step 28900 | Cosine Loss: 0.5089 | Gate Openness: 54.72% | LR: 1.25e-4
📐 GRADS -> Max: [spell_w: 0.3496] | Valve: 0.023 | Gate Bias: 0.0 | RAM: 368MB
➡️ Step 28950 | Cosine Loss: 0.5309 | Gate Openness: 54.01% | LR: 1.25e-4
📐 GRADS -> Max: [spell_w: 0.3386] | Valve: 0.0219 | Gate Bias: 0.0 | RAM: 376MB
➡️ Step 29000 | Cosine Loss: 0.5226 | Gate Openness: 54.52% | LR: 1.25e-4
📐 GRADS -> Max: [spell_w: 0.3742] | Valve: 0.0225 | Gate Bias: 0.0 | RAM: 386MB
💾 Auto-Saving Checkpoint Step 29000...
➡️ Step 29050 | Cosine Loss: 0.51 | Gate Openness: 53.56% | LR: 1.25e-4
📐 GRADS -> Max: [spell_w: 0.1987] | Valve: 0.0205 | Gate Bias: 0.0001 | RAM: 199MB
➡️ Step 29100 | Cosine Loss: 0.5143 | Gate Openness: 53.91% | LR: 1.25e-4
📐 GRADS -> Max: [spell_w: 0.2351] | Valve: 0.0234 | Gate Bias: 0.0 | RAM: 199MB
➡️ Step 29150 | Cosine Loss: 0.5315 | Gate Openness: 53.6% | LR: 1.25e-4
📐 GRADS -> Max: [spell_w: 0.2556] | Valve: 0.022 | Gate Bias: 0.0001 | RAM: 199MB
➡️ Step 29200 | Cosine Loss: 0.531 | Gate Openness: 52.13% | LR: 1.25e-4
📐 GRADS -> Max: [spell_w: 0.2524] | Valve: 0.0235 | Gate Bias: 0.0001 | RAM: 208MB
➡️ Step 29250 | Cosine Loss: 0.5299 | Gate Openness: 52.33% | LR: 1.25e-4
📐 GRADS -> Max: [spell_w: 0.2124] | Valve: 0.0229 | Gate Bias: 0.0002 | RAM: 216MB
➡️ Step 29300 | Cosine Loss: 0.5002 | Gate Openness: 54.6% | LR: 1.25e-4
📐 GRADS -> Max: [spell_w: 0.3087] | Valve: 0.0235 | Gate Bias: 0.0001 | RAM: 225MB
➡️ Step 29350 | Cosine Loss: 0.5194 | Gate Openness: 53.63% | LR: 1.24e-4
📐 GRADS -> Max: [spell_w: 0.3127] | Valve: 0.0232 | Gate Bias: 0.0 | RAM: 233MB
➡️ Step 29400 | Cosine Loss: 0.5108 | Gate Openness: 54.18% | LR: 1.24e-4
📐 GRADS -> Max: [spell_w: 0.3603] | Valve: 0.0269 | Gate Bias: 0.0001 | RAM: 242MB
➡️ Step 29450 | Cosine Loss: 0.5064 | Gate Openness: 55.63% | LR: 1.24e-4
📐 GRADS -> Max: [spell_w: 0.2499] | Valve: 0.0212 | Gate Bias: 0.0 | RAM: 250MB
➡️ Step 29500 | Cosine Loss: 0.52 | Gate Openness: 52.26% | LR: 1.24e-4
📐 GRADS -> Max: [spell_w: 0.2451] | Valve: 0.0221 | Gate Bias: 0.0001 | RAM: 259MB
➡️ Step 29550 | Cosine Loss: 0.5155 | Gate Openness: 54.15% | LR: 1.24e-4
📐 GRADS -> Max: [spell_w: 0.25] | Valve: 0.0221 | Gate Bias: 0.0 | RAM: 267MB
➡️ Step 29600 | Cosine Loss: 0.5098 | Gate Openness: 52.87% | LR: 1.24e-4
📐 GRADS -> Max: [spell_w: 0.3066] | Valve: 0.0228 | Gate Bias: 0.0001 | RAM: 275MB
➡️ Step 29650 | Cosine Loss: 0.5088 | Gate Openness: 54.24% | LR: 1.24e-4
📐 GRADS -> Max: [spell_w: 0.2974] | Valve: 0.0245 | Gate Bias: 0.0 | RAM: 284MB
➡️ Step 29700 | Cosine Loss: 0.5021 | Gate Openness: 54.66% | LR: 1.24e-4
📐 GRADS -> Max: [spell_w: 0.394] | Valve: 0.0251 | Gate Bias: 0.0 | RAM: 293MB
➡️ Step 29750 | Cosine Loss: 0.5089 | Gate Openness: 53.55% | LR: 1.24e-4
📐 GRADS -> Max: [spell_w: 0.2378] | Valve: 0.0221 | Gate Bias: 0.0 | RAM: 301MB
➡️ Step 29800 | Cosine Loss: 0.5131 | Gate Openness: 54.63% | LR: 1.24e-4
📐 GRADS -> Max: [spell_w: 0.2927] | Valve: 0.023 | Gate Bias: 0.0 | RAM: 310MB
➡️ Step 29850 | Cosine Loss: 0.5158 | Gate Openness: 53.85% | LR: 1.24e-4
📐 GRADS -> Max: [spell_w: 0.3016] | Valve: 0.0225 | Gate Bias: 0.0001 | RAM: 318MB
➡️ Step 29900 | Cosine Loss: 0.5268 | Gate Openness: 52.52% | LR: 1.24e-4
📐 GRADS -> Max: [spell_w: 0.2998] | Valve: 0.0238 | Gate Bias: 0.0001 | RAM: 326MB
➡️ Step 29950 | Cosine Loss: 0.5207 | Gate Openness: 53.4% | LR: 1.23e-4
📐 GRADS -> Max: [spell_w: 0.2368] | Valve: 0.0212 | Gate Bias: 0.0001 | RAM: 335MB
➡️ Step 30000 | Cosine Loss: 0.5096 | Gate Openness: 55.06% | LR: 1.23e-4
📐 GRADS -> Max: [spell_w: 0.2396] | Valve: 0.0251 | Gate Bias: 0.0 | RAM: 343MB
💾 Auto-Saving Checkpoint Step 30000...
➡️ Step 30050 | Cosine Loss: 0.5075 | Gate Openness: 54.02% | LR: 1.23e-4
📐 GRADS -> Max: [spell_w: 0.3743] | Valve: 0.0251 | Gate Bias: 0.0001 | RAM: 203MB
➡️ Step 30100 | Cosine Loss: 0.5252 | Gate Openness: 53.63% | LR: 1.23e-4
📐 GRADS -> Max: [spell_w: 0.2405] | Valve: 0.0227 | Gate Bias: 0.0001 | RAM: 203MB
➡️ Step 30150 | Cosine Loss: 0.5299 | Gate Openness: 52.86% | LR: 1.23e-4
📐 GRADS -> Max: [spell_w: 0.4157] | Valve: 0.0245 | Gate Bias: 0.0 | RAM: 212MB
➡️ Step 30200 | Cosine Loss: 0.5139 | Gate Openness: 53.92% | LR: 1.23e-4
📐 GRADS -> Max: [spell_w: 0.2107] | Valve: 0.0221 | Gate Bias: 0.0 | RAM: 220MB
➡️ Step 30250 | Cosine Loss: 0.5242 | Gate Openness: 54.76% | LR: 1.23e-4
📐 GRADS -> Max: [spell_w: 0.3045] | Valve: 0.0219 | Gate Bias: 0.0 | RAM: 228MB
➡️ Step 30300 | Cosine Loss: 0.5181 | Gate Openness: 55.53% | LR: 1.23e-4
📐 GRADS -> Max: [spell_w: 0.345] | Valve: 0.0233 | Gate Bias: 0.0001 | RAM: 237MB
➡️ Step 30350 | Cosine Loss: 0.5291 | Gate Openness: 53.76% | LR: 1.23e-4
📐 GRADS -> Max: [spell_w: 0.2627] | Valve: 0.0251 | Gate Bias: 0.0 | RAM: 245MB
➡️ Step 30400 | Cosine Loss: 0.5244 | Gate Openness: 54.97% | LR: 1.23e-4
📐 GRADS -> Max: [spell_w: 0.2903] | Valve: 0.0226 | Gate Bias: 0.0001 | RAM: 254MB
➡️ Step 30450 | Cosine Loss: 0.5288 | Gate Openness: 53.07% | LR: 1.23e-4
📐 GRADS -> Max: [spell_w: 0.2763] | Valve: 0.0226 | Gate Bias: 0.0001 | RAM: 262MB
➡️ Step 30500 | Cosine Loss: 0.5162 | Gate Openness: 52.76% | LR: 1.23e-4
📐 GRADS -> Max: [spell_w: 0.2008] | Valve: 0.022 | Gate Bias: 0.0001 | RAM: 203MB
➡️ Step 30550 | Cosine Loss: 0.5233 | Gate Openness: 54.74% | LR: 1.22e-4
📐 GRADS -> Max: [spell_w: 0.2214] | Valve: 0.0248 | Gate Bias: 0.0 | RAM: 219MB
➡️ Step 30600 | Cosine Loss: 0.5178 | Gate Openness: 55.15% | LR: 1.22e-4
📐 GRADS -> Max: [spell_w: 0.2858] | Valve: 0.0262 | Gate Bias: 0.0 | RAM: 208MB
➡️ Step 30650 | Cosine Loss: 0.5237 | Gate Openness: 53.23% | LR: 1.22e-4
📐 GRADS -> Max: [spell_w: 0.2803] | Valve: 0.0231 | Gate Bias: 0.0 | RAM: 208MB
➡️ Step 30700 | Cosine Loss: 0.5075 | Gate Openness: 54.91% | LR: 1.22e-4
📐 GRADS -> Max: [spell_w: 0.4332] | Valve: 0.0248 | Gate Bias: 0.0 | RAM: 227MB
➡️ Step 30750 | Cosine Loss: 0.5003 | Gate Openness: 53.83% | LR: 1.22e-4
📐 GRADS -> Max: [spell_w: 0.247] | Valve: 0.0233 | Gate Bias: 0.0 | RAM: 228MB
➡️ Step 30800 | Cosine Loss: 0.5155 | Gate Openness: 54.19% | LR: 1.22e-4
📐 GRADS -> Max: [spell_w: 0.2458] | Valve: 0.0236 | Gate Bias: 0.0001 | RAM: 210MB
➡️ Step 30850 | Cosine Loss: 0.5109 | Gate Openness: 54.19% | LR: 1.22e-4
📐 GRADS -> Max: [spell_w: 0.3589] | Valve: 0.0238 | Gate Bias: 0.0 | RAM: 210MB
➡️ Step 30900 | Cosine Loss: 0.5389 | Gate Openness: 53.97% | LR: 1.22e-4
📐 GRADS -> Max: [spell_w: 0.2994] | Valve: 0.0231 | Gate Bias: 0.0002 | RAM: 223MB
➡️ Step 30950 | Cosine Loss: 0.5131 | Gate Openness: 54.12% | LR: 1.22e-4
📐 GRADS -> Max: [spell_w: 0.298] | Valve: 0.0224 | Gate Bias: 0.0001 | RAM: 211MB
➡️ Step 31000 | Cosine Loss: 0.5247 | Gate Openness: 53.69% | LR: 1.22e-4
📐 GRADS -> Max: [spell_w: 0.2951] | Valve: 0.0257 | Gate Bias: 0.0 | RAM: 212MB
💾 Auto-Saving Checkpoint Step 31000...
➡️ Step 31050 | Cosine Loss: 0.4953 | Gate Openness: 53.62% | LR: 1.22e-4
📐 GRADS -> Max: [spell_w: 0.2286] | Valve: 0.0262 | Gate Bias: 0.0001 | RAM: 224MB
➡️ Step 31100 | Cosine Loss: 0.5187 | Gate Openness: 53.72% | LR: 1.21e-4
📐 GRADS -> Max: [spell_w: 0.6145] | Valve: 0.0253 | Gate Bias: 0.0001 | RAM: 224MB
➡️ Step 31150 | Cosine Loss: 0.5146 | Gate Openness: 52.73% | LR: 1.21e-4
📐 GRADS -> Max: [spell_w: 0.2059] | Valve: 0.022 | Gate Bias: 0.0 | RAM: 225MB
➡️ Step 31200 | Cosine Loss: 0.5047 | Gate Openness: 55.18% | LR: 1.21e-4
📐 GRADS -> Max: [spell_w: 0.2345] | Valve: 0.0223 | Gate Bias: 0.0 | RAM: 225MB
➡️ Step 31250 | Cosine Loss: 0.5025 | Gate Openness: 53.67% | LR: 1.21e-4
📐 GRADS -> Max: [spell_w: 0.4153] | Valve: 0.0251 | Gate Bias: 0.0 | RAM: 233MB
➡️ Step 31300 | Cosine Loss: 0.5217 | Gate Openness: 53.33% | LR: 1.21e-4
📐 GRADS -> Max: [spell_w: 0.312] | Valve: 0.0224 | Gate Bias: 0.0 | RAM: 242MB
➡️ Step 31350 | Cosine Loss: 0.4958 | Gate Openness: 54.66% | LR: 1.21e-4
📐 GRADS -> Max: [spell_w: 0.3892] | Valve: 0.0258 | Gate Bias: 0.0 | RAM: 250MB
➡️ Step 31400 | Cosine Loss: 0.5224 | Gate Openness: 53.93% | LR: 1.21e-4
📐 GRADS -> Max: [spell_w: 0.3001] | Valve: 0.0243 | Gate Bias: 0.0001 | RAM: 259MB
➡️ Step 31450 | Cosine Loss: 0.5217 | Gate Openness: 54.36% | LR: 1.21e-4
📐 GRADS -> Max: [spell_w: 0.2301] | Valve: 0.024 | Gate Bias: 0.0001 | RAM: 267MB
➡️ Step 31500 | Cosine Loss: 0.4943 | Gate Openness: 54.7% | LR: 1.21e-4
📐 GRADS -> Max: [spell_w: 0.2082] | Valve: 0.0228 | Gate Bias: 0.0 | RAM: 276MB
➡️ Step 31550 | Cosine Loss: 0.5096 | Gate Openness: 54.62% | LR: 1.21e-4
📐 GRADS -> Max: [spell_w: 0.4255] | Valve: 0.0239 | Gate Bias: 0.0 | RAM: 284MB
➡️ Step 31600 | Cosine Loss: 0.5083 | Gate Openness: 53.48% | LR: 1.21e-4
📐 GRADS -> Max: [spell_w: 0.2523] | Valve: 0.0238 | Gate Bias: 0.0 | RAM: 293MB
➡️ Step 31650 | Cosine Loss: 0.5471 | Gate Openness: 52.22% | LR: 1.21e-4
📐 GRADS -> Max: [spell_w: 0.7386] | Valve: 0.0463 | Gate Bias: 0.0001 | RAM: 301MB
➡️ Step 31700 | Cosine Loss: 0.5123 | Gate Openness: 54.55% | LR: 1.2e-4
📐 GRADS -> Max: [spell_w: 0.3778] | Valve: 0.026 | Gate Bias: 0.0001 | RAM: 309MB
➡️ Step 31750 | Cosine Loss: 0.5314 | Gate Openness: 52.99% | LR: 1.2e-4
📐 GRADS -> Max: [spell_w: 0.2671] | Valve: 0.0229 | Gate Bias: 0.0 | RAM: 318MB
➡️ Step 31800 | Cosine Loss: 0.527 | Gate Openness: 53.73% | LR: 1.2e-4
📐 GRADS -> Max: [spell_w: 0.3512] | Valve: 0.0251 | Gate Bias: 0.0 | RAM: 328MB
➡️ Step 31850 | Cosine Loss: 0.5245 | Gate Openness: 55.01% | LR: 1.2e-4
📐 GRADS -> Max: [spell_w: 0.4611] | Valve: 0.0243 | Gate Bias: 0.0001 | RAM: 336MB
➡️ Step 31900 | Cosine Loss: 0.5201 | Gate Openness: 53.63% | LR: 1.2e-4
📐 GRADS -> Max: [spell_w: 0.2768] | Valve: 0.0261 | Gate Bias: 0.0 | RAM: 344MB
➡️ Step 31950 | Cosine Loss: 0.5239 | Gate Openness: 54.09% | LR: 1.2e-4
📐 GRADS -> Max: [spell_w: 0.3984] | Valve: 0.0239 | Gate Bias: 0.0 | RAM: 353MB
➡️ Step 32000 | Cosine Loss: 0.5308 | Gate Openness: 54.26% | LR: 1.2e-4
📐 GRADS -> Max: [spell_w: 0.2697] | Valve: 0.0241 | Gate Bias: 0.0 | RAM: 361MB
💾 Auto-Saving Checkpoint Step 32000...
➡️ Step 32050 | Cosine Loss: 0.5215 | Gate Openness: 54.34% | LR: 1.2e-4
📐 GRADS -> Max: [spell_w: 0.4252] | Valve: 0.0245 | Gate Bias: 0.0001 | RAM: 225MB
➡️ Step 32100 | Cosine Loss: 0.5276 | Gate Openness: 52.32% | LR: 1.2e-4
📐 GRADS -> Max: [spell_w: 0.5711] | Valve: 0.0252 | Gate Bias: 0.0 | RAM: 226MB
➡️ Step 32150 | Cosine Loss: 0.5278 | Gate Openness: 55.95% | LR: 1.2e-4
📐 GRADS -> Max: [spell_w: 0.2698] | Valve: 0.0251 | Gate Bias: 0.0001 | RAM: 234MB
➡️ Step 32200 | Cosine Loss: 0.5042 | Gate Openness: 53.24% | LR: 1.2e-4
📐 GRADS -> Max: [spell_w: 0.4644] | Valve: 0.0298 | Gate Bias: 0.0001 | RAM: 246MB
➡️ Step 32250 | Cosine Loss: 0.5157 | Gate Openness: 55.82% | LR: 1.19e-4
📐 GRADS -> Max: [spell_w: 0.2443] | Valve: 0.0233 | Gate Bias: 0.0 | RAM: 255MB
➡️ Step 32300 | Cosine Loss: 0.5276 | Gate Openness: 53.05% | LR: 1.19e-4
📐 GRADS -> Max: [spell_w: 0.2662] | Valve: 0.0232 | Gate Bias: 0.0 | RAM: 263MB
➡️ Step 32350 | Cosine Loss: 0.5145 | Gate Openness: 54.01% | LR: 1.19e-4
📐 GRADS -> Max: [spell_w: 0.4754] | Valve: 0.0247 | Gate Bias: 0.0 | RAM: 272MB
➡️ Step 32400 | Cosine Loss: 0.5125 | Gate Openness: 54.61% | LR: 1.19e-4
📐 GRADS -> Max: [spell_w: 0.7697] | Valve: 0.0271 | Gate Bias: 0.0 | RAM: 280MB
➡️ Step 32450 | Cosine Loss: 0.522 | Gate Openness: 54.88% | LR: 1.19e-4
📐 GRADS -> Max: [spell_w: 0.3686] | Valve: 0.026 | Gate Bias: 0.0001 | RAM: 224MB
➡️ Step 32500 | Cosine Loss: 0.5165 | Gate Openness: 54.34% | LR: 1.19e-4
📐 GRADS -> Max: [spell_w: 0.2627] | Valve: 0.0253 | Gate Bias: 0.0001 | RAM: 224MB
➡️ Step 32550 | Cosine Loss: 0.5075 | Gate Openness: 55.99% | LR: 1.19e-4
📐 GRADS -> Max: [spell_w: 0.3201] | Valve: 0.0245 | Gate Bias: 0.0001 | RAM: 225MB
➡️ Step 32600 | Cosine Loss: 0.5189 | Gate Openness: 53.87% | LR: 1.19e-4
📐 GRADS -> Max: [spell_w: 0.3482] | Valve: 0.0238 | Gate Bias: 0.0 | RAM: 225MB
➡️ Step 32650 | Cosine Loss: 0.507 | Gate Openness: 53.77% | LR: 1.19e-4
📐 GRADS -> Max: [spell_w: 0.2367] | Valve: 0.0229 | Gate Bias: 0.0 | RAM: 225MB
➡️ Step 32700 | Cosine Loss: 0.5005 | Gate Openness: 53.22% | LR: 1.19e-4
📐 GRADS -> Max: [spell_w: 0.3354] | Valve: 0.0261 | Gate Bias: 0.0001 | RAM: 226MB
➡️ Step 32750 | Cosine Loss: 0.5003 | Gate Openness: 54.9% | LR: 1.19e-4
📐 GRADS -> Max: [spell_w: 0.3323] | Valve: 0.0268 | Gate Bias: 0.0001 | RAM: 226MB
➡️ Step 32800 | Cosine Loss: 0.5032 | Gate Openness: 54.82% | LR: 1.18e-4
📐 GRADS -> Max: [spell_w: 0.2336] | Valve: 0.0251 | Gate Bias: 0.0001 | RAM: 227MB
➡️ Step 32850 | Cosine Loss: 0.5133 | Gate Openness: 53.15% | LR: 1.18e-4
📐 GRADS -> Max: [spell_w: 0.1942] | Valve: 0.0246 | Gate Bias: 0.0 | RAM: 227MB
➡️ Step 32900 | Cosine Loss: 0.5261 | Gate Openness: 54.59% | LR: 1.18e-4
📐 GRADS -> Max: [spell_w: 0.4725] | Valve: 0.0243 | Gate Bias: 0.0001 | RAM: 228MB
➡️ Step 32950 | Cosine Loss: 0.5173 | Gate Openness: 53.61% | LR: 1.18e-4
📐 GRADS -> Max: [spell_w: 0.3252] | Valve: 0.0254 | Gate Bias: 0.0 | RAM: 228MB
➡️ Step 33000 | Cosine Loss: 0.519 | Gate Openness: 54.23% | LR: 1.18e-4
📐 GRADS -> Max: [spell_w: 0.5215] | Valve: 0.0251 | Gate Bias: 0.0 | RAM: 229MB
💾 Auto-Saving Checkpoint Step 33000...
➡️ Step 33050 | Cosine Loss: 0.5153 | Gate Openness: 54.01% | LR: 1.18e-4
📐 GRADS -> Max: [spell_w: 0.2513] | Valve: 0.0246 | Gate Bias: 0.0001 | RAM: 237MB
➡️ Step 33100 | Cosine Loss: 0.519 | Gate Openness: 54.62% | LR: 1.18e-4
📐 GRADS -> Max: [spell_w: 0.3115] | Valve: 0.0236 | Gate Bias: 0.0001 | RAM: 238MB
➡️ Step 33150 | Cosine Loss: 0.5058 | Gate Openness: 52.85% | LR: 1.18e-4
📐 GRADS -> Max: [spell_w: 0.4407] | Valve: 0.0251 | Gate Bias: 0.0 | RAM: 246MB
➡️ Step 33200 | Cosine Loss: 0.5201 | Gate Openness: 54.85% | LR: 1.18e-4
📐 GRADS -> Max: [spell_w: 0.3544] | Valve: 0.0263 | Gate Bias: 0.0001 | RAM: 255MB
➡️ Step 33250 | Cosine Loss: 0.508 | Gate Openness: 53.52% | LR: 1.18e-4
📐 GRADS -> Max: [spell_w: 0.3135] | Valve: 0.0232 | Gate Bias: 0.0 | RAM: 263MB
➡️ Step 33300 | Cosine Loss: 0.4998 | Gate Openness: 56.07% | LR: 1.18e-4
📐 GRADS -> Max: [spell_w: 0.3748] | Valve: 0.0235 | Gate Bias: 0.0 | RAM: 272MB
➡️ Step 33350 | Cosine Loss: 0.503 | Gate Openness: 53.48% | LR: 1.17e-4
📐 GRADS -> Max: [spell_w: 0.2723] | Valve: 0.0276 | Gate Bias: 0.0001 | RAM: 280MB
➡️ Step 33400 | Cosine Loss: 0.5276 | Gate Openness: 53.47% | LR: 1.17e-4
📐 GRADS -> Max: [spell_w: 0.2936] | Valve: 0.0238 | Gate Bias: 0.0 | RAM: 289MB
➡️ Step 33450 | Cosine Loss: 0.5076 | Gate Openness: 54.11% | LR: 1.17e-4
📐 GRADS -> Max: [spell_w: 0.2399] | Valve: 0.0236 | Gate Bias: 0.0 | RAM: 297MB
➡️ Step 33500 | Cosine Loss: 0.5121 | Gate Openness: 54.36% | LR: 1.17e-4
📐 GRADS -> Max: [spell_w: 0.2761] | Valve: 0.0247 | Gate Bias: 0.0001 | RAM: 305MB
➡️ Step 33550 | Cosine Loss: 0.5077 | Gate Openness: 54.8% | LR: 1.17e-4
📐 GRADS -> Max: [spell_w: 0.337] | Valve: 0.024 | Gate Bias: 0.0001 | RAM: 314MB
➡️ Step 33600 | Cosine Loss: 0.5094 | Gate Openness: 53.88% | LR: 1.17e-4
📐 GRADS -> Max: [spell_w: 0.2443] | Valve: 0.0246 | Gate Bias: 0.0 | RAM: 322MB
➡️ Step 33650 | Cosine Loss: 0.5086 | Gate Openness: 54.89% | LR: 1.17e-4
📐 GRADS -> Max: [spell_w: 0.2831] | Valve: 0.025 | Gate Bias: 0.0001 | RAM: 331MB
➡️ Step 33700 | Cosine Loss: 0.494 | Gate Openness: 55.47% | LR: 1.17e-4
📐 GRADS -> Max: [spell_w: 0.5495] | Valve: 0.0267 | Gate Bias: 0.0 | RAM: 339MB
➡️ Step 33750 | Cosine Loss: 0.5186 | Gate Openness: 53.05% | LR: 1.17e-4
📐 GRADS -> Max: [spell_w: 0.3156] | Valve: 0.0259 | Gate Bias: 0.0 | RAM: 348MB
➡️ Step 33800 | Cosine Loss: 0.5001 | Gate Openness: 53.83% | LR: 1.17e-4
📐 GRADS -> Max: [spell_w: 0.3515] | Valve: 0.0289 | Gate Bias: 0.0 | RAM: 356MB
➡️ Step 33850 | Cosine Loss: 0.5141 | Gate Openness: 54.56% | LR: 1.17e-4
📐 GRADS -> Max: [spell_w: 0.3255] | Valve: 0.0259 | Gate Bias: 0.0 | RAM: 365MB
➡️ Step 33900 | Cosine Loss: 0.512 | Gate Openness: 53.76% | LR: 1.16e-4
📐 GRADS -> Max: [spell_w: 0.301] | Valve: 0.026 | Gate Bias: 0.0 | RAM: 373MB
➡️ Step 33950 | Cosine Loss: 0.5435 | Gate Openness: 55.32% | LR: 1.16e-4
📐 GRADS -> Max: [spell_w: 0.2562] | Valve: 0.0259 | Gate Bias: 0.0 | RAM: 382MB
➡️ Step 34000 | Cosine Loss: 0.5108 | Gate Openness: 53.74% | LR: 1.16e-4
📐 GRADS -> Max: [spell_w: 0.2846] | Valve: 0.0242 | Gate Bias: 0.0 | RAM: 390MB
💾 Auto-Saving Checkpoint Step 34000...
➡️ Step 34050 | Cosine Loss: 0.5093 | Gate Openness: 54.22% | LR: 1.16e-4
📐 GRADS -> Max: [spell_w: 0.2494] | Valve: 0.0237 | Gate Bias: 0.0 | RAM: 246MB
➡️ Step 34100 | Cosine Loss: 0.5083 | Gate Openness: 54.08% | LR: 1.16e-4
📐 GRADS -> Max: [spell_w: 0.4403] | Valve: 0.0248 | Gate Bias: 0.0001 | RAM: 246MB
➡️ Step 34150 | Cosine Loss: 0.4943 | Gate Openness: 55.53% | LR: 1.16e-4
📐 GRADS -> Max: [spell_w: 0.457] | Valve: 0.0283 | Gate Bias: 0.0001 | RAM: 255MB
➡️ Step 34200 | Cosine Loss: 0.5143 | Gate Openness: 54.46% | LR: 1.16e-4
📐 GRADS -> Max: [spell_w: 0.3412] | Valve: 0.0241 | Gate Bias: 0.0 | RAM: 263MB
➡️ Step 34250 | Cosine Loss: 0.5167 | Gate Openness: 55.09% | LR: 1.16e-4
📐 GRADS -> Max: [spell_w: 0.2819] | Valve: 0.0269 | Gate Bias: 0.0 | RAM: 272MB
➡️ Step 34300 | Cosine Loss: 0.5149 | Gate Openness: 54.47% | LR: 1.16e-4
📐 GRADS -> Max: [spell_w: 0.4307] | Valve: 0.0252 | Gate Bias: 0.0001 | RAM: 280MB
➡️ Step 34350 | Cosine Loss: 0.4989 | Gate Openness: 54.56% | LR: 1.16e-4
📐 GRADS -> Max: [spell_w: 0.2705] | Valve: 0.0233 | Gate Bias: 0.0 | RAM: 289MB
➡️ Step 34400 | Cosine Loss: 0.5227 | Gate Openness: 55.7% | LR: 1.16e-4
📐 GRADS -> Max: [spell_w: 0.5889] | Valve: 0.0243 | Gate Bias: 0.0001 | RAM: 297MB
➡️ Step 34450 | Cosine Loss: 0.4952 | Gate Openness: 53.88% | LR: 1.15e-4
📐 GRADS -> Max: [spell_w: 0.285] | Valve: 0.026 | Gate Bias: 0.0 | RAM: 241MB
➡️ Step 34500 | Cosine Loss: 0.4992 | Gate Openness: 54.97% | LR: 1.15e-4
📐 GRADS -> Max: [spell_w: 0.2465] | Valve: 0.0261 | Gate Bias: 0.0001 | RAM: 241MB
➡️ Step 34550 | Cosine Loss: 0.5151 | Gate Openness: 54.24% | LR: 1.15e-4
📐 GRADS -> Max: [spell_w: 0.3054] | Valve: 0.0274 | Gate Bias: 0.0001 | RAM: 242MB
➡️ Step 34600 | Cosine Loss: 0.4944 | Gate Openness: 54.74% | LR: 1.15e-4
📐 GRADS -> Max: [spell_w: 0.3247] | Valve: 0.0271 | Gate Bias: 0.0001 | RAM: 242MB
➡️ Step 34650 | Cosine Loss: 0.5066 | Gate Openness: 54.26% | LR: 1.15e-4
📐 GRADS -> Max: [spell_w: 0.2901] | Valve: 0.025 | Gate Bias: 0.0001 | RAM: 243MB
➡️ Step 34700 | Cosine Loss: 0.5131 | Gate Openness: 54.96% | LR: 1.15e-4
📐 GRADS -> Max: [spell_w: 0.2294] | Valve: 0.0238 | Gate Bias: 0.0 | RAM: 243MB
➡️ Step 34750 | Cosine Loss: 0.5119 | Gate Openness: 53.51% | LR: 1.15e-4
📐 GRADS -> Max: [spell_w: 0.2381] | Valve: 0.0255 | Gate Bias: 0.0001 | RAM: 244MB
➡️ Step 34800 | Cosine Loss: 0.5195 | Gate Openness: 53.83% | LR: 1.15e-4
📐 GRADS -> Max: [spell_w: 0.2804] | Valve: 0.0251 | Gate Bias: 0.0001 | RAM: 244MB
➡️ Step 34850 | Cosine Loss: 0.491 | Gate Openness: 54.72% | LR: 1.15e-4
📐 GRADS -> Max: [spell_w: 0.2649] | Valve: 0.0244 | Gate Bias: 0.0001 | RAM: 244MB
➡️ Step 34900 | Cosine Loss: 0.5166 | Gate Openness: 54.32% | LR: 1.15e-4
📐 GRADS -> Max: [spell_w: 0.3305] | Valve: 0.0236 | Gate Bias: 0.0001 | RAM: 245MB
➡️ Step 34950 | Cosine Loss: 0.5043 | Gate Openness: 53.9% | LR: 1.14e-4
📐 GRADS -> Max: [spell_w: 0.3088] | Valve: 0.0255 | Gate Bias: 0.0 | RAM: 245MB
➡️ Step 35000 | Cosine Loss: 0.5194 | Gate Openness: 56.14% | LR: 1.14e-4
📐 GRADS -> Max: [spell_w: 0.253] | Valve: 0.0237 | Gate Bias: 0.0 | RAM: 246MB
💾 Auto-Saving Checkpoint Step 35000...
➡️ Step 35050 | Cosine Loss: 0.5218 | Gate Openness: 54.43% | LR: 1.14e-4
📐 GRADS -> Max: [spell_w: 0.332] | Valve: 0.0242 | Gate Bias: 0.0001 | RAM: 246MB
➡️ Step 35100 | Cosine Loss: 0.5087 | Gate Openness: 53.0% | LR: 1.14e-4
📐 GRADS -> Max: [spell_w: 0.2165] | Valve: 0.0232 | Gate Bias: 0.0001 | RAM: 246MB
➡️ Step 35150 | Cosine Loss: 0.5202 | Gate Openness: 54.6% | LR: 1.14e-4
📐 GRADS -> Max: [spell_w: 0.4291] | Valve: 0.0277 | Gate Bias: 0.0001 | RAM: 264MB
➡️ Step 35200 | Cosine Loss: 0.5095 | Gate Openness: 52.7% | LR: 1.14e-4
📐 GRADS -> Max: [spell_w: 0.4989] | Valve: 0.0268 | Gate Bias: 0.0001 | RAM: 264MB
➡️ Step 35250 | Cosine Loss: 0.5077 | Gate Openness: 53.35% | LR: 1.14e-4
📐 GRADS -> Max: [spell_w: 0.6926] | Valve: 0.0285 | Gate Bias: 0.0 | RAM: 264MB
➡️ Step 35300 | Cosine Loss: 0.5029 | Gate Openness: 55.2% | LR: 1.14e-4
📐 GRADS -> Max: [spell_w: 0.2776] | Valve: 0.0265 | Gate Bias: 0.0 | RAM: 265MB
➡️ Step 35350 | Cosine Loss: 0.5095 | Gate Openness: 53.74% | LR: 1.14e-4
📐 GRADS -> Max: [spell_w: 0.3353] | Valve: 0.0271 | Gate Bias: 0.0 | RAM: 265MB
➡️ Step 35400 | Cosine Loss: 0.5133 | Gate Openness: 54.55% | LR: 1.14e-4
📐 GRADS -> Max: [spell_w: 0.4828] | Valve: 0.0278 | Gate Bias: 0.0001 | RAM: 249MB
➡️ Step 35450 | Cosine Loss: 0.4997 | Gate Openness: 55.04% | LR: 1.14e-4
📐 GRADS -> Max: [spell_w: 0.2736] | Valve: 0.0255 | Gate Bias: 0.0002 | RAM: 250MB
➡️ Step 35500 | Cosine Loss: 0.5166 | Gate Openness: 54.1% | LR: 1.13e-4
📐 GRADS -> Max: [spell_w: 0.2253] | Valve: 0.0263 | Gate Bias: 0.0 | RAM: 250MB
➡️ Step 35550 | Cosine Loss: 0.5152 | Gate Openness: 55.46% | LR: 1.13e-4
📐 GRADS -> Max: [spell_w: 0.258] | Valve: 0.0242 | Gate Bias: 0.0 | RAM: 250MB
➡️ Step 35600 | Cosine Loss: 0.4998 | Gate Openness: 54.87% | LR: 1.13e-4
📐 GRADS -> Max: [spell_w: 0.4468] | Valve: 0.0291 | Gate Bias: 0.0 | RAM: 251MB
➡️ Step 35650 | Cosine Loss: 0.516 | Gate Openness: 53.84% | LR: 1.13e-4
📐 GRADS -> Max: [spell_w: 0.3027] | Valve: 0.0276 | Gate Bias: 0.0001 | RAM: 251MB
➡️ Step 35700 | Cosine Loss: 0.5021 | Gate Openness: 54.96% | LR: 1.13e-4
📐 GRADS -> Max: [spell_w: 0.3628] | Valve: 0.0254 | Gate Bias: 0.0 | RAM: 252MB
➡️ Step 35750 | Cosine Loss: 0.5172 | Gate Openness: 52.83% | LR: 1.13e-4
📐 GRADS -> Max: [spell_w: 0.276] | Valve: 0.0247 | Gate Bias: 0.0002 | RAM: 252MB
➡️ Step 35800 | Cosine Loss: 0.517 | Gate Openness: 54.43% | LR: 1.13e-4
📐 GRADS -> Max: [spell_w: 0.2985] | Valve: 0.0234 | Gate Bias: 0.0001 | RAM: 253MB
➡️ Step 35850 | Cosine Loss: 0.5118 | Gate Openness: 53.48% | LR: 1.13e-4
📐 GRADS -> Max: [spell_w: 0.343] | Valve: 0.0272 | Gate Bias: 0.0001 | RAM: 270MB
➡️ Step 35900 | Cosine Loss: 0.5103 | Gate Openness: 55.27% | LR: 1.13e-4
📐 GRADS -> Max: [spell_w: 0.3489] | Valve: 0.0259 | Gate Bias: 0.0 | RAM: 270MB
➡️ Step 35950 | Cosine Loss: 0.5098 | Gate Openness: 54.39% | LR: 1.13e-4
📐 GRADS -> Max: [spell_w: 0.3101] | Valve: 0.0271 | Gate Bias: 0.0 | RAM: 270MB
➡️ Step 36000 | Cosine Loss: 0.5063 | Gate Openness: 54.31% | LR: 1.12e-4
📐 GRADS -> Max: [spell_w: 0.2405] | Valve: 0.0245 | Gate Bias: 0.0 | RAM: 271MB
💾 Auto-Saving Checkpoint Step 36000...
➡️ Step 36050 | Cosine Loss: 0.5065 | Gate Openness: 53.98% | LR: 1.12e-4
📐 GRADS -> Max: [spell_w: 0.5392] | Valve: 0.0268 | Gate Bias: 0.0 | RAM: 263MB
➡️ Step 36100 | Cosine Loss: 0.5076 | Gate Openness: 55.75% | LR: 1.12e-4
📐 GRADS -> Max: [spell_w: 0.5903] | Valve: 0.0274 | Gate Bias: 0.0001 | RAM: 264MB
➡️ Step 36150 | Cosine Loss: 0.5045 | Gate Openness: 54.6% | LR: 1.12e-4
📐 GRADS -> Max: [spell_w: 0.3422] | Valve: 0.0272 | Gate Bias: 0.0002 | RAM: 273MB
➡️ Step 36200 | Cosine Loss: 0.5098 | Gate Openness: 54.78% | LR: 1.12e-4
📐 GRADS -> Max: [spell_w: 0.3326] | Valve: 0.0275 | Gate Bias: 0.0001 | RAM: 281MB
➡️ Step 36250 | Cosine Loss: 0.5137 | Gate Openness: 55.14% | LR: 1.12e-4
📐 GRADS -> Max: [spell_w: 0.3523] | Valve: 0.0271 | Gate Bias: 0.0 | RAM: 289MB
➡️ Step 36300 | Cosine Loss: 0.5002 | Gate Openness: 54.38% | LR: 1.12e-4
📐 GRADS -> Max: [spell_w: 0.305] | Valve: 0.0283 | Gate Bias: 0.0001 | RAM: 257MB
➡️ Step 36350 | Cosine Loss: 0.5024 | Gate Openness: 55.43% | LR: 1.12e-4
📐 GRADS -> Max: [spell_w: 0.4106] | Valve: 0.0279 | Gate Bias: 0.0001 | RAM: 257MB
➡️ Step 36400 | Cosine Loss: 0.5111 | Gate Openness: 54.49% | LR: 1.12e-4
📐 GRADS -> Max: [spell_w: 0.3456] | Valve: 0.025 | Gate Bias: 0.0001 | RAM: 258MB
➡️ Step 36450 | Cosine Loss: 0.5321 | Gate Openness: 55.08% | LR: 1.12e-4
📐 GRADS -> Max: [spell_w: 0.22] | Valve: 0.0247 | Gate Bias: 0.0002 | RAM: 275MB
➡️ Step 36500 | Cosine Loss: 0.4885 | Gate Openness: 54.61% | LR: 1.12e-4
📐 GRADS -> Max: [spell_w: 0.3394] | Valve: 0.0273 | Gate Bias: 0.0 | RAM: 275MB
➡️ Step 36550 | Cosine Loss: 0.5002 | Gate Openness: 55.49% | LR: 1.11e-4
📐 GRADS -> Max: [spell_w: 0.5683] | Valve: 0.0333 | Gate Bias: 0.0 | RAM: 276MB
➡️ Step 36600 | Cosine Loss: 0.4974 | Gate Openness: 52.24% | LR: 1.11e-4
📐 GRADS -> Max: [spell_w: 0.3453] | Valve: 0.026 | Gate Bias: 0.0 | RAM: 276MB
➡️ Step 36650 | Cosine Loss: 0.511 | Gate Openness: 55.67% | LR: 1.11e-4
📐 GRADS -> Max: [spell_w: 0.5279] | Valve: 0.0269 | Gate Bias: 0.0 | RAM: 260MB
➡️ Step 36700 | Cosine Loss: 0.5125 | Gate Openness: 56.59% | LR: 1.11e-4
📐 GRADS -> Max: [spell_w: 0.2659] | Valve: 0.0261 | Gate Bias: 0.0001 | RAM: 261MB
➡️ Step 36750 | Cosine Loss: 0.4937 | Gate Openness: 55.48% | LR: 1.11e-4
📐 GRADS -> Max: [spell_w: 0.4547] | Valve: 0.0288 | Gate Bias: 0.0 | RAM: 261MB
➡️ Step 36800 | Cosine Loss: 0.5174 | Gate Openness: 53.19% | LR: 1.11e-4
📐 GRADS -> Max: [spell_w: 0.3762] | Valve: 0.0258 | Gate Bias: 0.0001 | RAM: 262MB
➡️ Step 36850 | Cosine Loss: 0.5164 | Gate Openness: 53.51% | LR: 1.11e-4
📐 GRADS -> Max: [spell_w: 0.3736] | Valve: 0.0267 | Gate Bias: 0.0001 | RAM: 262MB
➡️ Step 36900 | Cosine Loss: 0.4958 | Gate Openness: 53.2% | LR: 1.11e-4
📐 GRADS -> Max: [spell_w: 0.2553] | Valve: 0.0279 | Gate Bias: 0.0 | RAM: 258MB
➡️ Step 36950 | Cosine Loss: 0.5065 | Gate Openness: 54.83% | LR: 1.11e-4
📐 GRADS -> Max: [spell_w: 0.3626] | Valve: 0.0225 | Gate Bias: 0.0 | RAM: 267MB
➡️ Step 37000 | Cosine Loss: 0.5029 | Gate Openness: 55.35% | LR: 1.11e-4
📐 GRADS -> Max: [spell_w: 0.3627] | Valve: 0.0261 | Gate Bias: 0.0001 | RAM: 259MB
💾 Auto-Saving Checkpoint Step 37000...
➡️ Step 37050 | Cosine Loss: 0.5188 | Gate Openness: 52.96% | LR: 1.1e-4
📐 GRADS -> Max: [spell_w: 0.3412] | Valve: 0.0296 | Gate Bias: 0.0001 | RAM: 280MB