Fix reduce identity encoding for floats and integers. (#186)

Float identities (e.g. -Inf for max, +Inf for min) were encoded as raw
bit patterns via plain varint, but the bytecode format expects ap_int
encoding (zigzag + varint). This caused the disassembler to show NaN
instead of -Inf for maximum, and 1.5 instead of +Inf for minimum.

Fix: use encode_signed_varint! with Int128-widened bits so the zigzag
shift-left never overflows.

Integer identities had a separate bug: they were zigzag-encoded, but the
format stores them as raw values (no zigzag). The existing zigzag also
had a UInt32 overflow on the shift-left for typemin(Int32). Fix: remove
zigzag from integer identities entirely and widen to UInt128 for the
varint encoder.

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: maleadt

src/bytecode/writer.jl

@@ -339,9 +339,10 @@ function encode_tagged_float!(cb::CodeBuilder, identity::FloatIdentityVal)
     push!(cb.buf, 0x02)
     # Type ID
     encode_typeid!(cb.buf, identity.type_id)
-    # Value as bits (using signed varint encoding for values <= 64 bits)
+    # Value as ap_int: zigzag-encode the bit pattern, then varint.
+    # Widen to Int128 so the shift-left never overflows.
     bits = float_to_bits(identity.value, identity.dtype)
-    encode_varint!(cb.buf, bits)
+    encode_signed_varint!(cb.buf, Int128(bits))
 end
 
 """
@@ -372,11 +373,9 @@ function mask_to_width(value::UInt128, ::Type{T}) where T <: Integer
     masked = value & mask
     U = unsigned(T)
     unsigned_masked = U(masked)
-    if T <: Signed # do zig-zag encoding
-        U((unsigned_masked << 1) ⊻ (unsigned_masked >>> (bits - 1)))
-    else
-        unsigned_masked
-    end
+    # Integer identities are stored as raw values (no zigzag encoding).
+    # Widen to UInt128 so the varint encoder handles any bit width.
+    UInt128(unsigned_masked)
 end
 
 """

test/codegen/operations.jl

@@ -726,6 +726,75 @@ spec4d = ct.ArraySpec{4}(16, true)
             end
         end
 
+        # reduce identity values (zigzag-encoded in bytecode)
+        @testset "identities" begin
+            # Float32: sum → 0.0, maximum → -Inf (0xFF800000), minimum → +Inf (0x7F800000)
+            @test @filecheck begin
+                @check_label "entry"
+                code_tiled(Tuple{ct.TileArray{Float32,2,spec2d}}) do a
+                    pid = ct.bid(1)
+                    tile = ct.load(a, pid, (4, 16))
+                    @check "identities=[0.000000e+00 : f32]"
+                    Base.donotdelete(sum(tile; dims=2))
+                    return
+                end
+            end
+            @test @filecheck begin
+                @check_label "entry"
+                code_tiled(Tuple{ct.TileArray{Float32,2,spec2d}}) do a
+                    pid = ct.bid(1)
+                    tile = ct.load(a, pid, (4, 16))
+                    @check "identities=[0xFF800000 : f32]"
+                    Base.donotdelete(maximum(tile; dims=2))
+                    return
+                end
+            end
+            @test @filecheck begin
+                @check_label "entry"
+                code_tiled(Tuple{ct.TileArray{Float32,2,spec2d}}) do a
+                    pid = ct.bid(1)
+                    tile = ct.load(a, pid, (4, 16))
+                    @check "identities=[0x7F800000 : f32]"
+                    Base.donotdelete(minimum(tile; dims=2))
+                    return
+                end
+            end
+
+            # Float16: maximum → -Inf (0xFC00), minimum → +Inf (0x7C00)
+            @test @filecheck begin
+                @check_label "entry"
+                code_tiled(Tuple{ct.TileArray{Float16,2,spec2d}}) do a
+                    pid = ct.bid(1)
+                    tile = ct.load(a, pid, (4, 16))
+                    @check "identities=[0xFC00 : f16]"
+                    Base.donotdelete(maximum(tile; dims=2))
+                    return
+                end
+            end
+            @test @filecheck begin
+                @check_label "entry"
+                code_tiled(Tuple{ct.TileArray{Float16,2,spec2d}}) do a
+                    pid = ct.bid(1)
+                    tile = ct.load(a, pid, (4, 16))
+                    @check "identities=[0x7C00 : f16]"
+                    Base.donotdelete(minimum(tile; dims=2))
+                    return
+                end
+            end
+
+            # Int32: maximum → typemin (zigzag of -2147483648)
+            @test @filecheck begin
+                @check_label "entry"
+                code_tiled(Tuple{ct.TileArray{Int32,2,spec2d}}) do a
+                    pid = ct.bid(1)
+                    tile = ct.load(a, pid, (4, 16))
+                    @check "identities=[-2147483648"
+                    Base.donotdelete(maximum(tile; dims=2))
+                    return
+                end
+            end
+        end
+
         # argmax
         @test @filecheck begin
             @check_label "entry"