fix: matrix math when non-uniform scale+rotation are in play

CHANGELOG.md

@@ -39,6 +39,7 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 
 ### Fixed
 
+- Fixed Matrix and AffineMatrix scale/rotation decomposition bug where getScaleX/getScaleY used wrong basis components for non-uniform scale combined with rotation, causing swapped scale values and corrupt transforms. Also fixed setRotation and setScaleX/setScaleY to operate on correct column basis vectors.
 - Fixed issue where the first action in a sequence would not execute after calling `clearActions()` mid-execution. All action types now properly reset their initialization state when stopped, resolving issue #3468
 - Performance: Font/Text now use smaller texture sizes, improving performance on Safari especially when rendering text
 

src/engine/math/affine-matrix.ts

@@ -324,31 +324,27 @@ export class AffineMatrix {
     const cosine = Math.cos(angle);
 
     this.data[0] = cosine * currentScale.x;
-    this.data[1] = sine * currentScale.y;
-    this.data[2] = -sine * currentScale.x;
+    this.data[1] = sine * currentScale.x;
+    this.data[2] = -sine * currentScale.y;
     this.data[3] = cosine * currentScale.y;
   }
 
   public getRotation(): number {
-    const angle = Math.atan2(this.data[1] / this.getScaleY(), this.data[0] / this.getScaleX());
+    const angle = Math.atan2(this.data[1] / this.getScaleX(), this.data[0] / this.getScaleX());
     return canonicalizeAngle(angle);
   }
 
   public getScaleX(): number {
-    // absolute scale of the matrix (we lose sign so need to add it back)
-    const xScaleSq = this.data[0] * this.data[0] + this.data[2] * this.data[2];
+    const xScaleSq = this.data[0] * this.data[0] + this.data[1] * this.data[1];
     if (xScaleSq === 1.0) {
-      // usually there isn't scale so we can avoid a sqrt
       return this._scaleSignX;
     }
     return this._scaleSignX * Math.sqrt(xScaleSq);
   }
 
   public getScaleY(): number {
-    // absolute scale of the matrix (we lose sign so need to add it back)
-    const yScaleSq = this.data[1] * this.data[1] + this.data[3] * this.data[3];
+    const yScaleSq = this.data[2] * this.data[2] + this.data[3] * this.data[3];
     if (yScaleSq === 1.0) {
-      // usually there isn't scale so we can avoid a sqrt
       return this._scaleSignY;
     }
     return this._scaleSignY * Math.sqrt(yScaleSq);
@@ -368,10 +364,9 @@ export class AffineMatrix {
       return;
     }
     this._scaleSignX = sign(val);
-    // negative scale acts like a 180 rotation, so flip
-    const xscale = vec(this.data[0] * this._scaleSignX, this.data[2] * this._scaleSignX).normalize();
+    const xscale = vec(this.data[0] * this._scaleSignX, this.data[1] * this._scaleSignX).normalize();
     this.data[0] = xscale.x * val;
-    this.data[2] = xscale.y * val;
+    this.data[1] = xscale.y * val;
     this._scale[0] = val;
   }
 
@@ -381,9 +376,8 @@ export class AffineMatrix {
       return;
     }
     this._scaleSignY = sign(val);
-    // negative scale acts like a 180 rotation, so flip
-    const yscale = vec(this.data[1] * this._scaleSignY, this.data[3] * this._scaleSignY).normalize();
-    this.data[1] = yscale.x * val;
+    const yscale = vec(this.data[2] * this._scaleSignY, this.data[3] * this._scaleSignY).normalize();
+    this.data[2] = yscale.x * val;
     this.data[3] = yscale.y * val;
     this._scale[1] = val;
   }

src/engine/math/matrix.ts

@@ -403,25 +403,23 @@ export class Matrix {
     const cosine = Math.cos(angle);
 
     this.data[0] = cosine * currentScale.x;
-    this.data[1] = sine * currentScale.y;
-    this.data[4] = -sine * currentScale.x;
+    this.data[1] = sine * currentScale.x;
+    this.data[4] = -sine * currentScale.y;
     this.data[5] = cosine * currentScale.y;
   }
 
   public getRotation(): number {
-    const angle = Math.atan2(this.data[1] / this.getScaleY(), this.data[0] / this.getScaleX());
+    const angle = Math.atan2(this.data[1] / this.getScaleX(), this.data[0] / this.getScaleX());
     return canonicalizeAngle(angle);
   }
 
   public getScaleX(): number {
-    // absolute scale of the matrix (we lose sign so need to add it back)
-    const xscale = vec(this.data[0], this.data[4]).magnitude;
+    const xscale = vec(this.data[0], this.data[1]).magnitude;
     return this._scaleSignX * xscale;
   }
 
   public getScaleY(): number {
-    // absolute scale of the matrix (we lose sign so need to add it back)
-    const yscale = vec(this.data[1], this.data[5]).magnitude;
+    const yscale = vec(this.data[4], this.data[5]).magnitude;
     return this._scaleSignY * yscale;
   }
 
@@ -440,10 +438,9 @@ export class Matrix {
     }
 
     this._scaleSignX = sign(val);
-    // negative scale acts like a 180 rotation, so flip
-    const xscale = vec(this.data[0] * this._scaleSignX, this.data[4] * this._scaleSignX).normalize();
+    const xscale = vec(this.data[0] * this._scaleSignX, this.data[1] * this._scaleSignX).normalize();
     this.data[0] = xscale.x * val;
-    this.data[4] = xscale.y * val;
+    this.data[1] = xscale.y * val;
     this._scaleX = val;
   }
 
@@ -454,9 +451,8 @@ export class Matrix {
       return;
     }
     this._scaleSignY = sign(val);
-    // negative scale acts like a 180 rotation, so flip
-    const yscale = vec(this.data[1] * this._scaleSignY, this.data[5] * this._scaleSignY).normalize();
-    this.data[1] = yscale.x * val;
+    const yscale = vec(this.data[4] * this._scaleSignY, this.data[5] * this._scaleSignY).normalize();
+    this.data[4] = yscale.x * val;
     this.data[5] = yscale.y * val;
     this._scaleY = val;
   }

src/spec/vitest/affine-matrix-spec.ts

@@ -187,4 +187,62 @@ describe('A AffineMatrix', () => {
 [0 0 1]
 `);
   });
+
+  it('can get correct scale with non-uniform scale and rotation', () => {
+    const mat = ex.AffineMatrix.identity();
+    mat.setScale(ex.vec(2, 3));
+    mat.setRotation(Math.PI / 2);
+
+    expect(mat.getScaleX()).toBeCloseTo(2, 5);
+    expect(mat.getScaleY()).toBeCloseTo(3, 5);
+  });
+
+  it('setScaleX and setScaleY modify correct basis columns', () => {
+    const mat = ex.AffineMatrix.identity();
+    mat.setRotation(Math.PI / 4);
+    mat.setScaleX(5);
+
+    const xscale = Math.sqrt(mat.data[0] * mat.data[0] + mat.data[1] * mat.data[1]);
+    expect(xscale).toBeCloseTo(5, 5);
+
+    mat.setScaleY(7);
+    const yscale = Math.sqrt(mat.data[2] * mat.data[2] + mat.data[3] * mat.data[3]);
+    expect(yscale).toBeCloseTo(7, 5);
+  });
+
+  it('round-trip setScale + setRotation preserves scale values', () => {
+    const mat = ex.AffineMatrix.identity();
+    mat.setScale(ex.vec(4, 9));
+    mat.setRotation(Math.PI / 3);
+
+    expect(mat.getScaleX()).toBeCloseTo(4, 5);
+    expect(mat.getScaleY()).toBeCloseTo(9, 5);
+    expect(mat.getRotation()).toBeCloseTo(Math.PI / 3, 5);
+  });
+
+  it('setRotation produces correct column-major data layout', () => {
+    const mat = ex.AffineMatrix.identity();
+    mat.setScale(ex.vec(2, 3));
+    mat.setRotation(Math.PI / 2);
+
+    const cos = Math.cos(Math.PI / 2);
+    const sin = Math.sin(Math.PI / 2);
+
+    expect(mat.data[0]).toBeCloseTo(cos * 2, 5);
+    expect(mat.data[1]).toBeCloseTo(sin * 2, 5);
+    expect(mat.data[2]).toBeCloseTo(-sin * 3, 5);
+    expect(mat.data[3]).toBeCloseTo(cos * 3, 5);
+  });
+
+  it('AffineMatrix and 4x4 Matrix agree on scale/rotation after decomposition', () => {
+    const affine = ex.AffineMatrix.identity();
+    affine.setScale(ex.vec(3, 7));
+    affine.setRotation(Math.PI / 5);
+
+    const mat4x4 = affine.to4x4();
+
+    expect(mat4x4.getScaleX()).toBeCloseTo(affine.getScaleX(), 5);
+    expect(mat4x4.getScaleY()).toBeCloseTo(affine.getScaleY(), 5);
+    expect(mat4x4.getRotation()).toBeCloseTo(affine.getRotation(), 5);
+  });
 });

src/spec/vitest/matrix-spec.ts

@@ -184,4 +184,50 @@ describe('A Matrix', () => {
 [0 0 0 1]
 `);
   });
+
+  it('can get correct scale with non-uniform scale and rotation', () => {
+    const mat = ex.Matrix.identity();
+    mat.setScale(ex.vec(2, 3));
+    mat.setRotation(Math.PI / 2);
+
+    expect(mat.getScaleX()).toBeCloseTo(2, 5);
+    expect(mat.getScaleY()).toBeCloseTo(3, 5);
+  });
+
+  it('setScaleX and setScaleY modify correct basis columns', () => {
+    const mat = ex.Matrix.identity();
+    mat.setRotation(Math.PI / 4);
+    mat.setScaleX(5);
+
+    const xscale = Math.sqrt(mat.data[0] * mat.data[0] + mat.data[1] * mat.data[1]);
+    expect(xscale).toBeCloseTo(5, 5);
+
+    mat.setScaleY(7);
+    const yscale = Math.sqrt(mat.data[4] * mat.data[4] + mat.data[5] * mat.data[5]);
+    expect(yscale).toBeCloseTo(7, 5);
+  });
+
+  it('round-trip setScale + setRotation preserves scale values', () => {
+    const mat = ex.Matrix.identity();
+    mat.setScale(ex.vec(4, 9));
+    mat.setRotation(Math.PI / 3);
+
+    expect(mat.getScaleX()).toBeCloseTo(4, 5);
+    expect(mat.getScaleY()).toBeCloseTo(9, 5);
+    expect(mat.getRotation()).toBeCloseTo(Math.PI / 3, 5);
+  });
+
+  it('setRotation produces correct column-major data layout', () => {
+    const mat = ex.Matrix.identity();
+    mat.setScale(ex.vec(2, 3));
+    mat.setRotation(Math.PI / 2);
+
+    const cos = Math.cos(Math.PI / 2);
+    const sin = Math.sin(Math.PI / 2);
+
+    expect(mat.data[0]).toBeCloseTo(cos * 2, 5);
+    expect(mat.data[1]).toBeCloseTo(sin * 2, 5);
+    expect(mat.data[4]).toBeCloseTo(-sin * 3, 5);
+    expect(mat.data[5]).toBeCloseTo(cos * 3, 5);
+  });
 });