fix camera project/unproject

- was wrong when using offset viewport

CHANGELOG.md

@@ -5,6 +5,7 @@ Note that since we don't clearly distinguish between a public and private interf
 
 
 ## [Unreleased]
+
 - Handle resizes of viewer element even when window remains the same size
 - Throttle canvas resize events
 - Selection toggle buttons hidden if selection mode is off
@@ -16,6 +17,7 @@ Note that since we don't clearly distinguish between a public and private interf
 - Apply bumpiness as lightness variation with `ignoreLight`
 - Remove `JSX` reference from `loci-labels.ts`
 - Fix overpaint/transparency/substance smoothing not updated when geometry changes
+- Fix camera project/unproject when using offset viewport
 
 ## [v3.32.0] - 2023-03-20
 

src/mol-canvas3d/_spec/camera.spec.ts

@@ -0,0 +1,37 @@
+/**
+ * Copyright (c) 2023 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ *
+ * @author Alexander Rose <alexander.rose@weirdbyte.de>
+ */
+
+import { Vec3, Vec4 } from '../../mol-math/linear-algebra';
+import { Mat4 } from '../../mol-math/linear-algebra/3d/mat4';
+import { Viewport, cameraProject, cameraUnproject } from '../camera/util';
+
+describe('camera', () => {
+    it('project/unproject', () => {
+        const proj = Mat4.perspective(Mat4(), -1, 1, 1, -1, 1, 100);
+        const invProj = Mat4.invert(Mat4(), proj);
+
+        const c = Vec4();
+        const po = Vec3();
+
+        const vp = Viewport.create(0, 0, 100, 100);
+        const pi = Vec3.create(0, 0, 1);
+        cameraProject(c, pi, vp, proj);
+        expect(Vec4.equals(c, Vec4.create(50, 50, 2.020202, -1))).toBe(true);
+        cameraUnproject(po, c, vp, invProj);
+        expect(Vec3.equals(po, pi)).toBe(true);
+
+        Vec3.set(pi, 0.5, 0.5, 1);
+        cameraProject(c, pi, vp, proj);
+        cameraUnproject(po, c, vp, invProj);
+        expect(Vec3.equals(po, pi)).toBe(true);
+
+        Viewport.set(vp, 50, 50, 100, 100);
+        Vec3.set(pi, 0.5, 0.5, 1);
+        cameraProject(c, pi, vp, proj);
+        cameraUnproject(po, c, vp, invProj);
+        expect(Vec3.equals(po, pi)).toBe(true);
+    });
+});

src/mol-canvas3d/camera.ts

@@ -194,7 +194,7 @@ class Camera implements ICamera {
     getPixelSize(point: Vec3) {
         // project -> unproject of `point` does not exactly return the same
         // to get a sufficiently accurate measure we unproject the original
-        // clip position in addition to the one shifted bey one pixel
+        // clip position in addition to the one shifted by one pixel
         this.project(tmpClip, point);
         this.unproject(tmpPos1, tmpClip);
         tmpClip[0] += 1;

src/mol-canvas3d/camera/util.ts

@@ -77,7 +77,7 @@ export function cameraProject(out: Vec4, point: Vec3, viewport: Viewport, projec
 
     // transform into window coordinates, set fourth component to 1 / clip.w as in gl_FragCoord.w
     out[0] = (tmpVec4[0] + 1) * width * 0.5 + x;
-    out[1] = (1 - tmpVec4[1]) * height * 0.5 + y; // flip Y
+    out[1] = (tmpVec4[1] + 1) * height * 0.5 + y;
     out[2] = (tmpVec4[2] + 1) * 0.5;
     out[3] = w === 0 ? 0 : 1 / w;
     return out;
@@ -92,7 +92,7 @@ export function cameraUnproject(out: Vec3, point: Vec3 | Vec4, viewport: Viewpor
     const { x, y, width, height } = viewport;
 
     const px = point[0] - x;
-    const py = (height - point[1] - 1) - y;
+    const py = point[1] - y;
     const pz = point[2];
 
     out[0] = (2 * px) / width - 1;

src/mol-canvas3d/passes/pick.ts

@@ -358,7 +358,7 @@ export class PickHelper {
 
         const z = this.getDepth(xp, yp);
         // console.log('z', z);
-        const position = Vec3.create(x, viewport.height - y, z);
+        const position = Vec3.create(x, y, z);
         if (StereoCamera.is(camera)) {
             const halfWidth = Math.floor(viewport.width / 2);
             if (x > viewport.x + halfWidth) {