Make entities inspectable via DevTools canvas hovering

[kungfooman]

Quick-link to example: https://playcanvas.github.io/web-components/examples/basic-shapes.html

Code for pasting in DevTools console:

const {app} = document.body.querySelector("pc-app");
const sceneDom = document.body.querySelector("pc-scene");
app.on('update', () => {
  const entitiesWithRender = [...sceneDom.children].filter(_ => _.entity.render);
  const camera = sceneDom.querySelector("pc-camera")._component;
  for (const dom of entitiesWithRender) {
    const {entity} = dom;
    const {aabb} = entity.render.meshInstances[0];
    const min = aabb.getMin();
    const max = aabb.getMax();
    const screenMin = camera.worldToScreen(min);
    const screenMax = camera.worldToScreen(max);
    const zIndex = Math.floor((1 / Math.max(screenMin.z, screenMax.z)) * 1000);
    const left = Math.min(screenMin.x, screenMax.x);
    const top = Math.min(screenMin.y, screenMax.y);
    const width = Math.abs(screenMax.x - screenMin.x);
    const height = Math.abs(screenMax.y - screenMin.y);
    dom.style.position = "absolute";
    dom.style.border = "1px solid black";
    // Hold Shift to select elements without pointer events!
    dom.style.pointerEvents = "none";
    dom.style.left = `${left}px`;
    dom.style.top = `${top}px`;
    dom.style.width = `${width}px`;
    dom.style.height = `${height}px`;
    dom.style.zIndex = `${zIndex}`;
    // console.log({left, top, width, height, screenMin, screenMax, zIndex});
  }
});

Effect:

It's not fully fleshed out, the bbox just doesn't really capture the entire entity 🤔

Otherwise it seems to work quite nice, even the zIndex works out well by simply inverting the worldToSceen z component. And currently the pointerEvents are not disabled, but if I disable them, the DevTools doesn't pick it up any longer when you hover over them... so definitely more to investigate here. 🙈

Ideas are welcome 💭 🥇

[Maksims]

It's not fully fleshed out, the bbox just doesn't really capture the entire entity

For that you would need to get screen-space min/max, the brut force solution would be to calculate screen space position of each AABB corner, and then calculate min/max screen X and Y of these vectors.

Also, don't call findByName on each iteration, that can easily be the most costly call here. Do it once before the loop.

[Maksims]

Here is a complete script, that can be added to any project, that implements everything to enable entity picking in your project:

Bear in mind that such pickers are extremely unpractical in anything bigger than couple of entities in a scene.

var DomPicker = pc.createScript('domPicker');

DomPicker.prototype.initialize = function () {
    this.entities = new Set();
    this.element = document.createElement('div');
    this.element.id = 'entities';
    this.element.addEventListener('contextmenu', (evt) => {
        evt.preventDefault();
    });
    document.body.appendChild(this.element);

    this.style = document.createElement('style');
    this.style.innerHTML = `
        #entities {
            position: absolute;
            top: 0px;
            left: 0px;
            right: 0px;
            bottom: 0px;
            user-select: none;
            overflow: hidden;
        }
        #entities > div {
            position: absolute;
            outline: 1px solid rgba(255, 255, 255, 0.4);
        }
        #entities > div:hover {
            background-color: rgba(255, 255, 255, 0.1);
            outline: 1px solid #7f7;
        }
    `;
    document.head.appendChild(this.style);

    this.vec3A = new pc.Vec3();
    this.vec3B = new pc.Vec3();
};

DomPicker.prototype.onEntityPicked = function (entity) {
    // entity picked
    this.app.fire('entity:picked', entity);
    console.log(entity.name, entity);
};

DomPicker.prototype.updateBounds = function (camera, bounds, center, offX, offY, offZ) {
    // move DOM based on AABB
    this.vec3A.copy(center);
    this.vec3A.x += offX;
    this.vec3A.y += offY;
    this.vec3A.z += offZ;
    camera.worldToScreen(this.vec3A, this.vec3B);
    bounds.minX = Math.min(bounds.minX, this.vec3B.x);
    bounds.minY = Math.min(bounds.minY, this.vec3B.y);
    bounds.minZ = Math.min(bounds.minZ, this.vec3B.z);
    bounds.maxX = Math.max(bounds.maxX, this.vec3B.x);
    bounds.maxY = Math.max(bounds.maxY, this.vec3B.y);
    bounds.maxZ = Math.max(bounds.maxZ, this.vec3B.z);
};

DomPicker.prototype.postUpdate = function (dt) {
    const camera = this.entity.camera;
    const position = this.entity.getPosition();

    this.app.root.forEach((entity) => {
        if (!entity.enabled) {
            if (entity.dom) {
                this.element.removeChild(entity.dom);
                entity.dom = null;
            }
            return;
        }

        if (!entity.render) return;

        // DOM
        if (!entity.dom) {
            entity.dom = document.createElement('div');
            entity.dom.enabled = true;
            entity.dom.addEventListener('click', (evt) => {
                this.onEntityPicked(entity);
            });

            this.element.appendChild(entity.dom);

            if (!entity.onDomDestroy) {
                entity.onDomDestroy = entity.once('destroy', () => {
                    if (entity.dom) {
                        this.element.removeChild(entity.dom);
                        entity.dom = null;
                    }
                    entity.onDomDestroy = null;
                });
            }
        }

        // AABB
        if (!entity.aabb) entity.aabb = new pc.BoundingBox();

        for (let i = 0; i < entity.render.meshInstances.length; i++) {
            const meshInstance = entity.render.meshInstances[i];
            if (i === 0) {
                entity.aabb.copy(meshInstance.aabb);
            } else {
                entity.aabb.add(meshInstance.aabb);
            }
        }

        // move DOM based on AABB
        const bounds = {
            minX: Infinity,
            minY: Infinity,
            minZ: Infinity,
            maxX: -Infinity,
            maxY: -Infinity,
            maxZ: -Infinity
        };

        this.updateBounds(camera, bounds, entity.aabb.center, +entity.aabb.halfExtents.x, +entity.aabb.halfExtents.y, +entity.aabb.halfExtents.z);
        this.updateBounds(camera, bounds, entity.aabb.center, -entity.aabb.halfExtents.x, +entity.aabb.halfExtents.y, +entity.aabb.halfExtents.z);
        this.updateBounds(camera, bounds, entity.aabb.center, +entity.aabb.halfExtents.x, -entity.aabb.halfExtents.y, +entity.aabb.halfExtents.z);
        this.updateBounds(camera, bounds, entity.aabb.center, -entity.aabb.halfExtents.x, -entity.aabb.halfExtents.y, +entity.aabb.halfExtents.z);
        this.updateBounds(camera, bounds, entity.aabb.center, +entity.aabb.halfExtents.x, +entity.aabb.halfExtents.y, -entity.aabb.halfExtents.z);
        this.updateBounds(camera, bounds, entity.aabb.center, -entity.aabb.halfExtents.x, +entity.aabb.halfExtents.y, -entity.aabb.halfExtents.z);
        this.updateBounds(camera, bounds, entity.aabb.center, -entity.aabb.halfExtents.x, -entity.aabb.halfExtents.y, -entity.aabb.halfExtents.z);
        this.updateBounds(camera, bounds, entity.aabb.center, +entity.aabb.halfExtents.x, -entity.aabb.halfExtents.y, -entity.aabb.halfExtents.z);

        if (bounds.maxZ < 0 || bounds.minZ < 0) {
            entity.dom.style.display = 'none';
        } else {
            entity.dom.style.display = '';
            entity.dom.style.left = Math.round(bounds.minX) + 'px';
            entity.dom.style.top = Math.round(bounds.minY) + 'px';
            entity.dom.style.width = Math.round(bounds.maxX - bounds.minX) + 'px';
            entity.dom.style.height = Math.round(bounds.maxY - bounds.minY) + 'px';

            entity.dom.style.zIndex = Math.max(0, Math.round(bounds.minZ * 1000));
        }
    });
};

[kungfooman]

@Maksims Thank you for the feedback! Yea, the script is just a quick proof-of-concept, nothing final. Something you can paste in DevTools console, having some fun editing and press Enter.

I only added borders for debugging and demonstration, in a normal scene you shouldn't see that. Only issue in a big scene might be performance updating the style's all the time. I mean, especially in a big scene people need this functionality to track down an entity. Without this, people try to "inspect" and the only handle they will find is a <canvas>.

So the main difference between your dom picker and this is DevTools integration, making the most powerful features of browser debuggers available: Press Ctrl+Shift+C and select whatever you need. No need for adding picker scripts etc., just make it as native as it gets for efficiency.

[Maksims]

You can modify a few lines of a code I've provided, and it will be copy-pasteable in console.

[kungfooman]

@Maksims Thank you, good point! How do you deal with mouse input, when the user e.g. wants to rotate an orbit camera instead of clicking on <div>'s? That's the biggest issue so far I'm facing with this - I don't want it to "conflict" with anything, hence I set pointer-events: none. I guess you specifically require the click, since that logs the entity... I investigated this a bit and turns out there is something I never knew: you can pick pointer-events: none if you hold Shift during inspector picking.

[Maksims]

First of all we need to ensure to disable selecting (user-select), that way dragging will not happen. Then I don't block mouse events, so if your mouse events are subscribed on window (by default in PlayCanvas they are), then all will work.

[kungfooman]

Funny, when I use user-select: none it just blocks the orbit camera in the example I linked in first post, I can only get it to work with pointer-events: none

[Maksims]

As I've mentioned: your events listeners should be on window, not canvas.

[kungfooman]

As I've mentioned: your events listeners should be on window, not canvas.

It's a limitation I don't understand, why should a script break based on these assumptions? Developers should be able to use whatever they want, e.g. the shipped CameraControls script is using <canvas>:

So currently I settled on pointerEvents: none.

[Maksims]

The problem is, if you want to have UI over the page, and still be able to catch events, then you need to subscribe to element that events will propagate to. As canvas element does not contain other DOM elements you interact with, this will lead to issues.

This topic been discussed for years, and consensus is: subscribe on window, and if you want to DOM element to "block", then call stopPropagation. If you don't do this, you end up with way more complications and edge cases which just will make your life worse. Just like you already experiencing here. So the solution is: subscribe on window.

Recommended and default target for events by PlayCanvas - is window.

[kungfooman]

Okay, updated to include every corner in screenToWorld test:

const {Vec3} = await import("playcanvas");
const {app} = document.body.querySelector("pc-app");
const sceneDom = document.body.querySelector("pc-scene");
const points = [new Vec3(), new Vec3(), new Vec3(), new Vec3(), new Vec3(), new Vec3(), new Vec3(), new Vec3()];
const tmp = new Vec3();
function updateInspectorPositions() {
  const entitiesWithRender = [...sceneDom.children].filter(_ => _.entity.render);
  const camera = sceneDom.querySelector('pc-camera')._component;
  for (const dom of entitiesWithRender) {
    const {entity} = dom;
    const {aabb} = entity.render.meshInstances[0];
    const {center, halfExtents} = aabb;
    for (let i=0; i<8; i++) {
      const point = points[i].copy(center);
      point.x += halfExtents.x * ((i & 1) ? 1 : -1); // 1 << 0 = 1
      point.y += halfExtents.y * ((i & 2) ? 1 : -1); // 1 << 1 = 2
      point.z += halfExtents.z * ((i & 4) ? 1 : -1); // 1 << 2 = 4
      camera.worldToScreen(point, tmp);
      point.copy(tmp);
    }
    let l = points[0].x, r = points[0].x; // left / right
    let t = points[0].y, b = points[0].y; // top / bottom
    let z = points[0].z; // z index
    for (let i=1; i<8; i++) {
      l = Math.min(points[i].x, l);
      r = Math.max(points[i].x, r);
      t = Math.min(points[i].y, t);
      b = Math.max(points[i].y, b);
      z = Math.max(points[i].z, z);
    }
    const zIndex = Math.floor((1 / z) * 1000);
    dom.style.position = 'absolute';
    // dom.style.border = '1px solid black';
    // Hold Shift to select elements without pointer events!
    dom.style.pointerEvents = 'none';
    dom.style.left = `${l}px`;
    dom.style.top = `${t}px`;
    dom.style.width = `${r - l}px`;
    dom.style.height = `${b - t}px`;
    dom.style.zIndex = `${zIndex}`;
  }
}
app.on('update', updateInspectorPositions);
// updateInspectorPositions();

Result: