[Interactive Camera] How to get the image data?

[MingshanHe]

Hi! I am working on the image process within sofa. I know how to control the camera state, like position and orientation. But I don't know how to get the image data from the camera. Please help me with this, thank you.

[fredroy]

Hello,
It depends on how you use SOFA (XML/C++, or with python3?).
There is no direct way to access the framebuffer of the current camera as it is not linked to any graphical API (e.g OpenGL), and as far as I know there is no component directly in SOFA who can do that.
if python3, are you using your own GUI ? you can have a look on how to do it from a python3 perspective here : https://github.com/sofa-framework/SofaPython3/blob/master/examples/additional-examples/pyqt_example.py

[MingshanHe]

Thanks for your answer. I have looked at the code which is mentioned by you and run it. But I have no got idea about this black screen. Have you ever been in this situation?

[fredroy]

Not really, as I dont really use python3.
I tried the other scene using pygame (in the same directory) and it seems to work/display something.
So if you modify a little bit the script, you can get for example the depth buffer
Here is a quick&dirty update of the pygame_example.py
It will save the pixels of the depth buffer on the disk.

import Sofa
import Sofa.SofaGL
import SofaRuntime
import Sofa.Simulation as sim
import os
import time
sofa_directory = os.environ['SOFA_ROOT']
import pygame
from OpenGL.GL import *
from OpenGL.GLU import *
import numpy as np
from PIL import Image

display_size = (800, 600)


def init_display(node):
    pygame.display.init()
    pygame.display.set_mode(display_size, pygame.DOUBLEBUF | pygame.OPENGL)

    glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
    glEnable(GL_LIGHTING)
    glEnable(GL_DEPTH_TEST)
    Sofa.SofaGL.glewInit()
    Sofa.Simulation.initVisual(node)
    Sofa.Simulation.initTextures(node)
    glMatrixMode(GL_PROJECTION)
    glLoadIdentity()
    gluPerspective(45, (display_size[0] / display_size[1]), 0.1, 50.0)

    glMatrixMode(GL_MODELVIEW)
    glLoadIdentity()

def simple_render(rootNode):
    """
     Get the OpenGL Context to render an image (snapshot) of the simulation state
     """
    glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
    glEnable(GL_LIGHTING)
    glEnable(GL_DEPTH_TEST)
    glMatrixMode(GL_PROJECTION)
    glLoadIdentity()
    gluPerspective(45, (display_size[0] / display_size[1]), 0.1, 50.0)
    glMatrixMode(GL_MODELVIEW)
    glLoadIdentity()

    cameraMVM = rootNode.camera.getOpenGLModelViewMatrix()
    glMultMatrixd(cameraMVM)
    Sofa.SofaGL.draw(rootNode)

    pygame.display.get_surface().fill((0,0,0))
    pygame.display.flip()

    _, _, width, height = glGetIntegerv(GL_VIEWPORT)
    buff = glReadPixels(0, 0, width, height, GL_DEPTH_COMPONENT, GL_FLOAT)
    image = np.frombuffer(buff, dtype=np.float32)
    image = image.reshape(height, width)
    image = np.flipud(image)  #<-- image is now a numpy array you can use


    z_far = rootNode.camera.findData('zFar').value
    z_near = rootNode.camera.findData('zNear').value
    depth_image = -z_far*z_near/(z_far + image*(z_near-z_far))
    depth_image = (depth_image - depth_image.min()) / (depth_image.max() - depth_image.min())
    depth_image = depth_image * 255
    print(depth_image.min())
    print(depth_image.max())
    print(depth_image.size)

    img2 = Image.fromarray(depth_image.astype(np.uint8), 'L')
    img2.save("/home/fred/test.bmp")


def createScene(root):
    # Register all the common component in the factory.
    SofaRuntime.PluginRepository.addFirstPath(os.path.join(sofa_directory, 'bin'))
    SofaRuntime.importPlugin("Sofa.Component")
    SofaRuntime.importPlugin('Sofa.GL.Component')


    ### these are just some things that stay still and move around
    # so you know the animation is actually happening
    root.gravity = [0, -1., 0]
    root.addObject("VisualStyle", displayFlags="showAll")
    root.addObject("MeshGmshLoader", name="meshLoaderCoarse",
                   filename="mesh/liver.msh")
    root.addObject("MeshObjLoader", name="meshLoaderFine",
                   filename="mesh/liver-smooth.obj")

    root.addObject("EulerImplicitSolver")
    root.addObject("CGLinearSolver", iterations="200",
                   tolerance="1e-09", threshold="1e-09")

    liver = root.addChild("liver")

    liver.addObject("TetrahedronSetTopologyContainer",
                    name="topo", src="@../meshLoaderCoarse")
    liver.addObject("TetrahedronSetGeometryAlgorithms",
                    template="Vec3d", name="GeomAlgo")
    liver.addObject("MechanicalObject",
                    template="Vec3d",
                    name="MechanicalModel", showObject="1", showObjectScale="3")

    liver.addObject("TetrahedronFEMForceField", name="fem", youngModulus="1000",
                    poissonRatio="0.4", method="large")

    liver.addObject("MeshMatrixMass", massDensity="1")
    liver.addObject("FixedConstraint", indices="2 3 50")
    visual = liver.addChild("visual")
    visual.addObject('MeshObjLoader', name="meshLoader_0", filename="mesh/liver-smooth.obj", handleSeams="1")
    visual.addObject('OglModel', name="VisualModel", src="@meshLoader_0", color='red')
    visual.addObject('BarycentricMapping', input="@..", output="@VisualModel", name="visual mapping")

    # place light and a camera
    root.addObject("LightManager")
    root.addObject("DirectionalLight", direction=[0,1,0])
    root.addObject("InteractiveCamera", name="camera", position=[0,15, 0],
                            lookAt=[0,0,0], distance=37,
                            fieldOfView=45, zNear=0.63, zFar=55.69)


if __name__ == '__main__':
    root = Sofa.Core.Node("myroot")
    createScene(root)
    Sofa.Simulation.init(root)
    init_display(root)
    try:
        while True:
            Sofa.Simulation.animate(root, root.getDt())
            Sofa.Simulation.updateVisual(root)
            simple_render(root)
            time.sleep(root.getDt())
    except KeyboardInterrupt:
        pass

[MingshanHe]

Thanks for your work. I will test it soon.
Best,
Mingshan

[yang-si-hang]

Hi, @fredroy. Your code is excellent and has provided me with some insights. However, I'm curious why, after switching from MeshObjLoader to MeshVTKLoader (with the file renamed to liver.vtk), PyGame displays a black screen with no content?

Any suggestions would be greatly appreciated.

[spbnb]

Hi，how can I control my beam using the onKeyPressedEvent from Beamadapter while rendering with pygame