Repository navigation
VisionPro cannot display the image in real time #42
Description
I changed the code based on IsaacGym in this project to IsaacSim. Everything looks normal from the simulation interface GUI, but when I map it to VisionPro, only one picture can be displayed, and the picture cannot change the perspective, so I would like to seek help from everyone. Here is my camera setting code:
class SceneCfg(InteractiveSceneCfg):
terrain = TerrainImporterCfg(
prim_path="/World/ground",
terrain_type="plane",
# collision_group=-1,
# physics_material=sim_utils.RigidBodyMaterialCfg(static_friction=1.0, dynamic_friction=1.0, restitution=0.0),
# debug_vis=False
)
robot = Wow_CFG.replace(prim_path="/World/envs/env_.*/Robot")
# scene = AssetBaseCfg(
# prim_path="/World/envs/env_.*/table",
# spawn=sim_utils.UsdFileCfg(
# # usd_path=ROOT_PATH+"/assets/usd_material/table/white_big/instance.usda",
# usd_path=ROOT_PATH+"/assets/usd_material/moon_cake/scene.usd",
# scale=(0.001, 0.001, 0.001),
# # scale=(1, 1, 1),
# rigid_props=sim_utils.RigidBodyPropertiesCfg(
# kinematic_enabled=True,
# disable_gravity=False,
# # enable_gyroscopic_forces=True,
# solver_position_iteration_count=8,
# solver_velocity_iteration_count=1,
# max_depenetration_velocity=5.0,
# # sleep_threshold=0.005,
# # stabilization_threshold=0.0025
# )
# ),
# init_state=RigidObjectCfg.InitialStateCfg(pos=[0.474, 0, 0], rot=[1, 0, 0, 0])
# # init_state=RigidObjectCfg.InitialStateCfg(pos=[0, 0, 0.474], rot=[1, 0, 0, 0])
# )
table = AssetBaseCfg(
prim_path = "/World/Objects/Table",
spawn = sim_utils.UsdFileCfg(
usd_path = f"{ISAAC_NUCLEUS_DIR}/Props/Mounts/SeattleLabTable/table_instanceable.usd",
scale = (0.5, 0.5, 0.5)
),
init_state=RigidObjectCfg.InitialStateCfg(pos=[0.5, 0, 0.5], rot=[1, 0, 0, 0])
)
cuboid = AssetBaseCfg(
prim_path = "/World/Objects/Cuboid",
spawn = sim_utils.CuboidCfg(
size = (0.1,0.1,0.1),
rigid_props=sim_utils.RigidBodyPropertiesCfg(),
mass_props=sim_utils.MassPropertiesCfg(mass=1.0),
collision_props=sim_utils.CollisionPropertiesCfg(),
visual_material=sim_utils.PreviewSurfaceCfg(diffuse_color=(0.0, 1.0, 0.0), metallic=0.2),
),
init_state=RigidObjectCfg.InitialStateCfg(pos=[0.5, 0, 0.7], rot=[1, 0, 0, 0])
)
# camera_right = CameraCfg(
# height=720,
# width=1280,
# offset=CameraCfg.OffsetCfg(pos=(0.15, 0.04, 0.1), rot=(0.92388, 0.0, 0.38268, 0.0), convention='world'), # wxyz in isaacsim
# # offset=CameraCfg.OffsetCfg(pos=(0.15, -0.06, 0.1), rot=(0.92388, 0.0, 0.38268, 0.0), convention='world'), # wxyz in isaacsim
# prim_path='/World/envs/env_.*/Robot/neck_yaw_Link/CameraRight',
# update_period=0.1,
# data_types=['rgb'],#,'depth'],
# spawn=sim_utils.PinholeCameraCfg(
# focal_length=1.5, focus_distance=10.0, horizontal_aperture=2, clipping_range=(0.05, 15)
# ),
# )
# camera_left = CameraCfg(
# height=720,
# width=1280,
# offset=CameraCfg.OffsetCfg(pos=(0.15, 0.16, 0.1), rot=(0.92388, 0.0, 0.38268, 0.0), convention='world'), # wxyz in isaacsim
# prim_path='/World/envs/env_.*/Robot/neck_yaw_Link/CameraLeft',
# update_period=0.1,
# data_types=['rgb'],#,'depth'],
# spawn=sim_utils.PinholeCameraCfg(
# focal_length=1.5, focus_distance=10.0, horizontal_aperture=2, clipping_range=(0.05, 15)
# ),
# )
camera_right = TiledCameraCfg(
prim_path='/World/envs/env_.*/Robot/neck_yaw_Link/CameraRight',
update_period=0.1,
offset=TiledCameraCfg.OffsetCfg(pos=(0.15, 0.04, 0.1), rot=(0.92388, 0.0, 0.38268, 0.0), convention="world"),
data_types=["rgb"],
spawn=sim_utils.PinholeCameraCfg(
focal_length=1.5, focus_distance=10.0, horizontal_aperture=2, clipping_range=(0.05, 15)
# ),
),
width=1280,
height=720,
)
camera_left = TiledCameraCfg(
prim_path='/World/envs/env_.*/Robot/neck_yaw_Link/CameraLeft',
update_period=0.1,
offset=TiledCameraCfg.OffsetCfg(pos=(0.15, 0.16, 0.1), rot=(0.92388, 0.0, 0.38268, 0.0), convention="world"),
data_types=["rgb"],
spawn=sim_utils.PinholeCameraCfg(
focal_length=1.5, focus_distance=10.0, horizontal_aperture=2, clipping_range=(0.05, 15)
# ),
),
width=1280,
height=720,
)
light = AssetBaseCfg(
prim_path="/World/light",
spawn=sim_utils.DistantLightCfg(color=(0.75, 0.75, 0.75), intensity=3000.0),
)
At first I thought the camera was fixed so the perspective couldn't be changed, but I compared it to the sample and found that there was no problem. The following is the code of OpenTeleVision class:
class OpenTeleVision:
def __init__(self, img_shape, shm_name, queue, toggle_streaming, stream_mode="image", cert_file="./cert.pem", key_file="./key.pem"):
# self.app=Vuer()
self.logger = setup_logging()
self.img_shape = (img_shape[0], 2*img_shape[1], 3)
self.img_height, self.img_width = img_shape[:2]
self.app = Vuer(host='0.0.0.0', cert=cert_file, key=key_file, queries=dict(grid=False))
self.app.add_handler("HAND_MOVE")(self.on_hand_move)
self.app.add_handler("CAMERA_MOVE")(self.on_cam_move)
if stream_mode == "image":
existing_shm = shared_memory.SharedMemory(name=shm_name)
self.img_array = np.ndarray((self.img_shape[0], self.img_shape[1], 3), dtype=np.uint8, buffer=existing_shm.buf)
self.app.spawn(start=False)(self.main_image)
elif stream_mode == "webrtc":
self.app.spawn(start=False)(self.main_webrtc)
else:
raise ValueError("stream_mode must be either 'webrtc' or 'image'")
self.left_hand_shared = Array('d', 16, lock=True)
self.right_hand_shared = Array('d', 16, lock=True)
self.left_landmarks_shared = Array('d', 75, lock=True)
self.right_landmarks_shared = Array('d', 75, lock=True)
self.head_matrix_shared = Array('d', 16, lock=True)
self.aspect_shared = Value('d', 1.0, lock=True)
self.begin_move = Value('b', False, lock=True)
if stream_mode == "webrtc":
# webrtc server
if Args.verbose:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(level=logging.INFO)
Args.img_shape = img_shape
# Args.shm_name = shm_name
Args.fps = 60
ssl_context = ssl.SSLContext()
ssl_context.load_cert_chain(cert_file, key_file)
app = web.Application()
cors = aiohttp_cors.setup(app, defaults={
"*": aiohttp_cors.ResourceOptions(
allow_credentials=True,
expose_headers="*",
allow_headers="*",
allow_methods="*",
)
})
rtc = RTC(img_shape, queue, toggle_streaming, 60)
app.on_shutdown.append(on_shutdown)
cors.add(app.router.add_get("/", index))
cors.add(app.router.add_get("/client.js", javascript))
cors.add(app.router.add_post("/offer", rtc.offer))
self.webrtc_process = Process(target=web.run_app, args=(app,), kwargs={"host": "0.0.0.0", "port": 8080, "ssl_context": ssl_context})
self.webrtc_process.daemon = True
self.webrtc_process.start()
# web.run_app(app, host="0.0.0.0", port=8080, ssl_context=ssl_context)
self.process = Process(target=self.run)
self.process.daemon = True
self.process.start()
Activity