diff --git a/docs/examples/viz_gltf_pbr.py b/docs/examples/viz_gltf_pbr.py new file mode 100644 index 000000000..470962854 --- /dev/null +++ b/docs/examples/viz_gltf_pbr.py @@ -0,0 +1,48 @@ +""" +========================================== +Visualizing a glTF file with PBR materials +========================================== +In this tutorial, we will show how to display a glTF file that uses PBR +materials. +""" + +from fury import window +from fury.data import fetch_gltf, read_viz_gltf +from fury.gltf import glTF + +############################################################################## +# Create a scene. + +scene = window.Scene() +scene.SetBackground(0.19, 0.21, 0.26) + +############################################################################## +# Fetch and read the glTF file 'DamagedHelmet'. +fetch_gltf("DamagedHelmet") +filename = read_viz_gltf("DamagedHelmet") + +############################################################################## +# Create a glTF object from the file and apply normals to the geometry. +gltf_obj = glTF(filename, apply_normals=True) + +############################################################################## +# Extract actors representing the geometry from the glTF object. +actors = gltf_obj.actors() + +############################################################################## +# Add all the actors representing the geometry to the scene. +scene.add(*actors) + +############################################################################## +# Applying camera from the glTF object to the scene. + +cameras = gltf_obj.cameras +if cameras: + scene.SetActiveCamera(cameras[0]) + +interactive = False + +if interactive: + window.show(scene, size=(1280, 720)) + +window.record(scene, out_path="viz_gltf_PBR.png", size=(1280, 720)) diff --git a/fury/gltf.py b/fury/gltf.py index 5b02ff68d..912f46ba9 100644 --- a/fury/gltf.py +++ b/fury/gltf.py @@ -112,10 +112,37 @@ def actors(self): actor.SetUserTransform(_transform) if self.materials[i] is not None: - base_col_tex = self.materials[i]["baseColorTexture"] - actor.SetTexture(base_col_tex) - base_color = self.materials[i]["baseColor"] - actor.GetProperty().SetColor(tuple(base_color[:3])) + pbr = self.materials[i]["pbr"] + if pbr is not None: + base_color = pbr["baseColor"] + actor.GetProperty().SetColor(tuple(base_color[:3])) + + metal = pbr["metallicValue"] + rough = pbr["roughnessValue"] + actor.GetProperty().SetInterpolationToPBR() + actor.GetProperty().SetMetallic(metal) + actor.GetProperty().SetRoughness(rough) + + base_col_tex = pbr["baseColorTexture"] + metal_rough_tex = pbr["metallicRoughnessTexture"] + actor.SetTexture(base_col_tex) + actor.GetProperty().SetBaseColorTexture(base_col_tex) + actor.GetProperty().SetORMTexture(metal_rough_tex) + + emissive = self.materials[i]["emissive"] + if emissive["texture"] is not None: + actor.GetProperty().SetEmissiveTexture(emissive["texture"]) + if emissive["factor"] is not None: + actor.GetProperty().SetEmissiveFactor(emissive["factor"]) + + normal = self.materials[i]["normal"] + if normal["texture"] is not None: + actor.GetProperty().SetNormalTexture(normal["texture"]) + actor.GetProperty().SetNormalScale(normal["scale"]) + + # occlusion = self.materials[i]['occlusion'] + # if occlusion["texture"] is not None: + # actor.GetProperty().SetOcclusionStrength(occlusion["strength"]) self._actors.append(actor) @@ -223,7 +250,7 @@ def load_mesh(self, mesh_id, transform_mat, parent): Mesh index to be loaded transform_mat : ndarray (4, 4) Transformation matrix. - + parent : list """ primitives = self.gltf.meshes[mesh_id].primitives @@ -238,13 +265,20 @@ def load_mesh(self, mesh_id, transform_mat, parent): if attributes.NORMAL is not None and self.apply_normals: normals = self.get_acc_data(attributes.NORMAL) - normals = transform.apply_transformation(normals, transform_mat) utils.set_polydata_normals(polydata, normals) if attributes.TEXCOORD_0 is not None: tcoords = self.get_acc_data(attributes.TEXCOORD_0) utils.set_polydata_tcoords(polydata, tcoords) + if attributes.TANGENT is not None: + tangents = self.get_acc_data(attributes.TANGENT) + utils.set_polydata_tangents(polydata, tangents[:, :3]) + elif attributes.NORMAL is not None and self.apply_normals: + doa = [0, 1, 0.5] + tangents = utils.tangents_from_direction_of_anisotropy(normals, doa) + utils.set_polydata_tangents(polydata, tangents) + if attributes.COLOR_0 is not None: color = self.get_acc_data(attributes.COLOR_0) color = color[:, :-1] * 255 @@ -310,7 +344,11 @@ def get_acc_data(self, acc_id): total_byte_offset = byte_offset + acc_byte_offset buff_array = self.get_buff_array( - buff_id, d_type["dtype"], byte_length, total_byte_offset, byte_stride + buff_id, + d_type["dtype"], + byte_length, + total_byte_offset, + byte_stride, ) return buff_array[:, :a_type] @@ -383,23 +421,67 @@ def get_materials(self, mat_id): """ material = self.gltf.materials[mat_id] - bct = None - + pbr_dict = None pbr = material.pbrMetallicRoughness + if pbr is not None: + bct, orm = None, None + if pbr.baseColorTexture is not None: + bct = pbr.baseColorTexture.index + bct = self.get_texture(bct, True) + if pbr.metallicRoughnessTexture is not None: + mrt = pbr.metallicRoughnessTexture.index + # find if there's any occulsion tex present + occ = material.occlusionTexture + if occ is not None and occ.index == mrt: + orm = self.get_texture(mrt) + else: + mrt = self.get_texture(mrt, rgb=True) + occ_tex = self.get_texture(occ.index, rgb=True) if occ else None + # generate orm texture + orm = self.generate_orm(mrt, occ_tex) + colors = pbr.baseColorFactor + metalvalue = pbr.metallicFactor + roughvalue = pbr.roughnessFactor + pbr_dict = { + "baseColorTexture": bct, + "metallicRoughnessTexture": orm, + "baseColor": colors, + "metallicValue": metalvalue, + "roughnessValue": roughvalue, + } + normal = material.normalTexture + normal_tex = { + "texture": self.get_texture(normal.index) if normal else None, + "scale": normal.scale if normal else 1.0, + } + occlusion = material.occlusionTexture + occ_tex = { + "texture": self.get_texture(occlusion.index) if occlusion else None, + "strength": occlusion.strength if occlusion else 1.0, + } + # must update pbr_dict with ORM texture + emissive = material.emissiveTexture + emi_tex = { + "texture": self.get_texture(emissive.index, True) if emissive else None, + "factor": material.emissiveFactor if emissive else [0, 0, 0], + } - if pbr.baseColorTexture is not None: - bct = pbr.baseColorTexture.index - bct = self.get_texture(bct) - colors = pbr.baseColorFactor - return {"baseColorTexture": bct, "baseColor": colors} + return { + "pbr": pbr_dict, + "normal": normal_tex, + "occlusion": occ_tex, + "emissive": emi_tex, + } - def get_texture(self, tex_id): + def get_texture(self, tex_id, srgb_colorspace=False, rgb=False): """Read and convert image into vtk texture. Parameters ---------- tex_id : int Texture index + srgb_colorspace : bool + Use vtkSRGB colorspace. (default=False) Returns ------- @@ -441,8 +523,51 @@ def get_texture(self, tex_id): else: image_path = os.path.join(self.pwd, file) - rgb = io.load_image(image_path) - grid = utils.rgb_to_vtk(np.flipud(rgb)) + rgb_array = io.load_image(image_path) + if rgb: + return rgb_array + grid = utils.rgb_to_vtk(np.flipud(rgb_array)) + atexture = Texture() + atexture.InterpolateOn() + atexture.EdgeClampOn() + atexture.SetInputDataObject(grid) + if srgb_colorspace: + atexture.UseSRGBColorSpaceOn() + atexture.Update() + + return atexture + + def generate_orm(self, metallic_roughness=None, occlusion=None): + """Generates ORM texture from O, R & M textures. + We do this by swapping Red channel of metallic_roughness with the + occlusion texture and adding metallic to Blue channel. + + Parameters + ---------- + metallic_roughness : ndarray + occlusion : ndarray + """ + shape = metallic_roughness.shape + rgb_array = np.copy(metallic_roughness) + # metallic is red if name starts as metallicRoughness, otherwise its + # in the green channel + # https://github.com/KhronosGroup/glTF/issues/857#issuecomment-290530762 + metal_arr = metallic_roughness[:, :, 2] + rough_arr = metallic_roughness[:, :, 1] + if occlusion is None: + occ_arr = np.full((shape[0], shape[1]), 256) + # print(occ_arr) + else: + if len(list(occlusion.shape)) > 2: + # occ_arr = np.dot(occlusion, + # np.array([0.2989, 0.5870, 0.1140])) + occ_arr = occlusion.sum(2) / 3 + # both equation grayscales but second one is less computation. + rgb_array[:, :, 0][:] = metal_arr # blue channel + rgb_array[:, :, 1][:] = rough_arr + rgb_array[:, :, 2][:] = occ_arr # red channel + + grid = utils.rgb_to_vtk(rgb_array) atexture = Texture() atexture.InterpolateOn() atexture.EdgeClampOn() @@ -722,7 +847,11 @@ def update_skin(self, animation): parent_transform = np.identity(4) for child in _animation.child_animations: self.transverse_animations( - child, self.bones[0], timestamp, joint_matrices, parent_transform + child, + self.bones[0], + timestamp, + joint_matrices, + parent_transform, ) for i, vertex in enumerate(self._vertices): vertex[:] = self.apply_skin_matrix(self._vcopy[i], joint_matrices, i) @@ -1004,17 +1133,26 @@ def get_animations(self): if prop == "rotation": animation.set_rotation( - time[0], trs, in_tangent=in_tan, out_tangent=out_tan + time[0], + trs, + in_tangent=in_tan, + out_tangent=out_tan, ) animation.set_rotation_interpolator(rot_interp) if prop == "translation": animation.set_position( - time[0], trs, in_tangent=in_tan, out_tangent=out_tan + time[0], + trs, + in_tangent=in_tan, + out_tangent=out_tan, ) animation.set_position_interpolator(interpolator) if prop == "scale": animation.set_scale( - time[0], trs, in_tangent=in_tan, out_tangent=out_tan + time[0], + trs, + in_tangent=in_tan, + out_tangent=out_tan, ) animation.set_scale_interpolator(interpolator) else: @@ -1060,7 +1198,6 @@ def export_scene(scene, filename="default.gltf"): primitives = [] buffer_size = 0 bview_count = 0 - for act in scene.GetActors(): prim, size, count = _connect_primitives( gltf_obj, act, buffer_file, buffer_size, bview_count, name @@ -1151,7 +1288,12 @@ def _connect_primitives(gltf, actor, buff_file, byteoffset, count, name): buff_file.write(indices.tobytes()) write_bufferview(gltf, 0, byteoffset, blength) write_accessor( - gltf, count, 0, gltflib.UNSIGNED_SHORT, len(indices), gltflib.SCALAR + gltf, + count, + 0, + gltflib.UNSIGNED_SHORT, + len(indices), + gltflib.SCALAR, ) byteoffset += blength index = count @@ -1410,7 +1552,14 @@ def write_material(gltf, basecolortexture: int, uri: str): def write_accessor( - gltf, bufferview, byte_offset, comp_type, count, accssor_type, max=None, min=None + gltf, + bufferview, + byte_offset, + comp_type, + count, + accssor_type, + max=None, + min=None, ): """Write accessor in the gltf. diff --git a/fury/tests/test_gltf.py b/fury/tests/test_gltf.py index 8e9febe2f..70e253b08 100644 --- a/fury/tests/test_gltf.py +++ b/fury/tests/test_gltf.py @@ -1,3 +1,4 @@ +from collections import Counter import itertools import os @@ -53,21 +54,24 @@ def test_load_texture(): scene.add(actor) display = window.snapshot(scene) res = window.analyze_snapshot( - display, bg_color=(0, 0, 0), colors=[(255, 216, 0)], find_objects=False + display, + bg_color=(0, 0, 0), + colors=[(149, 126, 19), (136, 115, 19), (143, 121, 19)], + find_objects=False, ) - npt.assert_equal(res.colors_found, [True]) + npt.assert_equal(res.colors_found, [True, True, True]) scene.clear() -@pytest.mark.skipif(True, reason="This test is failing on CI, not sure why yet") +@pytest.mark.skipif(True, reason="Failing! Needs more investigation") def test_colors(): # vertex colors fetch_gltf("BoxVertexColors") file = read_viz_gltf("BoxVertexColors", "glTF") importer = glTF(file) - actor = importer.actors()[0] + imp_actor = importer.actors()[0] scene = window.Scene() - scene.add(actor) + scene.add(imp_actor) display = window.snapshot(scene) res = window.analyze_snapshot( display, @@ -105,13 +109,17 @@ def test_orientation(): # if oriented correctly avg of blues on top half will be greater # than the bottom half display = window.snapshot(scene) + + # save screenshot + img = Image.fromarray(display) + img.save("test_orientation.png") res = window.analyze_snapshot( display, bg_color=(0, 0, 0), - colors=[(108, 173, 223), (92, 135, 39)], + colors=[(128, 128, 128), (56, 79, 30), (64, 101, 130)], find_objects=False, ) - npt.assert_equal(res.colors_found, [True, True]) + npt.assert_equal(res.colors_found, [True, True, True]) blue = display[:, :, -1:].reshape((300, 300)) upper, lower = np.split(blue, 2) upper = np.mean(upper) @@ -168,20 +176,23 @@ def test_export_gltf(): box_actor = gltf_obj.actors() scene.add(*box_actor) export_scene(scene, "test.gltf") + display_1 = window.snapshot(scene) scene.clear() gltf_obj = glTF("test.gltf") actors = gltf_obj.actors() scene.add(*actors) - display = window.snapshot(scene) - res = window.analyze_snapshot( - display, - bg_color=(0, 0, 0), - colors=[(108, 173, 223), (92, 135, 39)], - find_objects=False, - ) - npt.assert_equal(res.colors_found, [True, True]) + display_2 = window.snapshot(scene) + + colors_display_1 = Counter([tuple(color) for color in display_1.reshape(-1, 3)]) + colors_display_2 = Counter([tuple(color) for color in display_2.reshape(-1, 3)]) + is_equal_colors = colors_display_1.most_common(3) == colors_display_2.most_common(3) + + # TODO: Test for image similarity instead of color + # similarity after fixing the issue with exporting + # inverted textures + npt.assert_equal(is_equal_colors, True) def test_simple_animation(): @@ -203,16 +214,19 @@ def test_simple_animation(): timeline.seek(2.57) showm.save_screenshot("keyframe2.png") - res1 = window.analyze_snapshot( - "keyframe1.png", colors=[(77, 136, 204), (204, 106, 203)] + + res_1 = window.analyze_snapshot( + "keyframe1.png", colors=[(87, 112, 134), (134, 100, 133)] ) - res2 = window.analyze_snapshot( - "keyframe2.png", colors=[(77, 136, 204), (204, 106, 203)] + res_2 = window.analyze_snapshot( + "keyframe2.png", colors=[(87, 112, 134), (134, 100, 133)] ) - - assert_greater(res2.objects, res1.objects) - npt.assert_equal(res1.colors_found, [True, False]) - npt.assert_equal(res2.colors_found, [True, True]) + # inside box adds more colors + assert_greater(res_2.objects, res_1.objects) + # bluish box should exist in both images, but + # not the purple one in the first image + npt.assert_equal(res_1.colors_found, [True, False]) + npt.assert_equal(res_2.colors_found, [True, True]) def test_skinning():