From 830dc9269d7d5b5dcef688cd9a567a3267be938d Mon Sep 17 00:00:00 2001 From: Timo Korthals Date: Mon, 6 Jan 2025 16:14:13 +0100 Subject: [PATCH] Feature/fisheye (#18) * Add fisheye cameras Signed-off-by: Timo Korthals * Fix fisheye parameters for cycles Signed-off-by: Timo Korthals * chore: Update camera.schema.yaml with enum for lense_type and $ref for fisheye_fov * chore: Update camera.py to use "lens" instead of "lense" and fix typo in error message * Update camera.py to allow for dynamic change of lens parameter during job * Add fisheye camera configuration and update schema for lens types * Update camera documentation to include lens types and parameters * Enhance camera schema: set default lens type to PERSPECTIVE and update required fields for fisheye lens types * Add fisheye camera output paths to integration test workflow * Rename `scale_std` to `scale_standard_deviation` in documentation and configuration files for consistency * Bump version to 1.4.0 in pyproject.toml * Fix typo in simulated scatter schema: correct `scalscale_standard_deviatione_std` to `scale_standard_deviation` * Remove unused `res` parameter from perlin operation in test job configuration * Add structured light output schema and update camera schema references --------- Signed-off-by: Timo Korthals Co-authored-by: Anton Elmiger --- .github/workflows/integration_test.yaml | 2 + .../config_descriptions/camera.md | 28 ++++++- .../config_descriptions/simulated_scatter.md | 6 +- pyproject.toml | 2 +- .../__example_assets__/test_job.syclops.yaml | 19 ++++- .../schema/simulated_scatter.schema.yaml | 4 +- syclops/blender/plugins/simulated_scatter.py | 4 +- .../schema/structured_light.schema.yaml | 26 +++++++ syclops/blender/sensors/camera.py | 74 ++++++++++++++----- .../blender/sensors/schema/camera.schema.yaml | 35 +++++++-- 10 files changed, 163 insertions(+), 37 deletions(-) create mode 100644 syclops/blender/sensor_outputs/schema/structured_light.schema.yaml diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 142e7cf..d1cd1a7 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -60,6 +60,8 @@ jobs: "./output/*/main_camera_annotations/structured_light/*metadata.yaml" "./output/*/object_positions/*.json" "./output/*/object_positions/*metadata.yaml" + "./output/*/fisheye_camera/rect/*.png" + "./output/*/fisheye_camera/rect/*metadata.yaml" ) missing=false diff --git a/docs/docs/usage/job_description/config_descriptions/camera.md b/docs/docs/usage/job_description/config_descriptions/camera.md index 4229b85..cb2dbdb 100644 --- a/docs/docs/usage/job_description/config_descriptions/camera.md +++ b/docs/docs/usage/job_description/config_descriptions/camera.md @@ -1,6 +1,6 @@ # Camera Plugin Documentation -The Camera Plugin simulates a basic camera sensor, allowing you to configure the optical and digital properties of the camera within your scene. It supports various parameters such as resolution, focal length, exposure, depth of field, motion blur, and frustum visualization for debugging purposes. Additionally, it outputs the intrinsic and extrinsic camera parameters. +The Camera Plugin simulates a basic camera sensor, allowing you to configure the optical and digital properties of the camera within your scene. It supports various parameters such as resolution, lens types (perspective and fisheye), exposure, depth of field, motion blur, and frustum visualization for debugging purposes. Additionally, it outputs the intrinsic and extrinsic camera parameters. ## Configuration Parameters @@ -11,7 +11,9 @@ The following table describes each configuration parameter for the Camera Plugin | `name` | string | Unique identifier of the sensor. | **Required** | | `frame_id` | string | Transformation tree node the camera attaches to. | **Required** | | `resolution` | array (2 integers: width x height) | Width and height of the camera in pixels. | **Required** | -| `focal_length` | float | Focal length of the camera in mm. | **Required** | +| `lens_type` | string | Type of lens projection (PERSPECTIVE [default], FISHEYE_EQUISOLID, FISHEYE_EQUIDISTANT). | Optional | +| `focal_length` | float | Focal length of the camera in mm. Required for PERSPECTIVE and FISHEYE_EQUISOLID. | Conditional | +| `fisheye_fov` | float | Horizontal angular field of view in radians. Required for FISHEYE_EQUIDISTANT and FISHEYE_EQUISOLID. | Conditional | | `sensor_width` | float | Width of the sensor in mm. | **Required** | | `exposure` | float | Exposure offset in stops. | Optional | | `gamma` | float | Gamma correction applied to the image (1 means no change in gamma). | Optional | @@ -62,6 +64,28 @@ The following table describes each configuration parameter for the Camera Plugin | `enabled` | boolean | Whether to render as wireframe lines. | | `thickness` | number | Thickness of the wireframe lines. | +### Lens Types and Parameters + +The camera supports three types of lens projections: + +1. **PERSPECTIVE** (default) + - Requires `focal_length` + - Standard perspective projection + +2. **FISHEYE_EQUISOLID** + - Requires both `focal_length` and `fisheye_fov` + - Follows the equisolid angle projection formula + - Commonly used in real fisheye lenses + +3. **FISHEYE_EQUIDISTANT** + - Requires `fisheye_fov` + - Linear mapping between angle and image distance + - Theoretical fisheye projection + +!!! note + When using fisheye lens types, the frustum visualization is not supported and will be disabled automatically. + Camera extrinsic parameters are not output when using fisheye lens types as they are not supported. + !!! warning If `motion_blur` is enabled, `shutter_speed` becomes a required parameter. diff --git a/docs/docs/usage/job_description/config_descriptions/simulated_scatter.md b/docs/docs/usage/job_description/config_descriptions/simulated_scatter.md index 77f291a..9a7f83e 100644 --- a/docs/docs/usage/job_description/config_descriptions/simulated_scatter.md +++ b/docs/docs/usage/job_description/config_descriptions/simulated_scatter.md @@ -15,13 +15,13 @@ The following table describes each configuration parameter for the Simulated Sca | `decimate_mesh_factor` | number (0-1) | Factor between 0-1 that decimates the number of vertices of the mesh. Lower means less vertices. | Optional | | `density` | number | Density of objects per square meter. | **Required** | | `density_texture` | image/texture evaluation | Texture that alters the density per pixel. Needs to be a single channel image that is normalized to 0-1. | Optional | -| `scale_std` | number | Standard deviation of the scale randomization. | **Required** | +| `scale_standard_deviation` | number | Standard deviation of the scale randomization. | **Required** | | `convex_decomposition_quality` | integer (1-100) | Quality setting for the convex decomposition. Higher means more accurate but slower. | **Required** | | `simulation_steps` | integer | Number of simulation steps to run. | **Required** | ### Dynamic Evaluators -Parameters like `density_texture` and `scale_std` can be dynamically evaluated. This means that their values can be altered for each new frame. For more insights on dynamic evaluators and how to use them, kindly refer to [Dynamic Evaluators](../dynamic_evaluators.md). +Parameters like `density_texture` and `scale_standard_deviation` can be dynamically evaluated. This means that their values can be altered for each new frame. For more insights on dynamic evaluators and how to use them, kindly refer to [Dynamic Evaluators](../dynamic_evaluators.md). ## Example Configuration @@ -33,7 +33,7 @@ scene: floor_object: "Ground" max_texture_size: 2048 density: 5 - scale_std: 0.3 + scale_standard_deviation: 0.3 convex_decomposition_quality: 90 simulation_steps: 100 ``` diff --git a/pyproject.toml b/pyproject.toml index 250f858..365f66e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ readme = "README.md" requires-python = ">=3.8" license = {text = "GPLv3"} -version = "1.3.10" +version = "1.4.0" dynamic = ["dependencies"] diff --git a/syclops/__example_assets__/test_job.syclops.yaml b/syclops/__example_assets__/test_job.syclops.yaml index 22aa8b8..79043dc 100644 --- a/syclops/__example_assets__/test_job.syclops.yaml +++ b/syclops/__example_assets__/test_job.syclops.yaml @@ -32,7 +32,6 @@ textures: num_textures: 2 ops: - perlin: - res: 4 octaves: 4 - math_expression: "((x-0.5) * 100 + 65535 / 2)/65535" @@ -57,7 +56,7 @@ scene: floor_object: Ground class_id: 1 simulation_steps: 5 - std_scale: 1 + scale_standard_deviation: 1 density: 0.01 convex_decomposition_quality: 90 @@ -107,7 +106,7 @@ sensor: - name: "main_camera" # Location, rotation and velocity of camera is optional if frame_id is set frame_id: "camera_link" - resolution: [1280, 960] + resolution: [256, 256] focal_length: 65 # mm shutter_speed: 0.02 # s Currently only affects motion blur sensor_width: 35 # mm @@ -154,6 +153,20 @@ sensor: intensity: 10000 scale: 200 samples: 4 + - name: "fisheye_camera" + frame_id: "camera_link" + resolution: [256, 256] + fisheye_fov: 1.8 + lens_type: "FISHEYE_EQUIDISTANT" + sensor_width: 35 # mm + outputs: + syclops_output_rgb: + - samples: 4 + compositor: + chromatic_aberration: 0.007 #Strong aberration can cause shift between ground truth and rgb + bloom: + threshold: 0.99 # higher is less bloom + id: main_cam_rgb postprocessing: syclops_postprocessing_bounding_boxes: diff --git a/syclops/blender/plugins/schema/simulated_scatter.schema.yaml b/syclops/blender/plugins/schema/simulated_scatter.schema.yaml index dd1646b..08c5490 100644 --- a/syclops/blender/plugins/schema/simulated_scatter.schema.yaml +++ b/syclops/blender/plugins/schema/simulated_scatter.schema.yaml @@ -28,7 +28,7 @@ items: density_texture: description: Texture that alters the density per pixel. Needs to be a single channel image that is normalized to 0-1. $ref: "#/definitions/image_texture_evaluation" - scale_std: + scale_standard_deviation: description: Standard deviation of the scale randomization. type: number convex_decomposition_quality: @@ -44,7 +44,7 @@ items: models, floor_object, density, - scale_std, + scale_standard_deviation, convex_decomposition_quality, simulation_steps, ] diff --git a/syclops/blender/plugins/simulated_scatter.py b/syclops/blender/plugins/simulated_scatter.py index b4a8320..093f3af 100644 --- a/syclops/blender/plugins/simulated_scatter.py +++ b/syclops/blender/plugins/simulated_scatter.py @@ -42,8 +42,8 @@ def _simulate_convex_objects(self, scatter_points: np.array): conv_hulls = utility.filter_objects("PARENT_UUID", parent_uuid) random_rotation = Vector(np.random.uniform(0, 2 * np.pi, size=3)) random_scale_value = ( - np.random.normal(1, self.config["scale_std"]) - if "scale_std" in self.config + np.random.normal(1, self.config["scale_standard_deviation"]) + if "scale_standard_deviation" in self.config else 1 ) random_scale = Vector([random_scale_value] * 3) diff --git a/syclops/blender/sensor_outputs/schema/structured_light.schema.yaml b/syclops/blender/sensor_outputs/schema/structured_light.schema.yaml new file mode 100644 index 0000000..e394f4b --- /dev/null +++ b/syclops/blender/sensor_outputs/schema/structured_light.schema.yaml @@ -0,0 +1,26 @@ +description: Structured light pattern output from a camera with a projected pattern. +type: array +items: + type: object + properties: + id: + description: Unique identifier of the output + type: string + frame_id: + description: Frame ID to attach the structured light source to + type: string + samples: + description: Render quality of the image. More means a better quality and more samples per pixel. + type: integer + intensity: + description: Light intensity of the structured light pattern + type: number + scale: + description: Scale factor of the Voronoi pattern + type: number + debug_breakpoint: + description: Whether to break and open Blender before rendering. Only works if scene debugging is active. + type: boolean + required: [id, frame_id, samples, intensity, scale] +minItems: 1 +maxItems: 1 diff --git a/syclops/blender/sensors/camera.py b/syclops/blender/sensors/camera.py index ea27cf8..598c618 100644 --- a/syclops/blender/sensors/camera.py +++ b/syclops/blender/sensors/camera.py @@ -16,9 +16,7 @@ class Camera(SensorInterface): def setup_sensor(self): self.create_camera() # Create self.objs - for obj in self.objs: - if obj.get().type == "CAMERA": - cam = obj.get() + cam = self.get_camera() cam["name"] = self.config["name"] if "depth_of_field" in self.config: cam.data.dof.use_dof = True @@ -30,9 +28,7 @@ def create_frustum_pyramid(self): """Create sensor's frustum as pyramid""" # Get cam object - for obj in self.objs: - if obj.get().type == "CAMERA": - cam = obj.get() + cam = self.get_camera() cam_matrix = cam.matrix_world.normalized() scene = bpy.context.scene @@ -131,6 +127,11 @@ def create_frustum(self): """Create sensor's frustum""" if "frustum" in self.config: if self.config["frustum"]["enabled"]: + if self.get_camera().data.type != "PERSP": + logging.warning( + "Camera: create_frustum not supported for non-perspective cameras" + ) + return if self.config["frustum"]["type"] == "pyramid": self.create_frustum_pyramid() else: @@ -150,9 +151,7 @@ def render_outputs(self): scene.render.resolution_x = self.config["resolution"][0] scene.render.resolution_y = self.config["resolution"][1] # Set camera as active camera - for obj in self.objs: - if obj.get().type == "CAMERA": - cam = obj.get() + cam = self.get_camera() scene.camera = cam if cam.data.dof.use_dof: @@ -168,9 +167,22 @@ def render_outputs(self): cam.data.dof.focus_distance, ) + if "lens_type" in self.config: + lens_type = utility.eval_param(self.config["lens_type"]) + else: + lens_type = "PERSPECTIVE" + # Set camera settings - cam.data.lens = utility.eval_param(self.config["focal_length"]) cam.data.sensor_width = utility.eval_param(self.config["sensor_width"]) + if lens_type == "PERSPECTIVE": + cam.data.lens = utility.eval_param(self.config["focal_length"]) + elif lens_type == "FISHEYE_EQUIDISTANT": + cam.data.cycles.fisheye_fov = utility.eval_param(self.config["fisheye_fov"]) + elif lens_type == "FISHEYE_EQUISOLID": + cam.data.cycles.fisheye_lens = utility.eval_param(self.config["focal_length"]) + cam.data.cycles.fisheye_fov = utility.eval_param(self.config["fisheye_fov"]) + else: + raise ValueError("Camera: not supported lens type \"%s\"", lens_type) if "motion_blur" in self.config: if self.config["motion_blur"]["enabled"]: @@ -214,8 +226,28 @@ def create_camera(self): # Place Camera in scene camera_data = bpy.data.cameras.new(name=self.config["name"]) + # Set lens type and use "PERSPECTIVE" as default + if "lens_type" in self.config: + lens_type = utility.eval_param(self.config["lens_type"]) + else: + lens_type = "PERSPECTIVE" + # Initial camera settings - camera_data.lens = utility.eval_param(self.config["focal_length"]) + if lens_type == "PERSPECTIVE": + camera_data.type = "PERSP" + camera_data.lens = utility.eval_param(self.config["focal_length"]) + elif lens_type == "FISHEYE_EQUIDISTANT": + camera_data.type = "PANO" + camera_data.cycles.panorama_type = "FISHEYE_EQUIDISTANT" + camera_data.cycles.fisheye_fov = utility.eval_param(self.config["fisheye_fov"]) + elif lens_type == "FISHEYE_EQUISOLID": + camera_data.type = "PANO" + camera_data.cycles.panorama_type = "FISHEYE_EQUISOLID" + camera_data.cycles.fisheye_lens = utility.eval_param(self.config["focal_length"]) + camera_data.cycles.fisheye_fov = utility.eval_param(self.config["fisheye_fov"]) + else: + raise ValueError("Camera: not supported lens type \"%s\"", lens_type) + camera_data.sensor_width = utility.eval_param(self.config["sensor_width"]) camera_object = bpy.data.objects.new(self.config["name"], camera_data) @@ -274,7 +306,11 @@ def get_camera_matrix(self, camera): numpy.array: The camera matrix """ if camera.type != "PERSP": - raise ValueError("Non-perspective cameras not supported") + logging.warning( + "Camera: get_camera_matrix not supported for non-perspective cameras" + ) + return Matrix(((1, 0, 0), (0, 1, 0), (0, 0, 1))) + scene = bpy.context.scene f_in_mm = camera.lens resolution_x_in_px = self.config["resolution"][0] @@ -324,9 +360,7 @@ def get_camera_pose(camera): def write_intrinsics(self): """Write the camera intrinsics to a file""" - for obj in self.objs: - if obj.get().type == "CAMERA": - cam = obj.get() + cam = self.get_camera() curr_frame = bpy.context.scene.frame_current cam_name = cam["name"] calibration_folder = ( @@ -370,9 +404,7 @@ def write_intrinsics(self): def write_extrinsics(self): """Write the camera extrinsics to a file""" - for obj in self.objs: - if obj.get().type == "CAMERA": - cam = obj.get() + cam = self.get_camera() curr_frame = bpy.context.scene.frame_current cam_name = cam["name"] @@ -415,3 +447,9 @@ def write_extrinsics(self): writer.data["expected_steps"] = utility.get_job_conf()["steps"] writer.data["sensor"] = cam_name writer.data["id"] = cam_name + "_extrinsics" + + def get_camera(self): + for obj in self.objs: + if obj.get().type == "CAMERA": + return obj.get() + raise Exception("'self' has no object 'CAMERA'.") diff --git a/syclops/blender/sensors/schema/camera.schema.yaml b/syclops/blender/sensors/schema/camera.schema.yaml index cf640e1..f30f3a1 100644 --- a/syclops/blender/sensors/schema/camera.schema.yaml +++ b/syclops/blender/sensors/schema/camera.schema.yaml @@ -16,12 +16,20 @@ items: type: integer maxItems: 2 minItems: 2 + lens_type: + description: Lens type (PERSPECTIVE [default],FISHEYE_EQUISOLID,FISHEYE_EQUIDISTANT) + type: string + enum: [PERSPECTIVE, FISHEYE_EQUISOLID, FISHEYE_EQUIDISTANT] + default: PERSPECTIVE focal_length: - description: Focal length of the camera in mm. + description: Focal length of the camera in mm. Only effects lens_type=(PERSPECTIVE,FISHEYE_EQUISOLID). $ref: "#/definitions/number_evaluation" sensor_width: description: Width of the sensor in mm. $ref: "#/definitions/number_evaluation" + fisheye_fov: + description: Horizontal angular field of view in rad. Only effects lens_type=(FISHEYE_EQUIDISTANT,FISHEYE_EQUISOLID). + $ref: "#/definitions/number_evaluation" exposure: description: Exposure offset in stops. $ref: "#/definitions/number_evaluation" @@ -60,7 +68,7 @@ items: type: number frustum: description: Settings for the camera frustum visualization - type: object + type: object properties: enabled: description: Whether to enable frustum visualization @@ -85,7 +93,7 @@ items: properties: enabled: description: Whether to render as wireframe - type: boolean + type: boolean thickness: description: Thickness of wireframe lines type: number @@ -103,15 +111,30 @@ items: $ref: "#/definitions/syclops_output_object_positions" syclops_output_keypoints: $ref: "#/definitions/syclops_output_keypoints" + syclops_output_structured_light: + $ref: "#/definitions/syclops_output_structured_light" additionalProperties: False - required: [name, frame_id, resolution, focal_length, sensor_width, outputs] + required: [name, frame_id, resolution, sensor_width, outputs] allOf: - if: + required: [motion_blur] properties: motion_blur: properties: enabled: const: true - required: [motion_blur] then: - required: [shutter_speed] \ No newline at end of file + required: [shutter_speed] + - anyOf: + - properties: + lens_type: + enum: [PERSPECTIVE, null] + required: [focal_length] + - properties: + lens_type: + const: FISHEYE_EQUIDISTANT + required: [fisheye_fov] + - properties: + lens_type: + const: FISHEYE_EQUISOLID + required: [fisheye_fov, focal_length]