forked from facebookresearch/habitat-sim
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathviewer.cpp
2633 lines (2372 loc) · 97.3 KB
/
viewer.cpp
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
// Copyright (c) Facebook, Inc. and its affiliates.
// This source code is licensed under the MIT license found in the
// LICENSE file in the root directory of this source tree.
#include <math.h>
#include <stdlib.h>
#include <ctime>
#include <fstream>
#include <iostream>
#include <Magnum/configure.h>
#ifdef MAGNUM_TARGET_WEBGL
#include <Magnum/Platform/EmscriptenApplication.h>
#else
#include <Magnum/Platform/GlfwApplication.h>
#endif
#include <Magnum/GL/Context.h>
#include <Magnum/GL/Extensions.h>
#include <Magnum/GL/Framebuffer.h>
#include <Magnum/GL/Renderbuffer.h>
#include <Magnum/GL/RenderbufferFormat.h>
#include <Magnum/Image.h>
#include <Magnum/PixelFormat.h>
#include <Magnum/SceneGraph/Camera.h>
#include <Magnum/Shaders/Shaders.h>
#include <Magnum/Timeline.h>
#include "esp/core/configure.h"
#include "esp/gfx/RenderCamera.h"
#include "esp/gfx/Renderer.h"
#include "esp/gfx/replay/Recorder.h"
#include "esp/gfx/replay/ReplayManager.h"
#include "esp/nav/PathFinder.h"
#include "esp/scene/ObjectControls.h"
#include "esp/scene/SceneNode.h"
#include "esp/sensor/configure.h"
#include <Corrade/PluginManager/Manager.h>
#include <Corrade/Utility/Arguments.h>
#include <Corrade/Utility/Assert.h>
#include <Corrade/Utility/Debug.h>
#include <Corrade/Utility/DebugStl.h>
#include <Corrade/Utility/FormatStl.h>
#include <Corrade/Utility/Path.h>
#include <Corrade/Utility/Resource.h>
#include <Corrade/Utility/String.h>
#include <Magnum/DebugTools/FrameProfiler.h>
#include <Magnum/DebugTools/Screenshot.h>
#include <Magnum/EigenIntegration/GeometryIntegration.h>
#include <Magnum/GL/DefaultFramebuffer.h>
#include <Magnum/GL/Renderer.h>
#include <Magnum/Shaders/VectorGL.h>
#include <Magnum/Text/AbstractFont.h>
#include <Magnum/Text/GlyphCache.h>
#include <Magnum/Text/Renderer.h>
#include "esp/core/Esp.h"
#include "esp/core/Utility.h"
#include "esp/gfx/Drawable.h"
#include "esp/scene/SemanticScene.h"
#ifdef ESP_BUILD_WITH_VHACD
#include "esp/geo/VoxelUtils.h"
#endif
#ifdef ESP_BUILD_WITH_AUDIO
#include "esp/sensor/AudioSensor.h"
#endif // ESP_BUILD_WITH_AUDIO
#include "esp/physics/configure.h"
#include "esp/sensor/CameraSensor.h"
#include "esp/sensor/EquirectangularSensor.h"
#include "esp/sensor/FisheyeSensor.h"
#include "esp/sim/Simulator.h"
#include "ObjectPickingHelper.h"
constexpr float moveSensitivity = 0.07f;
constexpr float lookSensitivity = 0.9f;
constexpr float rgbSensorHeight = 1.5f;
constexpr float agentActionsPerSecond = 60.0f;
// for ease of access
namespace Cr = Corrade;
namespace Mn = Magnum;
namespace {
//! return current time as string in format
//! "year_month_day_hour-minutes-seconds"
std::string getCurrentTimeString() {
time_t now = time(0);
tm* ltm = localtime(&now);
return std::to_string(1900 + ltm->tm_year) + "_" +
std::to_string(1 + ltm->tm_mon) + "_" + std::to_string(ltm->tm_mday) +
"_" + std::to_string(ltm->tm_hour) + "-" +
std::to_string(ltm->tm_min) + "-" + std::to_string(ltm->tm_sec);
}
using namespace Mn::Math::Literals;
using Mn::Math::Literals::operator""_degf;
//! Define different UI roles for the mouse
enum MouseInteractionMode {
LOOK,
GRAB,
NUM_MODES
};
std::map<MouseInteractionMode, std::string> mouseModeNames = {
{MouseInteractionMode::LOOK, "LOOK"},
{MouseInteractionMode::GRAB, "GRAB"}};
//! Create a MouseGrabber from RigidConstraintSettings to manipulate objects
struct MouseGrabber {
esp::physics::RigidConstraintSettings settings_;
esp::sim::Simulator& sim_;
// defines the distance of the grip point from the camera/eye for pivot
// updates
float gripDepth = 0;
int constraintId = esp::ID_UNDEFINED;
MouseGrabber(const esp::physics::RigidConstraintSettings& settings,
float _gripDepth,
esp::sim::Simulator& sim)
: sim_(sim),
settings_(settings),
gripDepth(_gripDepth),
constraintId(sim_.createRigidConstraint(settings_)) {}
virtual ~MouseGrabber() { sim_.removeRigidConstraint(constraintId); }
//! update global pivot position for the constraint
virtual void updatePivot(const Mn::Vector3& pos) {
settings_.pivotB = pos;
sim_.updateRigidConstraint(constraintId, settings_);
}
//! update global rotation frame for the constraint
virtual void updateFrame(const Mn::Matrix3x3& frame) {
settings_.frameB = frame;
sim_.updateRigidConstraint(constraintId, settings_);
}
//! update global rotation frame and pivot position for the constraint
virtual void updateTransform(const Mn::Matrix4& transform) {
settings_.frameB = transform.rotation();
settings_.pivotB = transform.translation();
sim_.updateRigidConstraint(constraintId, settings_);
}
//! rotate the object's local constraint frame with a global angle axis input
virtual void rotateLocalFrameByGlobalAngleAxis(const Mn::Vector3 axis,
const Mn::Rad angle) {
Mn::Matrix4 objectTransform;
auto rom = sim_.getRigidObjectManager();
auto aom = sim_.getArticulatedObjectManager();
if (rom->getObjectLibHasID(settings_.objectIdA)) {
objectTransform =
rom->getObjectByID(settings_.objectIdA)->getTransformation();
} else {
objectTransform = aom->getObjectByID(settings_.objectIdA)
->getLinkSceneNode(settings_.linkIdA)
->transformation();
}
// convert the axis into the object local space
Mn::Vector3 localAxis = objectTransform.inverted().transformVector(axis);
// create a rotation matrix
Mn::Matrix4 R = Mn::Matrix4::rotation(angle, localAxis.normalized());
// apply to the frame
settings_.frameA = R.rotation() * settings_.frameA;
sim_.updateRigidConstraint(constraintId, settings_);
}
//! Render a downward projection of the grasped object's COM
virtual void renderDebugLines() {
// cast a ray downward from COM to determine approximate landing point
Mn::Matrix4 objectTransform;
auto rom = sim_.getRigidObjectManager();
auto aom = sim_.getArticulatedObjectManager();
std::vector<int> objectRelatedIds(1, settings_.objectIdA);
if (rom->getObjectLibHasID(settings_.objectIdA)) {
objectTransform =
rom->getObjectByID(settings_.objectIdA)->getTransformation();
} else {
auto ao = aom->getObjectByID(settings_.objectIdA);
objectTransform =
ao->getLinkSceneNode(settings_.linkIdA)->transformation();
for (const auto& entry : ao->getLinkObjectIds()) {
objectRelatedIds.push_back(entry.first);
}
}
esp::physics::RaycastResults raycastResults = sim_.castRay(
esp::geo::Ray(objectTransform.translation(), Mn::Vector3(0, -1, 0)));
float lineLength = 9999.9;
for (auto& hit : raycastResults.hits) {
if (!std::count(objectRelatedIds.begin(), objectRelatedIds.end(),
hit.objectId)) {
lineLength = hit.rayDistance;
break;
}
}
sim_.getDebugLineRender()->drawLine(
objectTransform.translation(),
objectTransform.translation() + Mn::Vector3(0, -lineLength, 0),
Mn::Color4::green());
}
};
class Viewer : public Mn::Platform::Application {
public:
explicit Viewer(const Arguments& arguments);
private:
// Keys for moving/looking are recorded according to whether they are
// currently being pressed
std::map<KeyEvent::Key, bool> keysPressed = {
{KeyEvent::Key::Left, false}, {KeyEvent::Key::Right, false},
{KeyEvent::Key::Up, false}, {KeyEvent::Key::Down, false},
{KeyEvent::Key::A, false}, {KeyEvent::Key::D, false},
{KeyEvent::Key::S, false}, {KeyEvent::Key::W, false},
{KeyEvent::Key::X, false}, {KeyEvent::Key::Z, false}};
MouseInteractionMode mouseInteractionMode = LOOK;
Mn::Vector2i previousMousePoint{};
void drawEvent() override;
void viewportEvent(ViewportEvent& event) override;
void mousePressEvent(MouseEvent& event) override;
void mouseReleaseEvent(MouseEvent& event) override;
void mouseMoveEvent(MouseMoveEvent& event) override;
void mouseScrollEvent(MouseScrollEvent& event) override;
void keyPressEvent(KeyEvent& event) override;
void keyReleaseEvent(KeyEvent& event) override;
void moveAndLook(int repetitions);
/**
* @brief This function will get a screen-space mouse position appropriately
* scaled based on framebuffer size and window size. Generally these would be
* the same value, but on certain HiDPI displays (Retina displays) they may be
* different.
* @param event The mouse event we wish to extract a location of
* @return The screen-space location of the mouse event.
*/
Mn::Vector2i getMousePosition(Mn::Vector2i mouseEventPosition) {
// aquire the mouse position, and scale it based on ratio of framebuffer and
// window size.
// on retina displays this scaling calc is necessary to account for HiDPI
// monitors.
Mn::Vector2 scaling = Mn::Vector2{framebufferSize()} * dpiScaling() /
Mn::Vector2{windowSize()};
return Mn::Vector2i(mouseEventPosition * scaling);
}
// exists if a mouse grabbing constraint is active, destroyed on release
std::unique_ptr<MouseGrabber> mouseGrabber_ = nullptr;
//! Most recently custom loaded URDF ('t' key)
std::string cachedURDF_ = "";
/**
* @brief Instance an object from an ObjectAttributes.
* @param configHandle The handle referencing the object's template in the
* ObjectAttributesManager.
* @return The newly allocated object's ID for referencing it through the
* Simulator API.
*/
int addObject(const std::string& configHandle);
/**
* @brief Instance an object from an ObjectAttributes.
* @param objID The unique ID referencing the object's template in the
* ObjectAttributesManager.
* @return The newly allocated object's ID for referencing it through the
* Simulator API.
*/
int addObject(int objID);
/**
* @brief Instance a random object based on an imported asset if one exists.
* @return The newly allocated object's ID for referencing it through the
* Simulator API.
*/
int addTemplateObject();
/**
* @brief Instance a random object based on a primitive shape.
* @return The newly allocated object's ID for referencing it through the
* Simulator API.
*/
int addPrimitiveObject();
void removeLastObject();
void invertGravity();
#ifdef ESP_BUILD_WITH_VHACD
void displayStageDistanceGradientField();
void iterateAndDisplaySignedDistanceField();
void displayVoxelField(int objectID);
int objectDisplayed = -1;
//! The slice of the grid's SDF to visualize.
int voxelDistance = 0;
#endif
/**
* @brief Toggle between ortho and perspective camera
*/
void switchCameraType();
Mn::Vector3 randomDirection();
/**
* @brief Display information about the currently loaded scene.
*/
void dispMetadataInfo();
esp::agent::AgentConfiguration agentConfig_;
void saveAgentAndSensorTransformToFile();
void loadAgentAndSensorTransformFromFile();
//! string rep of time when viewer application was started
std::string viewerStartTimeString = getCurrentTimeString();
void screenshot();
esp::sensor::CameraSensor& getAgentCamera() {
esp::sensor::Sensor& cameraSensor =
agentBodyNode_->getNodeSensorSuite().get("rgba_camera");
return static_cast<esp::sensor::CameraSensor&>(cameraSensor);
}
#ifdef ESP_BUILD_WITH_AUDIO
esp::sensor::AudioSensor& getAgentAudioSensor() {
esp::sensor::Sensor& audioSensor =
agentBodyNode_->getNodeSensorSuite().get("audio");
return static_cast<esp::sensor::AudioSensor&>(audioSensor);
}
#endif // ESP_BUILD_WITH_AUDIO
std::string helpText = R"(
==================================================
Welcome to the Habitat-sim C++ Viewer application!
==================================================
Mouse Functions ('m' to toggle mode):
----------------
In LOOK mode (default):
LEFT:
Click and drag to rotate the agent and look up/down.
RIGHT:
(physics) Click a surface to instance a random primitive object at that location.
SHIFT-LEFT:
Read Semantic ID and tag of clicked object (Currently only HM3D);
SHIFT-RIGHT:
Click a mesh to highlight it.
CTRL-RIGHT:
(physics) Click on an object to voxelize it and display the voxelization.
WHEEL:
Modify orthographic camera zoom/perspective camera FOV (+SHIFT for fine grained control)
In GRAB mode (with 'enable-physics'):
LEFT:
Click and drag to pickup and move an object with a point-to-point constraint (e.g. ball joint).
RIGHT:
Click and drag to pickup and move an object with a fixed frame constraint.
WHEEL (with picked object):
Pull gripped object closer or push it away.
+ ALT: rotate object fixed constraint frame (yaw)
+ CTRL: rotate object fixed constraint frame (pitch)
+ ALT+CTRL: rotate object fixed constraint frame (roll)
Key Commands:
-------------
esc: Exit the application.
'H': Display this help message.
'm': Toggle mouse mode (LOOK | GRAB).
TAB/Shift-TAB : Cycle to next/previous scene in scene dataset.
ALT+TAB: Reload the current scene.
Agent Controls:
'wasd': Move the agent's body forward/backward, left/right.
'zx': Move the agent's body up/down.
arrow keys: Turn the agent's body left/right and camera look up/down.
'9': Randomly place agent on NavMesh (if loaded).
'q': Query the agent's state and print to terminal.
'[': Save agent position/orientation to "./saved_transformations/camera.year_month_day_hour-minute-second.txt".
']': Load agent position/orientation from file system, or else from last save in current instance.
Camera Settings
'4': Cycle through camera modes (Camera, Fisheye, Equirectangular)
'5': Switch ortho/perspective camera.
'6': Reset ortho camera zoom/perspective camera FOV.
'7': Cycle through rendering modes (RGB, depth, semantic)
Visualization Utilities:
'l': Override the default lighting setup with configured settings in `default_light_override.lighting_config.json`.
'e': Enable/disable frustum culling.
'c': Show/hide UI overlay.
'n': Show/hide NavMesh wireframe.
'i': Save a screenshot to "./screenshots/year_month_day_hour-minute-second/#.png".
',': Render a Bullet collision shape debug wireframe overlay (white=active, green=sleeping, blue=wants sleeping, red=can't sleep)
Object Interactions:
SPACE: Toggle physics simulation on/off
'.': Take a single simulation step if not simulating continuously.
'8': Instance a random primitive object in front of the agent.
'o': Instance a random file-based object in front of the agent.
'u': Remove most recently instanced rigid object.
't': Instance an ArticulatedObject in front of the camera from a URDF file by entering the filepath when prompted.
+ALT: Import the object with a fixed base.
+SHIFT Quick-reload the previously specified URDF.
'b': Toggle display of object bounding boxes.
'p': Save current simulation state to SceneInstanceAttributes JSON file (with non-colliding filename).
'v': (physics) Invert gravity.
'g': (physics) Display a stage's signed distance gradient vector field.
'k': (physics) Iterate through different ranges of the stage's voxelized signed distance field.
Additional Utilities:
'r': Write a replay of the recent simulated frames to a file specified by --gfx-replay-record-filepath.
'/': Write the current scene's metadata information to console.
Nav Trajectory Visualization:
'1': Toggle recording locations for trajectory visualization.
'2': Build and display trajectory visualization.
'3': Toggle single color/multi-color trajectory.
'+': Increase trajectory diameter.
'-': Decrease trajectory diameter.
'F': (audio) Add audio source in front of the agent
'0': (audio) Run audio simulation
==================================================
)";
//! Print viewer help text to terminal output.
void printHelpText() { ESP_DEBUG() << helpText; };
// single inline for logging agent state msgs, so can be easily modified
inline void showAgentStateMsg(bool showPos, bool showOrient) {
std::stringstream strDat("");
if (showPos) {
strDat << "Agent position "
<< Eigen::Map<esp::vec3f>(agentBodyNode_->translation().data())
<< " ";
}
if (showOrient) {
strDat << "Agent orientation "
<< esp::quatf(agentBodyNode_->rotation()).coeffs().transpose();
}
auto str = strDat.str();
if (str.size() > 0) {
ESP_DEBUG() << str;
}
}
/**
* @brief vector holding past agent locations to build trajectory
* visualization
*/
std::vector<Mn::Vector3> agentLocs_;
float agentTrajRad_ = .01f;
bool agentLocRecordOn_ = false;
bool singleColorTrajectory_ = true;
std::string semanticTag_ = "";
// Wireframe bounding box around semantic region
int semanticBBID_ = -1;
/**
* @brief Generate and save semantic CC reports for all scenes in dataset
*/
void generateAndSaveAllSemanticCCReports();
/**
* @brief Generate and save semantic CC report
* @return Whether successful or not.
*/
bool generateAndSaveSemanticCCReport();
/**
* @brief Generate and save vertex-color-semantic object mapping reports for
* all scenes.
*/
void generateAndSaveAllVertColorMapReports();
/**
* @brief Generate and save vertex-color-semantic object mapping reports
* @return Whether successful or not.
*/
bool generateAndSaveVertColorMapReports();
/**
* @brief Build semantic region prims.
*/
void buildSemanticPrims(int semanticID,
const std::string& semanticTag,
const Mn::Vector3& semanticCtr,
const Mn::Vector3& semanticSize,
const Mn::Quaternion& rotation);
/**
* @brief Set whether agent locations should be recorded or not. If toggling
* on then clear old locations
*/
inline void setAgentLocationRecord(bool enable) {
if (enable == !agentLocRecordOn_) {
agentLocRecordOn_ = enable;
if (enable) { // if turning on, clear old data
agentLocs_.clear();
recAgentLocation();
}
}
ESP_DEBUG() << "Agent location recording"
<< (agentLocRecordOn_ ? "on" : "off");
} // setAgentLocationRecord
/**
* @brief Record agent location if enabled. Called after any movement.
*/
inline void recAgentLocation() {
if (agentLocRecordOn_) {
auto pt = agentBodyNode_->translation() +
Mn::Vector3{0, (2.0f * agentTrajRad_), 0};
agentLocs_.push_back(pt);
ESP_DEBUG() << "Recording agent location : {" << pt.x() << "," << pt.y()
<< "," << pt.z() << "}";
}
}
/**
* @brief Build trajectory visualization
*/
void buildTrajectoryVis();
void modTrajRad(bool bigger) {
std::string mod = "";
if (bigger) {
if (agentTrajRad_ < 1.0) {
agentTrajRad_ += 0.001f;
mod = "increased to ";
}
} else {
if (agentTrajRad_ > 0.001f) {
agentTrajRad_ -= 0.001f;
mod = "decreased to ";
}
}
esp::geo::clamp(agentTrajRad_, 0.001f, 1.0f);
ESP_DEBUG() << "Agent Trajectory Radius" << mod << ":" << agentTrajRad_;
}
esp::logging::LoggingContext loggingContext_;
// Configuration to use to set up simulator_
esp::sim::SimulatorConfiguration simConfig_;
// NavMesh customization options, from args
bool disableNavmesh_ = false;
bool recomputeNavmesh_ = false;
std::string navmeshFilename_{};
// if currently cycling through available SceneInstances, this is the list of
// SceneInstances available
std::vector<std::string> curSceneInstances_;
// current index in SceneInstance array, if cycling through scenes
int curSceneInstanceIDX_ = 0;
// increment/decrement currently displayed scene instance
int getNextSceneInstanceIDX(int incr) {
if (curSceneInstances_.size() == 0) {
return 0;
}
return (curSceneInstanceIDX_ + curSceneInstances_.size() + incr) %
curSceneInstances_.size();
}
/**
* @brief Set the scene instance to use to build the scene.
*/
void setSceneInstanceFromListAndShow(int nextSceneInstanceIDX);
// The MetadataMediator can exist independent of simulator
// and provides access to all managers for currently active scene dataset
std::shared_ptr<esp::metadata::MetadataMediator> MM_;
// The managers belonging to MetadataMediator
std::shared_ptr<esp::metadata::managers::ObjectAttributesManager>
objectAttrManager_ = nullptr;
// The simulator object backend for this viewer instance
std::unique_ptr<esp::sim::Simulator> simulator_;
// Initialize simulator after a scene has been loaded - handle navmesh, agent
// and sensor configs
void initSimPostReconfigure();
// Toggle physics simulation on/off
bool simulating_ = true;
// Toggle a single simulation step at the next opportunity if not simulating
// continuously.
bool simulateSingleStep_ = false;
bool debugBullet_ = false;
esp::scene::SceneNode* agentBodyNode_ = nullptr;
const int defaultAgentId_ = 0;
esp::agent::Agent::ptr defaultAgent_ = nullptr;
// if currently orthographic
bool isOrtho_ = false;
esp::gfx::RenderCamera* renderCamera_ = nullptr;
esp::scene::SceneGraph* activeSceneGraph_ = nullptr;
bool drawObjectBBs = false;
std::string gfxReplayRecordFilepath_;
std::string agentTransformLoadPath_;
Cr::Containers::Optional<Mn::Matrix4> savedAgentTransform_ =
Cr::Containers::NullOpt;
// there could be a couple of sensors, just save the rgb CameraSensor's
// transformation
Cr::Containers::Optional<Mn::Matrix4> savedSensorTransform_ =
Cr::Containers::NullOpt;
Mn::Timeline timeline_;
/* Text rendering */
Cr::PluginManager::Manager<Mn::Text::AbstractFont> fontManager_;
Cr::Containers::Pointer<Mn::Text::AbstractFont> font_;
Cr::Containers::Optional<Mn::Text::GlyphCache> fontGlyphCache_;
Mn::Shaders::VectorGL2D fontShader_;
Cr::Containers::Optional<Mn::Text::Renderer2D> fontText_;
bool showFPS_ = true;
// NOTE: Mouse + shift is to select object on the screen!!
void createPickedObjectVisualizer(unsigned int objectId);
std::unique_ptr<ObjectPickingHelper> objectPickingHelper_;
enum class VisualizeMode : uint8_t {
RGBA = 0,
Depth,
Semantic,
VisualizeModeCount,
};
VisualizeMode visualizeMode_ = VisualizeMode::RGBA;
Mn::DebugTools::FrameProfilerGL profiler_{};
enum class VisualSensorMode : uint8_t {
Camera = 0,
Fisheye,
Equirectangular,
VisualSensorModeCount,
};
VisualSensorMode sensorMode_ = VisualSensorMode::Camera;
/**
* @brief Set the sensorVisID_ string used to determine which sensor to pull
* an observation from. Should be set whenever visualizeMode_ or sensorMode_
* change.
*/
void setSensorVisID();
/**
* @brief String key used to determine which sensor from a suite of sensors to
* pull an observation from.
*/
std::string sensorVisID_ = "rgba_camera";
void bindRenderTarget();
#ifdef ESP_BUILD_WITH_AUDIO
/**
* @brief Add an audio source to the scene
* The source is added in front of the agent
*/
void addAudioSource();
/**
* @brief Run the audio simulation and get the observations
*/
void runAudioSimulation();
#endif // ESP_BUILD_WITH_AUDIO
}; // class viewer declaration
void addSensors(esp::agent::AgentConfiguration& agentConfig, bool isOrtho) {
const auto viewportSize = Mn::GL::defaultFramebuffer.viewport().size();
auto addCameraSensor = [&](const std::string& uuid,
esp::sensor::SensorType sensorType) {
agentConfig.sensorSpecifications.emplace_back(
esp::sensor::CameraSensorSpec::create());
auto spec = static_cast<esp::sensor::CameraSensorSpec*>(
agentConfig.sensorSpecifications.back().get());
spec->uuid = uuid;
spec->sensorSubType = isOrtho ? esp::sensor::SensorSubType::Orthographic
: esp::sensor::SensorSubType::Pinhole;
spec->sensorType = sensorType;
if (sensorType == esp::sensor::SensorType::Depth ||
sensorType == esp::sensor::SensorType::Semantic) {
spec->channels = 1;
}
spec->position = {0.0f, rgbSensorHeight, 0.0f};
spec->orientation = {0, 0, 0};
spec->resolution = esp::vec2i(viewportSize[1], viewportSize[0]);
};
// add the camera color sensor
// for historical reasons, we call it "rgba_camera"
addCameraSensor("rgba_camera", esp::sensor::SensorType::Color);
// add the camera depth sensor
addCameraSensor("depth_camera", esp::sensor::SensorType::Depth);
// add the camera semantic sensor
addCameraSensor("semantic_camera", esp::sensor::SensorType::Semantic);
auto addFisheyeSensor = [&](const std::string& uuid,
esp::sensor::SensorType sensorType,
esp::sensor::FisheyeSensorModelType modelType) {
// TODO: support the other model types in the future.
CORRADE_INTERNAL_ASSERT(modelType ==
esp::sensor::FisheyeSensorModelType::DoubleSphere);
agentConfig.sensorSpecifications.emplace_back(
esp::sensor::FisheyeSensorDoubleSphereSpec::create());
auto spec = static_cast<esp::sensor::FisheyeSensorDoubleSphereSpec*>(
agentConfig.sensorSpecifications.back().get());
spec->uuid = uuid;
spec->sensorType = sensorType;
if (sensorType == esp::sensor::SensorType::Depth ||
sensorType == esp::sensor::SensorType::Semantic) {
spec->channels = 1;
}
spec->sensorSubType = esp::sensor::SensorSubType::Fisheye;
spec->fisheyeModelType = modelType;
spec->resolution = esp::vec2i(viewportSize[1], viewportSize[0]);
// default viewport size: 1600 x 1200
spec->principalPointOffset =
Mn::Vector2(viewportSize[0] / 2, viewportSize[1] / 2);
if (modelType == esp::sensor::FisheyeSensorModelType::DoubleSphere) {
// in this demo, we choose "GoPro":
spec->focalLength = {364.84f, 364.86f};
spec->xi = -0.27;
spec->alpha = 0.57;
// Certainly you can try your own lenses.
// For your convenience, there are some other lenses, e.g., BF2M2020S23,
// BM2820, BF5M13720, BM4018S118, whose parameters can be found at:
// Vladyslav Usenko, Nikolaus Demmel and Daniel Cremers: The Double
// Sphere Camera Model, The International Conference on 3D Vision(3DV),
// 2018
// BF2M2020S23
// spec->focalLength = Mn::Vector2(313.21, 313.21);
// spec->xi = -0.18;
// spec->alpha = 0.59;
}
};
// add the fisheye sensor
addFisheyeSensor("rgba_fisheye", esp::sensor::SensorType::Color,
esp::sensor::FisheyeSensorModelType::DoubleSphere);
// add the fisheye depth sensor
addFisheyeSensor("depth_fisheye", esp::sensor::SensorType::Depth,
esp::sensor::FisheyeSensorModelType::DoubleSphere);
// add the fisheye semantic sensor
addFisheyeSensor("semantic_fisheye", esp::sensor::SensorType::Semantic,
esp::sensor::FisheyeSensorModelType::DoubleSphere);
auto addEquirectangularSensor = [&](const std::string& uuid,
esp::sensor::SensorType sensorType) {
agentConfig.sensorSpecifications.emplace_back(
esp::sensor::EquirectangularSensorSpec::create());
auto spec = static_cast<esp::sensor::EquirectangularSensorSpec*>(
agentConfig.sensorSpecifications.back().get());
spec->uuid = uuid;
spec->sensorType = sensorType;
if (sensorType == esp::sensor::SensorType::Depth ||
sensorType == esp::sensor::SensorType::Semantic) {
spec->channels = 1;
}
spec->sensorSubType = esp::sensor::SensorSubType::Equirectangular;
spec->resolution = esp::vec2i(viewportSize[1], viewportSize[0]);
};
// add the equirectangular sensor
addEquirectangularSensor("rgba_equirectangular",
esp::sensor::SensorType::Color);
// add the equirectangular depth sensor
addEquirectangularSensor("depth_equirectangular",
esp::sensor::SensorType::Depth);
// add the equirectangular semantic sensor
addEquirectangularSensor("semantic_equirectangular",
esp::sensor::SensorType::Semantic);
// add audio sensor
#ifdef ESP_BUILD_WITH_AUDIO
ESP_DEBUG() << "Adding audio sendor";
auto addAudioSensor = [&](const std::string& uuid,
esp::sensor::SensorType sensorType,
esp::sensor::SensorSubType sensorSubType) {
agentConfig.sensorSpecifications.emplace_back(
esp::sensor::AudioSensorSpec::create());
auto spec = static_cast<esp::sensor::AudioSensorSpec*>(
agentConfig.sensorSpecifications.back().get());
spec->uuid = uuid;
spec->sensorType = sensorType;
spec->sensorSubType = sensorSubType;
// Set the audio sensor configs
spec->acousticsConfig_.dumpWaveFiles = true;
spec->acousticsConfig_.enableMaterials = true;
spec->acousticsConfig_.writeIrToFile = true;
// Set the output directory
spec->outputDirectory_ = "/tmp/AudioSimulation";
// Set the output channel layout
spec->channelLayout_.channelCount = 2;
spec->channelLayout_.channelType =
RLRAudioPropagation::ChannelLayoutType::Binaural;
};
addAudioSensor("audio", esp::sensor::SensorType::Audio,
esp::sensor::SensorSubType::ImpulseResponse);
#endif // ESP_BUILD_WITH_AUDIO
} // addSensors
Viewer::Viewer(const Arguments& arguments)
: Mn::Platform::Application{arguments,
Configuration{}
.setTitle("Viewer")
.setWindowFlags(
Configuration::WindowFlag::Resizable),
GLConfiguration{}
.setColorBufferSize(
Mn::Vector4i(8, 8, 8, 8))
.setSampleCount(4)},
loggingContext_{},
simConfig_(),
MM_(std::make_shared<esp::metadata::MetadataMediator>(simConfig_)),
curSceneInstances_{} {
Cr::Utility::Arguments args;
#ifdef CORRADE_TARGET_EMSCRIPTEN
args.addNamedArgument("scene")
#else
args.addArgument("scene")
#endif
.setHelp("scene", "scene/stage file to load")
.addSkippedPrefix("magnum", "engine-specific options")
.setGlobalHelp("Displays a 3D scene file provided on command line")
.addOption("dataset", "default")
.setHelp("dataset", "dataset configuration file to use")
.addBooleanOption("enable-physics")
.addBooleanOption("stage-requires-lighting")
.setHelp("stage-requires-lighting",
"Stage asset should be lit with Phong shading.")
.addBooleanOption("debug-bullet")
.setHelp("debug-bullet", "Render Bullet physics debug wireframes.")
.addOption("gfx-replay-record-filepath")
.setHelp("gfx-replay-record-filepath",
"Enable replay recording with R key.")
.addOption("physics-config", ESP_DEFAULT_PHYSICS_CONFIG_REL_PATH)
.setHelp("physics-config",
"Provide a non-default PhysicsManager config file.")
.addOption("object-dir", "data/objects/example_objects")
.setHelp("object-dir",
"Provide a directory to search for object config files "
"(relative to habitat-sim directory).")
.addBooleanOption("no-semantic-textures")
.setHelp("no-semantic-textures",
"If specified, force vertex semantic annotations even if the "
"scene/dataset support texture-based.")
.addBooleanOption("disable-navmesh")
.setHelp("disable-navmesh",
"Disable the navmesh, disabling agent navigation constraints.")
.addOption("navmesh-file")
.setHelp("navmesh-file", "Manual override path to scene navmesh file.")
.addBooleanOption("recompute-navmesh")
.setHelp("recompute-navmesh",
"Programmatically re-generate the scene navmesh.")
.addOption("agent-transform-filepath")
.setHelp("agent-transform-filepath",
"Specify path to load camera transform from.")
.addBooleanOption("shadows")
.setHelp("shadows", "Rendering shadows. (only works with PBR rendering.")
.addBooleanOption("ibl")
.setHelp("ibl",
"Image Based Lighting (it works only when PBR models exist in "
"the scene.")
.parse(arguments.argc, arguments.argv);
const auto viewportSize = Mn::GL::defaultFramebuffer.viewport().size();
/* Set up text rendering */
{
/* Render the font in a size corresponding to actual pixels, taking DPI
scaling into account. Docs for reference:
https://doc.magnum.graphics/magnum/classMagnum_1_1Platform_1_1Sdl2Application.html#Platform-Sdl2Application-dpi
*/
const Mn::Float fontSize =
13.0f * (Mn::Vector2{framebufferSize()} * dpiScaling() /
Mn::Vector2{windowSize()})
.x();
Cr::Utility::Resource rs{"fonts"};
font_ = fontManager_.loadAndInstantiate("TrueTypeFont");
if (!font_ || !font_->openData(rs.getRaw("ProggyClean.ttf"), fontSize))
Mn::Fatal{} << "Cannot open font file";
fontGlyphCache_.emplace(Mn::Vector2i{256});
/* Don't destroy the bitmap font with smooth scaling */
fontGlyphCache_->texture().setMagnificationFilter(
Mn::SamplerFilter::Nearest);
font_->fillGlyphCache(*fontGlyphCache_,
"abcdefghijklmnopqrstuvwxyz"
"ABCDEFGHIJKLMNOPQRSTUVWXYZ"
"0123456789:-_+,.! %µ");
/* But as the text projection uses virtual window pixels, use just the
unscaled size for the actual text */
fontText_.emplace(*font_, *fontGlyphCache_, fontSize,
Mn::Text::Alignment::TopLeft);
fontText_->reserve(1024, Mn::GL::BufferUsage::DynamicDraw,
Mn::GL::BufferUsage::StaticDraw);
}
// Setup renderer and shader defaults
Mn::GL::Renderer::enable(Mn::GL::Renderer::Feature::DepthTest);
Mn::GL::Renderer::enable(Mn::GL::Renderer::Feature::FaceCulling);
if (args.isSet("enable-physics") && (args.isSet("debug-bullet"))) {
debugBullet_ = true;
}
agentTransformLoadPath_ = args.value("agent-transform-filepath");
gfxReplayRecordFilepath_ = args.value("gfx-replay-record-filepath");
// NavMesh customization options
disableNavmesh_ = args.isSet("disable-navmesh");
recomputeNavmesh_ = args.isSet("recompute-navmesh");
navmeshFilename_ = args.value("navmesh-file").empty();
// configure default Agent Config for actions and sensors
agentConfig_ = esp::agent::AgentConfiguration();
agentConfig_.height = rgbSensorHeight;
agentConfig_.actionSpace = {
// setup viewer action space
{"moveForward",
esp::agent::ActionSpec::create(
"moveForward",
esp::agent::ActuationMap{{"amount", moveSensitivity}})},
{"moveBackward",
esp::agent::ActionSpec::create(
"moveBackward",
esp::agent::ActuationMap{{"amount", moveSensitivity}})},
{"moveLeft",
esp::agent::ActionSpec::create(
"moveLeft", esp::agent::ActuationMap{{"amount", moveSensitivity}})},
{"moveRight",
esp::agent::ActionSpec::create(
"moveRight", esp::agent::ActuationMap{{"amount", moveSensitivity}})},
{"moveDown",
esp::agent::ActionSpec::create(
"moveDown", esp::agent::ActuationMap{{"amount", moveSensitivity}})},
{"moveUp",
esp::agent::ActionSpec::create(
"moveUp", esp::agent::ActuationMap{{"amount", moveSensitivity}})},
{"turnLeft",
esp::agent::ActionSpec::create(
"turnLeft", esp::agent::ActuationMap{{"amount", lookSensitivity}})},
{"turnRight",
esp::agent::ActionSpec::create(
"turnRight", esp::agent::ActuationMap{{"amount", lookSensitivity}})},
{"lookUp",
esp::agent::ActionSpec::create(
"lookUp", esp::agent::ActuationMap{{"amount", lookSensitivity}})},
{"lookDown",
esp::agent::ActionSpec::create(
"lookDown", esp::agent::ActuationMap{{"amount", lookSensitivity}})},
};
// add sensor specifications to agent config
addSensors(agentConfig_, isOrtho_);
// setup SimulatorConfig from args.
simConfig_.activeSceneName = args.value("scene");
simConfig_.sceneDatasetConfigFile = args.value("dataset");
simConfig_.enablePhysics = args.isSet("enable-physics");
simConfig_.frustumCulling = true;
simConfig_.requiresTextures = true;
simConfig_.enableGfxReplaySave = !gfxReplayRecordFilepath_.empty();
simConfig_.useSemanticTexturesIfFound = !args.isSet("no-semantic-textures");
if (args.isSet("stage-requires-lighting")) {
ESP_DEBUG() << "Stage using DEFAULT_LIGHTING_KEY";
simConfig_.overrideSceneLightDefaults = true;
simConfig_.sceneLightSetupKey = esp::DEFAULT_LIGHTING_KEY;
}
// setup the PhysicsManager config file
std::string physicsConfig =
Cr::Utility::Path::join(*Corrade::Utility::Path::currentDirectory(),
args.value("physics-config"));
if (Cr::Utility::Path::exists(physicsConfig)) {
ESP_DEBUG() << "Using PhysicsManager config:" << physicsConfig;
simConfig_.physicsConfigFile = physicsConfig;