-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathindex.html
257 lines (235 loc) · 10.5 KB
/
index.html
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="description"
content="Instance-Level Semantic Maps for Vision Language Navigation">
<meta name="keywords" content="instance segmentation, occupancy grids, VLN">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Instance-Level Semantic Maps for Vision Language Navigation</title>
<link href="https://fonts.googleapis.com/css?family=Google+Sans|Noto+Sans|Castoro"
rel="stylesheet">
<link rel="stylesheet" href="./static/css/bulma.min.css">
<link rel="stylesheet" href="./static/css/fontawesome.all.min.css">
<link rel="stylesheet"
href="https://cdn.jsdelivr.net/gh/jpswalsh/academicons@1/css/academicons.min.css">
<link rel="stylesheet" href="./static/css/index.css">
<!-- <link rel="icon" href="./static/images/favicon.svg"> -->
<script defer src="./static/js/fontawesome.all.min.js"></script>
</head>
<body>
<nav class="navbar" role="navigation" aria-label="main navigation">
<div class="navbar-brand">
<a role="button" class="navbar-burger" aria-label="menu" aria-expanded="false">
<span aria-hidden="true"></span>
<span aria-hidden="true"></span>
<span aria-hidden="true"></span>
</a>
</div>
<div class="navbar-menu">
<div class="navbar-start" style="flex-grow: 1; justify-content: center;">
<a class="navbar-item" href="https://www.linkedin.com/in/laksh-nanwani/">
<span class="icon">
<i class="fas fa-home"></i>
</span>
</a>
<div class="navbar-item has-dropdown is-hoverable">
<a class="navbar-link">
More Research
</a>
<div class="navbar-dropdown">
<a class="navbar-item" href="https://smart-wheelchair-rrc.github.io/o3d-sim-webpage/">
O3D-SIM
</a>
<a class="navbar-item" href="https://smart-wheelchair-rrc.github.io/CrowdSurfer-webpage/">
CrowdSurfer
</a>
</div>
</div>
</div>
</div>
</nav>
<section class="hero">
<div class="hero-body">
<div class="container is-max-desktop">
<div class="columns is-centered">
<div class="column has-text-centered">
<h1 class="title is-1 publication-title">Instance-Level Semantic Maps for Vision Language Navigation — SI Maps</h1>
<div class="is-size-5 publication-authors">
<span class="author-block">
<a href="https://www.linkedin.com/in/laksh-nanwani/">Laksh Nanwani</a><sup>1</sup>,</span>
<span class="author-block">
<a href="https://anmolagarwal999.github.io/">Anmol Agarwal</a><sup>1</sup>,</span>
<span class="author-block">
<a href="https://kanji95.github.io/">Kanishk Jain</a><sup>1</sup>,
</span>
<span class="author-block">
<a href="https://www.raghavprabhakar.com/">Raghav Prabhakar</a><sup>1</sup>,
</span>
<span class="author-block">
<a href="https://www.linkedin.com/in/aaron-monis-250241248/">Aaron Monis</a><sup>1</sup>,
</span>
<span class="author-block">
<a href="https://www.linkedin.com/in/aditya-mathur-07042a172/">Aditya Mathur</a><sup>1</sup>,
</span>
<span class="author-block">
<a href="https://krrish94.github.io/">Krishna Murthy Jatavallabhula</a><sup>2</sup>,
</span>
<span class="author-block">
<a href="https://www.linkedin.com/in/a-h-abdul-hafez-53634656/">A. H. Abdul Hafez</a><sup>3</sup>,
</span>
<span class="author-block">
<a href="https://faculty.iiit.ac.in/~vgandhi/">Vineet Gandhi</a><sup>1</sup>,
</span>
<span class="author-block">
<a href="https://robotics.iiit.ac.in/faculty_mkrishna/">K. Madhava Krishna</a><sup>1</sup>
</span>
</div>
<div class="is-size-5 publication-authors">
<span class="author-block"><sup>1</sup>KCIS, International Institute of Information Technology, Hyderabad,
India;</span>
<span class="author-block"><sup>2</sup>CSAIL, MIT, Cambridge, United States;</span>
<span class="author-block"><sup>3</sup>Hasan Kalyoncu University, Turkey</span>
</div>
<div class="column has-text-centered">
<div class="publication-links">
<!-- PDF Link. -->
<!-- <span class="link-block">
<a href=""
class="external-link button is-normal is-rounded is-dark">
<span class="icon">
<i class="fas fa-file-pdf"></i>
</span>
<span>Paper</span>
</a>
</span> -->
<span class="link-block">
<a href="https://ieeexplore.ieee.org/abstract/document/10309534"
class="external-link button is-normal is-rounded is-dark">
<span class="icon">
<i class="ai ai-arxiv"></i>
</span>
<span>IEEE</span>
</a>
</span>
<span class="link-block">
<a href="https://arxiv.org/abs/2305.12363"
class="external-link button is-normal is-rounded is-dark">
<span class="icon">
<i class="ai ai-arxiv"></i>
</span>
<span>arXiv</span>
</a>
</span>
<!-- Video Link. -->
<span class="link-block">
<a href="https://www.youtube.com/watch?v=IHtMMeya4ic"
class="external-link button is-normal is-rounded is-dark">
<span class="icon">
<i class="fab fa-youtube"></i>
</span>
<span>Video</span>
</a>
</span>
</div>
</div>
<div class="is-size-6 publication-authors">
<span class="author-block"><b>Note</b> - Published at the conference
<a href="https://ro-man2023.org/main" target="_blank">IEEE RO-MAN 2023</a></span>
</div>
</div>
</div>
</div>
</div>
</section>
<section class="hero teaser">
<div class="container is-max-desktop">
<div class="hero-body">
<img src="teaser.png" alt="SI-Maps Teaser">
</img>
<h2 class="subtitle has-text-centered">
<span class="dnerf">Teaser
</h2>
</div>
</div>
</section>
<section class="section">
<div class="container is-max-desktop">
<!-- Abstract. -->
<div class="columns is-centered has-text-centered">
<div class="column is-four-fifths">
<h2 class="title is-3">Abstract</h2>
<div class="content has-text-justified">
<p>
Humans have a natural ability to perform semantic associations with the surrounding objects in the environment.
This allows them to create a mental map of the environment, allowing them to navigate on-demand when given linguistic instructions.
A natural goal in Vision Language Navigation (VLN) research is to impart autonomous agents with similar capabilities.
Recent works take a step towards this goal by creating a semantic spatial map representation of the environment without any labeled data.
However, their representations are limited for practical applicability as they do not distinguish between different instances of the same object.
In this work, we address this limitation by integrating instance-level information into spatial map representation using a
community detection algorithm and utilizing word ontology learned by large language models (LLMs) to perform open-set semantic associations
in the mapping representation. The resulting map representation improves the navigation performance by two-fold (233%) on realistic
language commands with instance-specific descriptions compared to the baseline. We validate the practicality and effectiveness
of our approach through extensive qualitative and quantitative experiments.
</p>
</div>
</div>
</div>
<!--/ Abstract. -->
</section>
<section class="section">
<div class="container is-max-desktop">
<div class="columns is-centered has-text-centered">
<div class="column is-four-fifths">
<h2 class="title is-4">Language Navigation Demo With SI Maps</h2>
<div class="publication-video">
<iframe src="https://www.youtube.com/embed/I5cjqTAG1y0?si=Bu7PYuyHdfysZn68" title="SI Maps Demo" frameborder="0"
allow="autoplay; encrypted-media;" allowfullscreen></iframe>
</div>
</div>
</div>
<section class="section" id="BibTeX">
<div class="container is-max-desktop content">
<h2 class="title">BibTeX</h2>
<pre><code>@INPROCEEDINGS{10309534,
author={Nanwani, Laksh and Agarwal, Anmol and Jain, Kanishk and Prabhakar, Raghav and Monis, Aaron and Mathur, Aditya and Jatavallabhula, Krishna Murthy and Abdul Hafez, A. H. and Gandhi, Vineet and Krishna, K. Madhava},
booktitle={2023 32nd IEEE International Conference on Robot and Human Interactive Communication (RO-MAN)},
title={Instance-Level Semantic Maps for Vision Language Navigation},
year={2023},
volume={},
number={},
pages={507-512},
keywords={Measurement;Visualization;Three-dimensional displays;Navigation;Semantics;Linguistics;Ontologies},
doi={10.1109/RO-MAN57019.2023.10309534}}</code></pre>
</div>
</section>
<footer class="footer">
<div class="container">
<div class="content has-text-centered">
<a class="icon-link"
href="./static/videos/nerfies_paper.pdf">
<i class="fas fa-file-pdf"></i>
</a>
<a class="icon-link" href="https://github.com/keunhong" class="external-link" disabled>
<i class="fab fa-github"></i>
</a>
</div>
<div class="columns is-centered">
<div class="column is-8">
<div class="content">
<p>
This website is licensed under a <a rel="license"
href="http://creativecommons.org/licenses/by-sa/4.0/">Creative
Commons Attribution-ShareAlike 4.0 International License</a>.
</p>
<p>
It borrows the <a
href="https://github.com/nerfies/nerfies.github.io">source code</a> of this website.
</p>
</div>
</div>
</div>
</div>
</footer>
</body>
</html>