Skip to content

Commit

Permalink
Handle when /collections endpoint returns paginated results (#60435)
Browse files Browse the repository at this point in the history 
* Handle when /collections endpoint returns paginated results

Servers usually return all collections in a single page.
If the results are paginated, get the 5 first pages, just in case the total number of pages is huge.
If 5 pages are still not enough, more will be fetched when scrolling the Collections list view.

* Reset collections page counter when connecting
  • Loading branch information
@uclaros
uclaros authored Feb 18, 2025
1 parent 427dd0e commit 64d82d4
Show file tree
Hide file tree
Showing 4 changed files with 79 additions and 54 deletions.
53 changes: 33 additions & 20 deletions src/gui/stac/qgsstacsearchparametersdialog.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,17 +19,20 @@
#include "qgsmapcanvas.h"
#include "qgsprojecttimesettings.h"
#include "qgsstaccollection.h"
#include "qgsstaccontroller.h"

#include <QPushButton>
#include <QScrollBar>
#include <QStandardItemModel>
#include <QSortFilterProxyModel>
#include <QMenu>
#include <QTextDocument>

///@cond PRIVATE

QgsStacSearchParametersDialog::QgsStacSearchParametersDialog( QgsMapCanvas *canvas, QWidget *parent )
QgsStacSearchParametersDialog::QgsStacSearchParametersDialog( QgsStacController *stac, QgsMapCanvas *canvas, QWidget *parent )
: QDialog( parent )
, mStac( stac )
{
setupUi( this );
QgsGui::enableAutoGeometryRestore( this );
Expand Down Expand Up @@ -64,6 +67,7 @@ QgsStacSearchParametersDialog::QgsStacSearchParametersDialog( QgsMapCanvas *canv
connect( mCollectionsFilterLineEdit, &QgsFilterLineEdit::textChanged, mCollectionsProxyModel, &QSortFilterProxyModel::setFilterFixedString );
connect( mSelectAllCollectionsButton, &QPushButton::clicked, this, &QgsStacSearchParametersDialog::selectAllCollections );
connect( mDeselectAllCollectionsButton, &QPushButton::clicked, this, &QgsStacSearchParametersDialog::deselectAllCollections );
connect( mCollectionsListView->verticalScrollBar(), &QScrollBar::valueChanged, this, &QgsStacSearchParametersDialog::onCollectionsListViewScroll );
connect( mTemporalToolButton, &QToolButton::clicked, this, &QgsStacSearchParametersDialog::readTemporalExtentsFromProject );
}

Expand All @@ -90,17 +94,6 @@ void QgsStacSearchParametersDialog::accept()
mSelectedCollections.insert( mCollectionsModel->data( index, Qt::UserRole ).toString() );
}

// if none was checked, check them all!
if ( mSelectedCollections.isEmpty() )
{
for ( int i = 0; i < mCollectionsModel->rowCount(); ++i )
{
const QModelIndex index = mCollectionsModel->index( i, 0 );
mCollectionsModel->setData( index, Qt::Checked, Qt::CheckStateRole );
mSelectedCollections.insert( mCollectionsModel->data( index, Qt::UserRole ).toString() );
}
}

QDialog::accept();
}

Expand All @@ -127,6 +120,11 @@ void QgsStacSearchParametersDialog::setMapCanvas( QgsMapCanvas *canvas )
mSpatialExtent->setMapCanvas( canvas );
}

void QgsStacSearchParametersDialog::setCollectionsUrl( const QString &url )
{
mCollectionsUrl = url;
}

bool QgsStacSearchParametersDialog::hasTemporalFilter() const
{
return mTemporalFilterEnabled && !( mFromDateTimeEdit->dateTime().isNull() || mToDateTimeEdit->dateTime().isNull() );
Expand Down Expand Up @@ -172,16 +170,21 @@ QSet<QString> QgsStacSearchParametersDialog::selectedCollections() const
return mSelectedCollections;
}

void QgsStacSearchParametersDialog::setCollections( const QVector<QgsStacCollection *> &collections )
void QgsStacSearchParametersDialog::clearCollections()
{
qDeleteAll( mCollections );
mCollections.clear();
mSelectedCollections.clear();
mCollectionsModel->clear();
mCollectionsGroupBox->setChecked( false );
mCollections = collections;
mCollectionsFilterEnabled = false;
}

void QgsStacSearchParametersDialog::appendCollections( const QVector<QgsStacCollection *> &collections )
{
mCollections.append( collections );

QTextDocument descr;
for ( QgsStacCollection *c : std::as_const( mCollections ) )
for ( QgsStacCollection *c : collections )
{
descr.setMarkdown( c->description() );

Expand All @@ -191,7 +194,6 @@ void QgsStacSearchParametersDialog::setCollections( const QVector<QgsStacCollect
i->setCheckable( true );
i->setCheckState( Qt::Checked );
mCollectionsModel->appendRow( i );
mSelectedCollections.insert( c->id() );
}
}

Expand All @@ -202,9 +204,11 @@ QVector<QgsStacCollection *> QgsStacSearchParametersDialog::collections() const

QString QgsStacSearchParametersDialog::activeFiltersPreview()
{
QString str = tr( "%1/%2 Collections" )
.arg( mCollectionsFilterEnabled ? mSelectedCollections.size() : mCollectionsModel->rowCount() )
.arg( mCollectionsModel->rowCount() );
QString str;
if ( mSelectedCollections.isEmpty() || !mCollectionsFilterEnabled )
str = tr( "All Collections" );
else
str = tr( "%1/%2 Collections" ).arg( mSelectedCollections.size() ).arg( mCollectionsModel->rowCount() );

if ( mSpatialFilterEnabled )
{
Expand Down Expand Up @@ -236,6 +240,15 @@ void QgsStacSearchParametersDialog::deselectAllCollections()
}
}

void QgsStacSearchParametersDialog::onCollectionsListViewScroll( int value )
{
if ( !mCollectionsUrl.isEmpty() && value == mCollectionsListView->verticalScrollBar()->maximum() )
{
QgsDebugMsgLevel( QStringLiteral( "Scrolled to end, fetching next page" ), 3 );
mStac->fetchCollectionsAsync( mCollectionsUrl );
}
}

void QgsStacSearchParametersDialog::readTemporalExtentsFromProject()
{
const QgsDateTimeRange projectRange = QgsProject::instance()->timeSettings()->temporalRange();
Expand Down
12 changes: 10 additions & 2 deletions src/gui/stac/qgsstacsearchparametersdialog.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,19 +28,21 @@
class QgsMapCanvas;
class QStandardItemModel;
class QSortFilterProxyModel;
class QgsStacController;

class QgsStacSearchParametersDialog : public QDialog, private Ui::QgsStacSearchParametersDialog
{
Q_OBJECT

public:
QgsStacSearchParametersDialog( QgsMapCanvas *canvas, QWidget *parent = nullptr );
QgsStacSearchParametersDialog( QgsStacController *stac, QgsMapCanvas *canvas, QWidget *parent = nullptr );
~QgsStacSearchParametersDialog();

void accept() override;
void reject() override;

void setMapCanvas( QgsMapCanvas *canvas );
void setCollectionsUrl( const QString &url );

bool hasTemporalFilter() const;
bool hasSpatialFilter() const;
Expand All @@ -50,8 +52,10 @@ class QgsStacSearchParametersDialog : public QDialog, private Ui::QgsStacSearchP
QgsDateTimeRange temporalRange() const;
QSet<QString> selectedCollections() const;

//! clears model, deletes pointers
void clearCollections();
//! takes ownership
void setCollections( const QVector<QgsStacCollection *> &collections );
void appendCollections( const QVector<QgsStacCollection *> &collections );

//! ownership not transferred
QVector<QgsStacCollection *> collections() const;
Expand All @@ -61,9 +65,11 @@ class QgsStacSearchParametersDialog : public QDialog, private Ui::QgsStacSearchP
private:
void selectAllCollections();
void deselectAllCollections();
void onCollectionsListViewScroll( int value );

void readTemporalExtentsFromProject();

QString mCollectionsUrl;
bool mSpatialFilterEnabled = false;
bool mTemporalFilterEnabled = false;
bool mCollectionsFilterEnabled = false;
Expand All @@ -73,6 +79,8 @@ class QgsStacSearchParametersDialog : public QDialog, private Ui::QgsStacSearchP
QDateTime mTemporalTo;
QSet<QString> mSelectedCollections;

QgsStacController *mStac = nullptr;

QMenu *mMenu = nullptr;
QAction *mTemporalExtentFromProjectAction = nullptr;

Expand Down
62 changes: 33 additions & 29 deletions src/gui/stac/qgsstacsourceselect.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,8 +67,6 @@ QgsStacSourceSelect::QgsStacSourceSelect( QWidget *parent, Qt::WindowFlags fl, Q
connect( mStac, &QgsStacController::finishedItemCollectionRequest, this, &QgsStacSourceSelect::onItemCollectionRequestFinished );
connect( mStac, &QgsStacController::finishedCollectionsRequest, this, &QgsStacSourceSelect::onCollectionsRequestFinished );

connect( mFiltersButton, &QToolButton::clicked, this, &QgsStacSourceSelect::openSearchParametersDialog );

mItemsView->setModel( mItemsModel );
mItemsView->setItemDelegate( new QgsStacItemDelegate );
mItemsView->setVerticalScrollMode( QAbstractItemView::ScrollPerPixel );
Expand All @@ -81,7 +79,11 @@ QgsStacSourceSelect::QgsStacSourceSelect( QWidget *parent, Qt::WindowFlags fl, Q

connect( mFootprintsCheckBox, &QCheckBox::clicked, this, &QgsStacSourceSelect::showFootprints );

mParametersDialog = new QgsStacSearchParametersDialog( mapCanvas(), this );
mParametersDialog = new QgsStacSearchParametersDialog( mStac, mapCanvas(), this );

connect( mFiltersButton, &QToolButton::clicked, mParametersDialog, &QgsStacSearchParametersDialog::open );
connect( mParametersDialog, &QgsStacSearchParametersDialog::finished, this, &QgsStacSourceSelect::onSearchParametersDialogClosed );

mFiltersLabel->clear();
}

Expand Down Expand Up @@ -162,12 +164,12 @@ void QgsStacSourceSelect::btnConnect_clicked()
const QgsStacConnection::Data connection = QgsStacConnection::connection( cmbConnections->currentText() );

mStac->setAuthCfg( connection.authCfg );
mCollectionsUrl.clear();
mSearchUrl.clear();
mNextPageUrl.clear();
mItemsModel->clear();
qDeleteAll( mRubberBands );
mRubberBands.clear();
mCollectionsPageCounter = 0;
mStatusLabel->setText( tr( "Connecting…" ) );
mStac->cancelPendingAsyncRequests();
mStac->fetchStacObjectAsync( connection.url );
Expand Down Expand Up @@ -288,6 +290,7 @@ void QgsStacSourceSelect::onStacObjectRequestFinished( int requestId, QString er
const bool supportsCollections = cat->conformsTo( QStringLiteral( "https://api.stacspec.org/v1.0.0/collections" ) );
const bool supportsSearch = cat->conformsTo( QStringLiteral( "https://api.stacspec.org/v1.0.0/item-search" ) );
QgsDebugMsgLevel( QStringLiteral( "STAC catalog supports API: %1" ).arg( supportsCollections && supportsSearch ), 2 );
QString collectionsUrl;

if ( supportsCollections && supportsSearch )
{
Expand All @@ -296,24 +299,24 @@ void QgsStacSourceSelect::onStacObjectRequestFinished( int requestId, QString er
// collections endpoint should have a "data" relation according to spec but some servers don't
// so let's be less strict and only check the href
if ( l.href().endsWith( "/collections" ) )
mCollectionsUrl = l.href();
collectionsUrl = l.href();
else if ( l.relation() == "search" )
mSearchUrl = l.href();

if ( !mCollectionsUrl.isEmpty() && !mSearchUrl.isEmpty() )
if ( !collectionsUrl.isEmpty() && !mSearchUrl.isEmpty() )
break;
}
}

if ( mCollectionsUrl.isEmpty() || mSearchUrl.isEmpty() )
if ( collectionsUrl.isEmpty() || mSearchUrl.isEmpty() )
{
mStatusLabel->setText( tr( "Server does not support STAC search API" ) );
}
else
{
mStatusLabel->setText( tr( "Fetching Collections…" ) );
mStac->cancelPendingAsyncRequests();
mStac->fetchCollectionsAsync( mCollectionsUrl );
mStac->fetchCollectionsAsync( collectionsUrl );
mParametersDialog->clearCollections();
mStatusLabel->setText( tr( "Fetching collections…" ) );
}
}

Expand All @@ -325,14 +328,24 @@ void QgsStacSourceSelect::onCollectionsRequestFinished( int requestId, QString e
if ( !cols )
{
mStatusLabel->setText( error );
mParametersDialog->setCollections( {} );
updateFilterPreview();
return;
}

const QVector<QgsStacCollection *> vcols = cols->takeCollections();
mParametersDialog->setCollections( vcols );
mParametersDialog->appendCollections( vcols );
mItemsModel->setCollections( vcols );

// Let's try to grab the first 5 pages of /collections endpoint before searching
// In most cases all collections will be returned in the first page but some servers are weird
if ( mCollectionsPageCounter < 5 && !cols->nextUrl().isEmpty() )
{
++mCollectionsPageCounter;
mStac->fetchCollectionsAsync( cols->nextUrl() );
return;
}

mParametersDialog->setCollectionsUrl( cols->nextUrl().toString() );

mStatusLabel->setText( tr( "Searching…" ) );
mFiltersButton->setEnabled( true );
updateFilterPreview();
Expand Down Expand Up @@ -392,26 +405,17 @@ void QgsStacSourceSelect::onItemCollectionRequestFinished( int requestId, QStrin

void QgsStacSourceSelect::search()
{
QUrlQuery q;

QStringList collections;
if ( mParametersDialog->hasCollectionsFilter() )
{
const QSet<QString> collectionsSet = mParametersDialog->selectedCollections();
collections = QStringList( collectionsSet.constBegin(), collectionsSet.constEnd() );
}
else
{
const QVector<QgsStacCollection *> allCollections = mParametersDialog->collections();
for ( QgsStacCollection *col : allCollections )
{
collections.append( col->id() );
}
const QList<QPair<QString, QString>> collectionsParameters = { qMakePair( QStringLiteral( "collections" ), collections.join( "," ) ) };
q.setQueryItems( collectionsParameters );
}

QUrlQuery q;

QList<QPair<QString, QString>> collectionsParameters = { qMakePair( QStringLiteral( "collections" ), collections.join( "," ) ) };
q.setQueryItems( collectionsParameters );

if ( mParametersDialog->hasSpatialFilter() )
{
const QgsGeometry geom = mParametersDialog->spatialExtent();
Expand Down Expand Up @@ -453,15 +457,15 @@ void QgsStacSourceSelect::search()
mStac->fetchItemCollectionAsync( searchUrl );
}

void QgsStacSourceSelect::openSearchParametersDialog()
void QgsStacSourceSelect::onSearchParametersDialogClosed( int result )
{
if ( mParametersDialog->exec() == QDialog::Rejected )
if ( result == QDialog::Rejected )
return;

mItemsModel->clear();
mItemsModel->setCollections( mParametersDialog->collections() );
qDeleteAll( mRubberBands );
mRubberBands.clear();
mItemsModel->setCollections( mParametersDialog->collections() );
mNextPageUrl.clear();
mStatusLabel->setText( tr( "Searching…" ) );
updateFilterPreview();
Expand Down
6 changes: 3 additions & 3 deletions src/gui/stac/qgsstacsourceselect.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,8 +75,8 @@ class GUI_EXPORT QgsStacSourceSelect : public QgsAbstractDataSourceWidget, priva
//! Called when search requests finish
void onItemCollectionRequestFinished( int requestId, QString error );

//! Opens the filter parameters dialog
void openSearchParametersDialog();
//! Called when the filter parameters dialog is closed
void onSearchParametersDialogClosed( int result );

//! Called when scrolling to fetch next page when scrolled to the end
void onItemsViewScroll( int value );
Expand All @@ -102,9 +102,9 @@ class GUI_EXPORT QgsStacSourceSelect : public QgsAbstractDataSourceWidget, priva
void showFootprints( bool enable );
void loadUri( const QgsMimeDataUtils::Uri &uri );

QString mCollectionsUrl;
QString mSearchUrl;
QUrl mNextPageUrl;
int mCollectionsPageCounter = 0;

QgsStacController *mStac = nullptr;
QgsStacItemListModel *mItemsModel = nullptr;
Expand Down

0 comments on commit 64d82d4

Please sign in to comment.