From ed4a681b22a498ab2bd026abe99173b1f035d427 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Ko=C5=88a=C5=99=C3=ADk?= Date: Thu, 14 Nov 2024 15:23:31 +0100 Subject: [PATCH] Merge remote and local QgsEptPointCloudIndex classes --- src/core/CMakeLists.txt | 2 - src/core/pointcloud/qgseptpointcloudindex.cpp | 417 +++++++++++++----- src/core/pointcloud/qgseptpointcloudindex.h | 15 +- src/core/pointcloud/qgspointcloudindex.cpp | 13 +- src/core/pointcloud/qgspointcloudindex.h | 5 - .../qgsremoteeptpointcloudindex.cpp | 275 ------------ .../pointcloud/qgsremoteeptpointcloudindex.h | 86 ---- src/core/providers/ept/qgseptprovider.cpp | 6 +- src/core/providers/ept/qgseptprovider.h | 1 - .../vpc/qgsvirtualpointcloudprovider.cpp | 19 +- 10 files changed, 335 insertions(+), 504 deletions(-) delete mode 100644 src/core/pointcloud/qgsremoteeptpointcloudindex.cpp delete mode 100644 src/core/pointcloud/qgsremoteeptpointcloudindex.h diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index e8f9f5a074e9..91cc4dadd3a1 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -2236,13 +2236,11 @@ if (WITH_EPT) set(QGIS_CORE_SRCS ${QGIS_CORE_SRCS} providers/ept/qgseptprovider.cpp pointcloud/qgseptpointcloudindex.cpp - pointcloud/qgsremoteeptpointcloudindex.cpp pointcloud/qgseptpointcloudblockrequest.cpp ) set(QGIS_CORE_HDRS ${QGIS_CORE_HDRS} providers/ept/qgseptprovider.h pointcloud/qgseptpointcloudindex.h - pointcloud/qgsremoteeptpointcloudindex.h pointcloud/qgseptpointcloudblockrequest.h ) diff --git a/src/core/pointcloud/qgseptpointcloudindex.cpp b/src/core/pointcloud/qgseptpointcloudindex.cpp index 0a89995aacef..726438553cba 100644 --- a/src/core/pointcloud/qgseptpointcloudindex.cpp +++ b/src/core/pointcloud/qgseptpointcloudindex.cpp @@ -26,21 +26,31 @@ #include #include #include +#include +#include "qgsapplication.h" +#include "qgsblockingnetworkrequest.h" +#include "qgscachedpointcloudblockrequest.h" #include "qgseptdecoder.h" +#include "qgseptpointcloudblockrequest.h" #include "qgslazdecoder.h" #include "qgscoordinatereferencesystem.h" +#include "qgspointcloudblockrequest.h" #include "qgspointcloudrequest.h" #include "qgspointcloudattribute.h" #include "qgslogger.h" #include "qgspointcloudexpression.h" +#include "qgssetrequestinitiator_p.h" ///@cond PRIVATE #define PROVIDER_KEY QStringLiteral( "ept" ) #define PROVIDER_DESCRIPTION QStringLiteral( "EPT point cloud provider" ) -QgsEptPointCloudIndex::QgsEptPointCloudIndex() = default; +QgsEptPointCloudIndex::QgsEptPointCloudIndex() +{ + mHierarchyNodes.insert( IndexedPointCloudNode( 0, 0, 0, 0 ) ); +} QgsEptPointCloudIndex::~QgsEptPointCloudIndex() = default; @@ -52,37 +62,79 @@ std::unique_ptr QgsEptPointCloudIndex::clone() const return std::unique_ptr( clone ); } -void QgsEptPointCloudIndex::load( const QString &fileName ) +void QgsEptPointCloudIndex::load( const QString &urlString ) { - mUri = fileName; - QFile f( fileName ); - if ( !f.open( QIODevice::ReadOnly ) ) + QUrl url = urlString; + // Treat non-URLs as local files + if ( url.isValid() && ( url.scheme() == "http" || url.scheme() == "https" ) ) + mAccessType = Remote; + else + mAccessType = Local; + mUri = urlString; + + QStringList splitUrl = mUri.split( '/' ); + splitUrl.pop_back(); + mUrlDirectoryPart = splitUrl.join( '/' ); + + QByteArray content; + if ( mAccessType == Remote ) { - mError = tr( "Unable to open %1 for reading" ).arg( fileName ); - mIsValid = false; - return; - } - - const QDir directory = QFileInfo( fileName ).absoluteDir(); - mDirectory = directory.absolutePath(); + QNetworkRequest nr = QNetworkRequest( QUrl( mUri ) ); + QgsSetRequestInitiatorClass( nr, QStringLiteral( "QgsEptPointCloudIndex" ) ); - const QByteArray dataJson = f.readAll(); - bool success = loadSchema( dataJson ); + QgsBlockingNetworkRequest req; + if ( req.get( nr ) != QgsBlockingNetworkRequest::NoError ) + { + QgsDebugError( QStringLiteral( "Request failed: " ) + mUri ); + mIsValid = false; + mError = req.errorMessage(); + return; + } + content = req.reply().content(); + } + else + { + QFile f( mUri ); + if ( !f.open( QIODevice::ReadOnly ) ) + { + mError = tr( "Unable to open %1 for reading" ).arg( mUri ); + mIsValid = false; + return; + } + content = f.readAll(); + } + bool success = loadSchema( content ); if ( success ) { // try to import the metadata too! - QFile manifestFile( mDirectory + QStringLiteral( "/ept-sources/manifest.json" ) ); - if ( manifestFile.open( QIODevice::ReadOnly ) ) + const QString manifestPath = mUrlDirectoryPart + QStringLiteral( "/ept-sources/manifest.json" ); + QByteArray manifestJson; + if ( mAccessType == Remote ) { - const QByteArray manifestJson = manifestFile.readAll(); - loadManifest( manifestJson ); + QUrl manifestUrl( manifestPath ); + + QNetworkRequest nr = QNetworkRequest( QUrl( manifestUrl ) ); + QgsSetRequestInitiatorClass( nr, QStringLiteral( "QgsEptPointCloudIndex" ) ); + QgsBlockingNetworkRequest req; + if ( req.get( nr ) == QgsBlockingNetworkRequest::NoError ) + manifestJson = req.reply().content(); + } + else + { + QFile manifestFile( manifestPath ); + if ( manifestFile.open( QIODevice::ReadOnly ) ) + manifestJson = manifestFile.readAll(); } + + if ( !manifestJson.isEmpty() ) + loadManifest( manifestJson ); } - if ( success ) + if ( !loadNodeHierarchy( IndexedPointCloudNode( 0, 0, 0, 0 ) ) ) { - success = loadHierarchy(); + QgsDebugError( QStringLiteral( "Failed to load root EPT node" ) ); + success = false; } mIsValid = success; @@ -93,31 +145,47 @@ void QgsEptPointCloudIndex::loadManifest( const QByteArray &manifestJson ) QJsonParseError err; // try to import the metadata too! const QJsonDocument manifestDoc = QJsonDocument::fromJson( manifestJson, &err ); - if ( err.error == QJsonParseError::NoError ) + if ( err.error != QJsonParseError::NoError ) + return; + + const QJsonArray manifestArray = manifestDoc.array(); + if ( manifestArray.empty() ) + return; + + // TODO how to handle multiple? + const QJsonObject sourceObject = manifestArray.at( 0 ).toObject(); + const QString metadataPath = sourceObject.value( QStringLiteral( "metadataPath" ) ).toString(); + const QString fullMetadataPath = mUrlDirectoryPart + QStringLiteral( "/ept-sources/" ) + metadataPath; + + QByteArray metadataJson; + if ( mAccessType == Remote ) { - const QJsonArray manifestArray = manifestDoc.array(); - // TODO how to handle multiple? - if ( ! manifestArray.empty() ) - { - const QJsonObject sourceObject = manifestArray.at( 0 ).toObject(); - const QString metadataPath = sourceObject.value( QStringLiteral( "metadataPath" ) ).toString(); - QFile metadataFile( mDirectory + QStringLiteral( "/ept-sources/" ) + metadataPath ); - if ( metadataFile.open( QIODevice::ReadOnly ) ) - { - const QByteArray metadataJson = metadataFile.readAll(); - const QJsonDocument metadataDoc = QJsonDocument::fromJson( metadataJson, &err ); - if ( err.error == QJsonParseError::NoError ) - { - const QJsonObject metadataObject = metadataDoc.object().value( QStringLiteral( "metadata" ) ).toObject(); - if ( !metadataObject.empty() ) - { - const QJsonObject sourceMetadata = metadataObject.constBegin().value().toObject(); - mOriginalMetadata = sourceMetadata.toVariantMap(); - } - } - } - } + QUrl metadataUrl( fullMetadataPath ); + QNetworkRequest nr = QNetworkRequest( QUrl( metadataUrl ) ); + QgsSetRequestInitiatorClass( nr, QStringLiteral( "QgsEptPointCloudIndex" ) ); + QgsBlockingNetworkRequest req; + if ( req.get( nr ) != QgsBlockingNetworkRequest::NoError ) + return; + metadataJson = req.reply().content(); } + else + { + QFile metadataFile( fullMetadataPath ); + if ( ! metadataFile.open( QIODevice::ReadOnly ) ) + return; + metadataJson = metadataFile.readAll(); + } + + const QJsonDocument metadataDoc = QJsonDocument::fromJson( metadataJson, &err ); + if ( err.error != QJsonParseError::NoError ) + return; + + const QJsonObject metadataObject = metadataDoc.object().value( QStringLiteral( "metadata" ) ).toObject(); + if ( metadataObject.empty() ) + return; + + const QJsonObject sourceMetadata = metadataObject.constBegin().value().toObject(); + mOriginalMetadata = sourceMetadata.toVariantMap(); } bool QgsEptPointCloudIndex::loadSchema( const QByteArray &dataJson ) @@ -315,47 +383,98 @@ std::unique_ptr QgsEptPointCloudIndex::nodeData( const Index return std::unique_ptr( cached ); } - mHierarchyMutex.lock(); - const bool found = mHierarchy.contains( n ); - mHierarchyMutex.unlock(); - if ( !found ) + std::unique_ptr block; + if ( mAccessType == Remote ) + { + std::unique_ptr blockRequest( asyncNodeData( n, request ) ); + if ( !blockRequest ) + return nullptr; + + QEventLoop loop; + connect( blockRequest.get(), &QgsPointCloudBlockRequest::finished, &loop, &QEventLoop::quit ); + loop.exec(); + + block = blockRequest->takeBlock(); + if ( !block ) + { + QgsDebugError( QStringLiteral( "Error downloading node %1 data, error : %2 " ).arg( n.toString(), blockRequest->errorStr() ) ); + } + } + else + { + // we need to create a copy of the expression to pass to the decoder + // as the same QgsPointCloudExpression object mighgt be concurrently + // used on another thread, for example in a 3d view + QgsPointCloudExpression filterExpression = mFilterExpression; + QgsPointCloudAttributeCollection requestAttributes = request.attributes(); + requestAttributes.extend( attributes(), filterExpression.referencedAttributes() ); + QgsRectangle filterRect = request.filterRect(); + + if ( mDataType == QLatin1String( "binary" ) ) + { + const QString filename = QStringLiteral( "%1/ept-data/%2.bin" ).arg( mUrlDirectoryPart, n.toString() ); + block = QgsEptDecoder::decompressBinary( filename, attributes(), requestAttributes, scale(), offset(), filterExpression, filterRect ); + } + else if ( mDataType == QLatin1String( "zstandard" ) ) + { + const QString filename = QStringLiteral( "%1/ept-data/%2.zst" ).arg( mUrlDirectoryPart, n.toString() ); + block = QgsEptDecoder::decompressZStandard( filename, attributes(), request.attributes(), scale(), offset(), filterExpression, filterRect ); + } + else if ( mDataType == QLatin1String( "laszip" ) ) + { + const QString filename = QStringLiteral( "%1/ept-data/%2.laz" ).arg( mUrlDirectoryPart, n.toString() ); + block = QgsLazDecoder::decompressLaz( filename, requestAttributes, filterExpression, filterRect ); + } + } + + storeNodeDataToCache( block.get(), n, request ); + return block; +} + +QgsPointCloudBlockRequest *QgsEptPointCloudIndex::asyncNodeData( const IndexedPointCloudNode &n, const QgsPointCloudRequest &request ) +{ + if ( QgsPointCloudBlock *cached = getNodeDataFromCache( n, request ) ) + { + return new QgsCachedPointCloudBlockRequest( cached, n, mUri, attributes(), request.attributes(), + scale(), offset(), mFilterExpression, request.filterRect() ); + } + + if ( mAccessType != Remote ) return nullptr; - // we need to create a copy of the expression to pass to the decoder - // as the same QgsPointCloudExpression object mighgt be concurrently - // used on another thread, for example in a 3d view - QgsPointCloudExpression filterExpression = mFilterExpression; - QgsPointCloudAttributeCollection requestAttributes = request.attributes(); - requestAttributes.extend( attributes(), filterExpression.referencedAttributes() ); - QgsRectangle filterRect = request.filterRect(); + if ( !loadNodeHierarchy( n ) ) + return nullptr; - std::unique_ptr decoded; + QString fileUrl; if ( mDataType == QLatin1String( "binary" ) ) { - const QString filename = QStringLiteral( "%1/ept-data/%2.bin" ).arg( mDirectory, n.toString() ); - decoded = QgsEptDecoder::decompressBinary( filename, attributes(), requestAttributes, scale(), offset(), filterExpression, filterRect ); + fileUrl = QStringLiteral( "%1/ept-data/%2.bin" ).arg( mUrlDirectoryPart, n.toString() ); } else if ( mDataType == QLatin1String( "zstandard" ) ) { - const QString filename = QStringLiteral( "%1/ept-data/%2.zst" ).arg( mDirectory, n.toString() ); - decoded = QgsEptDecoder::decompressZStandard( filename, attributes(), request.attributes(), scale(), offset(), filterExpression, filterRect ); + fileUrl = QStringLiteral( "%1/ept-data/%2.zst" ).arg( mUrlDirectoryPart, n.toString() ); } else if ( mDataType == QLatin1String( "laszip" ) ) { - const QString filename = QStringLiteral( "%1/ept-data/%2.laz" ).arg( mDirectory, n.toString() ); - decoded = QgsLazDecoder::decompressLaz( filename, requestAttributes, filterExpression, filterRect ); + fileUrl = QStringLiteral( "%1/ept-data/%2.laz" ).arg( mUrlDirectoryPart, n.toString() ); + } + else + { + return nullptr; } - storeNodeDataToCache( decoded.get(), n, request ); - return decoded; + // we need to create a copy of the expression to pass to the decoder + // as the same QgsPointCloudExpression object might be concurrently + // used on another thread, for example in a 3d view + QgsPointCloudExpression filterExpression = mFilterExpression; + QgsPointCloudAttributeCollection requestAttributes = request.attributes(); + requestAttributes.extend( attributes(), filterExpression.referencedAttributes() ); + return new QgsEptPointCloudBlockRequest( n, fileUrl, mDataType, attributes(), requestAttributes, scale(), offset(), filterExpression, request.filterRect() ); } -QgsPointCloudBlockRequest *QgsEptPointCloudIndex::asyncNodeData( const IndexedPointCloudNode &n, const QgsPointCloudRequest &request ) +bool QgsEptPointCloudIndex::hasNode( const IndexedPointCloudNode &n ) const { - Q_UNUSED( n ); - Q_UNUSED( request ); - Q_ASSERT( false ); - return nullptr; // unsupported + return loadNodeHierarchy( n ); } QgsCoordinateReferenceSystem QgsEptPointCloudIndex::crs() const @@ -368,6 +487,32 @@ qint64 QgsEptPointCloudIndex::pointCount() const return mPointCount; } +qint64 QgsEptPointCloudIndex::nodePointCount( const IndexedPointCloudNode &nodeId ) const +{ + // First try loading our cached value + { + QMutexLocker locker( &mHierarchyMutex ); + qint64 pointCount = mHierarchy.value( nodeId, -1 ); + if ( pointCount != -1 ) + return pointCount; + } + + // Try loading all nodes' hierarchy files on the path from root and stop when + // one contains the point count for nodeId + QVector pathToRoot = nodePathToRoot( nodeId ); + for ( int i = pathToRoot.size() - 1; i >= 0; --i ) + { + loadSingleNodeHierarchy( pathToRoot[i] ); + + QMutexLocker locker( &mHierarchyMutex ); + qint64 pointCount = mHierarchy.value( nodeId, -1 ); + if ( pointCount != -1 ) + return pointCount; + } + + return -1; +} + bool QgsEptPointCloudIndex::hasStatisticsMetadata() const { return !mMetadataStats.isEmpty(); @@ -439,57 +584,126 @@ QVariant QgsEptPointCloudIndex::metadataClassStatistic( const QString &attribute return values.value( value.toInt() ); } -bool QgsEptPointCloudIndex::loadHierarchy() +bool QgsEptPointCloudIndex::loadSingleNodeHierarchy( const IndexedPointCloudNode &nodeId ) const { - QQueue queue; - queue.enqueue( QStringLiteral( "0-0-0-0" ) ); - while ( !queue.isEmpty() ) + mHierarchyMutex.lock(); + const bool foundInHierarchy = mHierarchy.contains( nodeId ); + const bool foundInHierarchyNodes = mHierarchyNodes.contains( nodeId ); + mHierarchyMutex.unlock(); + // The hierarchy of the node is found => No need to load its file + if ( foundInHierarchy ) + return true; + // We don't know that this node has a hierarchy file => We have nothing to load + if ( !foundInHierarchyNodes ) + return true; + + const QString filePath = QStringLiteral( "%1/ept-hierarchy/%2.json" ).arg( mUrlDirectoryPart, nodeId.toString() ); + + QByteArray dataJsonH; + if ( mAccessType == Remote ) { - const QString filename = QStringLiteral( "%1/ept-hierarchy/%2.json" ).arg( mDirectory, queue.dequeue() ); - QFile fH( filename ); - if ( !fH.open( QIODevice::ReadOnly ) ) + QNetworkRequest nr( filePath ); + QgsSetRequestInitiatorClass( nr, QStringLiteral( "QgsEptPointCloudIndex" ) ); + nr.setAttribute( QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::PreferCache ); + nr.setAttribute( QNetworkRequest::CacheSaveControlAttribute, true ); + + std::unique_ptr reply( QgsApplication::tileDownloadManager()->get( nr ) ); + + QEventLoop loop; + connect( reply.get(), &QgsTileDownloadManagerReply::finished, &loop, &QEventLoop::quit ); + loop.exec(); + + if ( reply->error() != QNetworkReply::NoError ) { - QgsDebugMsgLevel( QStringLiteral( "unable to read hierarchy from file %1" ).arg( filename ), 2 ); - mError = QStringLiteral( "unable to read hierarchy from file %1" ).arg( filename ); + QgsDebugError( QStringLiteral( "Request failed: " ) + filePath ); return false; } - const QByteArray dataJsonH = fH.readAll(); - QJsonParseError errH; - const QJsonDocument docH = QJsonDocument::fromJson( dataJsonH, &errH ); - if ( errH.error != QJsonParseError::NoError ) + dataJsonH = reply->data(); + } + else + { + QFile file( filePath ); + if ( ! file.open( QIODevice::ReadOnly ) ) { - QgsDebugMsgLevel( QStringLiteral( "QJsonParseError when reading hierarchy from file %1" ).arg( filename ), 2 ); - mError = QStringLiteral( "QJsonParseError when reading hierarchy from file %1" ).arg( filename ); + QgsDebugError( QStringLiteral( "Loading file failed: " ) + filePath ); return false; } + dataJsonH = file.readAll(); + } - const QJsonObject rootHObj = docH.object(); - for ( auto it = rootHObj.constBegin(); it != rootHObj.constEnd(); ++it ) - { - const QString nodeIdStr = it.key(); - const int nodePointCount = it.value().toInt(); - if ( nodePointCount < 0 ) - { - queue.enqueue( nodeIdStr ); - } - else - { - const IndexedPointCloudNode nodeId = IndexedPointCloudNode::fromString( nodeIdStr ); - mHierarchyMutex.lock(); - mHierarchy[nodeId] = nodePointCount; - mHierarchyMutex.unlock(); - } - } + QJsonParseError errH; + const QJsonDocument docH = QJsonDocument::fromJson( dataJsonH, &errH ); + if ( errH.error != QJsonParseError::NoError ) + { + QgsDebugMsgLevel( QStringLiteral( "QJsonParseError when reading hierarchy from file %1" ).arg( filePath ), 2 ); + return false; + } + + QMutexLocker locker( &mHierarchyMutex ); + const QJsonObject rootHObj = docH.object(); + for ( auto it = rootHObj.constBegin(); it != rootHObj.constEnd(); ++it ) + { + const QString nodeIdStr = it.key(); + const int nodePointCount = it.value().toInt(); + const IndexedPointCloudNode nodeId = IndexedPointCloudNode::fromString( nodeIdStr ); + if ( nodePointCount >= 0 ) + mHierarchy[nodeId] = nodePointCount; + else if ( nodePointCount == -1 ) + mHierarchyNodes.insert( nodeId ); } + return true; } +QVector QgsEptPointCloudIndex::nodePathToRoot( const IndexedPointCloudNode &nodeId ) const +{ + QVector path; + IndexedPointCloudNode currentNode = nodeId; + do + { + path.push_back( currentNode ); + currentNode = currentNode.parentNode(); + } + while ( currentNode.d() >= 0 ); + + return path; +} + +bool QgsEptPointCloudIndex::loadNodeHierarchy( const IndexedPointCloudNode &nodeId ) const +{ + mHierarchyMutex.lock(); + bool found = mHierarchy.contains( nodeId ); + mHierarchyMutex.unlock(); + if ( found ) + return true; + + QVector pathToRoot = nodePathToRoot( nodeId ); + for ( int i = pathToRoot.size() - 1; i >= 0 && !mHierarchy.contains( nodeId ); --i ) + { + const IndexedPointCloudNode node = pathToRoot[i]; + if ( !loadSingleNodeHierarchy( node ) ) + return false; + } + + mHierarchyMutex.lock(); + found = mHierarchy.contains( nodeId ); + mHierarchyMutex.unlock(); + + return found; +} + + bool QgsEptPointCloudIndex::isValid() const { return mIsValid; } +QgsPointCloudIndex::AccessType QgsEptPointCloudIndex::accessType() const +{ + return mAccessType; +} + void QgsEptPointCloudIndex::copyCommonProperties( QgsEptPointCloudIndex *destination ) const { QgsPointCloudIndex::copyCommonProperties( destination ); @@ -497,8 +711,9 @@ void QgsEptPointCloudIndex::copyCommonProperties( QgsEptPointCloudIndex *destina // QgsEptPointCloudIndex specific fields destination->mIsValid = mIsValid; destination->mDataType = mDataType; - destination->mDirectory = mDirectory; + destination->mUrlDirectoryPart = mUrlDirectoryPart; destination->mWkt = mWkt; + destination->mHierarchyNodes = mHierarchyNodes; destination->mPointCount = mPointCount; destination->mMetadataStats = mMetadataStats; destination->mAttributeClasses = mAttributeClasses; diff --git a/src/core/pointcloud/qgseptpointcloudindex.h b/src/core/pointcloud/qgseptpointcloudindex.h index 8d99dd4048d4..3a7259c1f68f 100644 --- a/src/core/pointcloud/qgseptpointcloudindex.h +++ b/src/core/pointcloud/qgseptpointcloudindex.h @@ -48,9 +48,11 @@ class CORE_EXPORT QgsEptPointCloudIndex: public QgsPointCloudIndex std::unique_ptr nodeData( const IndexedPointCloudNode &n, const QgsPointCloudRequest &request ) override; QgsPointCloudBlockRequest *asyncNodeData( const IndexedPointCloudNode &n, const QgsPointCloudRequest &request ) override; + bool hasNode( const IndexedPointCloudNode &n ) const override; QgsCoordinateReferenceSystem crs() const override; qint64 pointCount() const override; + virtual qint64 nodePointCount( const IndexedPointCloudNode &n ) const override; bool hasStatisticsMetadata() const override; QVariant metadataStatistic( const QString &attribute, Qgis::Statistic statistic ) const override; QVariantList metadataClasses( const QString &attribute ) const override; @@ -58,7 +60,7 @@ class CORE_EXPORT QgsEptPointCloudIndex: public QgsPointCloudIndex QVariantMap originalMetadata() const override { return mOriginalMetadata; } bool isValid() const override; - QgsPointCloudIndex::AccessType accessType() const override { return QgsPointCloudIndex::Local; }; + QgsPointCloudIndex::AccessType accessType() const override; /** * Copies common properties to the \a destination index @@ -70,13 +72,20 @@ class CORE_EXPORT QgsEptPointCloudIndex: public QgsPointCloudIndex bool loadSchema( const QByteArray &dataJson ); void loadManifest( const QByteArray &manifestJson ); bool loadSchema( QFile &f ); - bool loadHierarchy(); + bool loadSingleNodeHierarchy( const IndexedPointCloudNode &nodeId ) const; + QVector nodePathToRoot( const IndexedPointCloudNode &nodeId ) const; + bool loadNodeHierarchy( const IndexedPointCloudNode &nodeId ) const; bool mIsValid = false; + QgsPointCloudIndex::AccessType mAccessType = Local; QString mDataType; - QString mDirectory; QString mWkt; + QString mUrlDirectoryPart; + + //! Contains the nodes that will have */ept-hierarchy/d-x-y-z.json file + mutable QSet mHierarchyNodes; + qint64 mPointCount = 0; struct AttributeStatistics diff --git a/src/core/pointcloud/qgspointcloudindex.cpp b/src/core/pointcloud/qgspointcloudindex.cpp index 962e72e41dcd..12f0e4f8955c 100644 --- a/src/core/pointcloud/qgspointcloudindex.cpp +++ b/src/core/pointcloud/qgspointcloudindex.cpp @@ -201,8 +201,7 @@ qint64 QgsPointCloudIndex::nodePointCount( const IndexedPointCloudNode &n ) cons QList QgsPointCloudIndex::nodeChildren( const IndexedPointCloudNode &n ) const { - QMutexLocker locker( &mHierarchyMutex ); - Q_ASSERT( mHierarchy.contains( n ) ); + Q_ASSERT( hasNode( n ) ); QList lst; const int d = n.d() + 1; const int x = n.x() * 2; @@ -213,7 +212,7 @@ QList QgsPointCloudIndex::nodeChildren( const IndexedPoin { int dx = i & 1, dy = !!( i & 2 ), dz = !!( i & 4 ); const IndexedPointCloudNode n2( d, x + dx, y + dy, z + dz ); - if ( mHierarchy.contains( n2 ) ) + if ( hasNode( n2 ) ) lst.append( n2 ); } return lst; @@ -278,14 +277,6 @@ int QgsPointCloudIndex::span() const return mSpan; } -int QgsPointCloudIndex::nodePointCount( const IndexedPointCloudNode &n ) -{ - mHierarchyMutex.lock(); - const int count = mHierarchy.value( n, -1 ); - mHierarchyMutex.unlock(); - return count; -} - bool QgsPointCloudIndex::setSubsetString( const QString &subset ) { const QString lastExpression = mFilterExpression; diff --git a/src/core/pointcloud/qgspointcloudindex.h b/src/core/pointcloud/qgspointcloudindex.h index c77dfe86b31f..e889e4c2ef85 100644 --- a/src/core/pointcloud/qgspointcloudindex.h +++ b/src/core/pointcloud/qgspointcloudindex.h @@ -344,11 +344,6 @@ class CORE_EXPORT QgsPointCloudIndex: public QObject */ int span() const; - /** - * Returns the number of points of indexed point cloud node \a n - */ - int nodePointCount( const IndexedPointCloudNode &n ); - /** * Sets the string used to define a subset of the point cloud. * \param subset The subset string to be used in a \a QgsPointCloudExpression diff --git a/src/core/pointcloud/qgsremoteeptpointcloudindex.cpp b/src/core/pointcloud/qgsremoteeptpointcloudindex.cpp deleted file mode 100644 index 0ac0630b2387..000000000000 --- a/src/core/pointcloud/qgsremoteeptpointcloudindex.cpp +++ /dev/null @@ -1,275 +0,0 @@ -/*************************************************************************** - qgsremoteeptpointcloudindex.cpp - -------------------- - begin : March 2021 - copyright : (C) 2021 by Belgacem Nedjima - email : belgacem dot nedjima at gmail dot com - ***************************************************************************/ - -/*************************************************************************** - * * - * This program is free software; you can redistribute it and/or modify * - * it under the terms of the GNU General Public License as published by * - * the Free Software Foundation; either version 2 of the License, or * - * (at your option) any later version. * - * * - ***************************************************************************/ - -#include "qgsremoteeptpointcloudindex.h" -#include "moc_qgsremoteeptpointcloudindex.cpp" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "qgsapplication.h" -#include "qgspointcloudrequest.h" -#include "qgspointcloudattribute.h" -#include "qgslogger.h" -#include "qgsfeedback.h" -#include "qgstiledownloadmanager.h" -#include "qgsblockingnetworkrequest.h" -#include "qgseptpointcloudblockrequest.h" -#include "qgscachedpointcloudblockrequest.h" -#include "qgspointcloudexpression.h" -#include "qgsnetworkaccessmanager.h" -#include "qgssetrequestinitiator_p.h" - -///@cond PRIVATE - -QgsRemoteEptPointCloudIndex::QgsRemoteEptPointCloudIndex() : QgsEptPointCloudIndex() -{ - mHierarchyNodes.insert( IndexedPointCloudNode( 0, 0, 0, 0 ) ); -} - -QgsRemoteEptPointCloudIndex::~QgsRemoteEptPointCloudIndex() = default; - -std::unique_ptr QgsRemoteEptPointCloudIndex::clone() const -{ - QgsRemoteEptPointCloudIndex *clone = new QgsRemoteEptPointCloudIndex; - QMutexLocker locker( &mHierarchyMutex ); - copyCommonProperties( clone ); - return std::unique_ptr( clone ); -} - -QList QgsRemoteEptPointCloudIndex::nodeChildren( const IndexedPointCloudNode &n ) const -{ - QList lst; - if ( !loadNodeHierarchy( n ) ) - return lst; - - const int d = n.d() + 1; - const int x = n.x() * 2; - const int y = n.y() * 2; - const int z = n.z() * 2; - - lst.reserve( 8 ); - for ( int i = 0; i < 8; ++i ) - { - int dx = i & 1, dy = !!( i & 2 ), dz = !!( i & 4 ); - const IndexedPointCloudNode n2( d, x + dx, y + dy, z + dz ); - if ( loadNodeHierarchy( n2 ) ) - lst.append( n2 ); - } - return lst; -} - -void QgsRemoteEptPointCloudIndex::load( const QString &uri ) -{ - mUri = uri; - - QStringList splitUrl = uri.split( '/' ); - - mUrlFileNamePart = splitUrl.back(); - splitUrl.pop_back(); - mUrlDirectoryPart = splitUrl.join( '/' ); - - QNetworkRequest nr = QNetworkRequest( QUrl( mUri ) ); - - QgsBlockingNetworkRequest req; - const QgsBlockingNetworkRequest::ErrorCode errCode = req.get( nr ); - if ( errCode != QgsBlockingNetworkRequest::NoError ) - { - QgsDebugError( QStringLiteral( "Request failed: " ) + uri ); - mIsValid = false; - mError = req.errorMessage(); - return; - } - - const QgsNetworkReplyContent reply = req.reply(); - mIsValid = loadSchema( reply.content() ); -} - -std::unique_ptr QgsRemoteEptPointCloudIndex::nodeData( const IndexedPointCloudNode &n, const QgsPointCloudRequest &request ) -{ - if ( QgsPointCloudBlock *cached = getNodeDataFromCache( n, request ) ) - { - return std::unique_ptr( cached ); - } - - std::unique_ptr blockRequest( asyncNodeData( n, request ) ); - if ( !blockRequest ) - return nullptr; - - QEventLoop loop; - connect( blockRequest.get(), &QgsPointCloudBlockRequest::finished, &loop, &QEventLoop::quit ); - loop.exec(); - - std::unique_ptr block = blockRequest->takeBlock(); - if ( !block ) - { - QgsDebugError( QStringLiteral( "Error downloading node %1 data, error : %2 " ).arg( n.toString(), blockRequest->errorStr() ) ); - } - - storeNodeDataToCache( block.get(), n, request ); - return block; -} - -QgsPointCloudBlockRequest *QgsRemoteEptPointCloudIndex::asyncNodeData( const IndexedPointCloudNode &n, const QgsPointCloudRequest &request ) -{ - if ( QgsPointCloudBlock *cached = getNodeDataFromCache( n, request ) ) - { - return new QgsCachedPointCloudBlockRequest( cached, n, mUri, attributes(), request.attributes(), - scale(), offset(), mFilterExpression, request.filterRect() ); - } - - if ( !loadNodeHierarchy( n ) ) - return nullptr; - - QString fileUrl; - if ( mDataType == QLatin1String( "binary" ) ) - { - fileUrl = QStringLiteral( "%1/ept-data/%2.bin" ).arg( mUrlDirectoryPart, n.toString() ); - } - else if ( mDataType == QLatin1String( "zstandard" ) ) - { - fileUrl = QStringLiteral( "%1/ept-data/%2.zst" ).arg( mUrlDirectoryPart, n.toString() ); - } - else if ( mDataType == QLatin1String( "laszip" ) ) - { - fileUrl = QStringLiteral( "%1/ept-data/%2.laz" ).arg( mUrlDirectoryPart, n.toString() ); - } - else - { - return nullptr; - } - - // we need to create a copy of the expression to pass to the decoder - // as the same QgsPointCloudExpression object might be concurrently - // used on another thread, for example in a 3d view - QgsPointCloudExpression filterExpression = mFilterExpression; - QgsPointCloudAttributeCollection requestAttributes = request.attributes(); - requestAttributes.extend( attributes(), filterExpression.referencedAttributes() ); - return new QgsEptPointCloudBlockRequest( n, fileUrl, mDataType, attributes(), requestAttributes, scale(), offset(), filterExpression, request.filterRect() ); -} - -bool QgsRemoteEptPointCloudIndex::hasNode( const IndexedPointCloudNode &n ) const -{ - return loadNodeHierarchy( n ); -} - -bool QgsRemoteEptPointCloudIndex::loadNodeHierarchy( const IndexedPointCloudNode &nodeId ) const -{ - mHierarchyMutex.lock(); - bool found = mHierarchy.contains( nodeId ); - mHierarchyMutex.unlock(); - if ( found ) - return true; - - QVector nodePathToRoot; - { - IndexedPointCloudNode currentNode = nodeId; - do - { - nodePathToRoot.push_back( currentNode ); - currentNode = currentNode.parentNode(); - } - while ( currentNode.d() >= 0 ); - } - - for ( int i = nodePathToRoot.size() - 1; i >= 0 && !mHierarchy.contains( nodeId ); --i ) - { - const IndexedPointCloudNode node = nodePathToRoot[i]; - //! The hierarchy of the node is found => No need to load its file - mHierarchyMutex.lock(); - const bool foundInHierarchy = mHierarchy.contains( node ); - const bool foundInHierarchyNodes = mHierarchyNodes.contains( node ); - mHierarchyMutex.unlock(); - if ( foundInHierarchy ) - continue; - - if ( !foundInHierarchyNodes ) - continue; - - const QString fileUrl = QStringLiteral( "%1/ept-hierarchy/%2.json" ).arg( mUrlDirectoryPart, node.toString() ); - QNetworkRequest nr( fileUrl ); - QgsSetRequestInitiatorClass( nr, QStringLiteral( "QgsRemoteEptPointCloudIndex" ) ); - nr.setAttribute( QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::PreferCache ); - nr.setAttribute( QNetworkRequest::CacheSaveControlAttribute, true ); - - std::unique_ptr reply( QgsApplication::tileDownloadManager()->get( nr ) ); - - QEventLoop loop; - connect( reply.get(), &QgsTileDownloadManagerReply::finished, &loop, &QEventLoop::quit ); - loop.exec(); - - if ( reply->error() != QNetworkReply::NoError ) - { - QgsDebugError( QStringLiteral( "Request failed: " ) + mUri ); - return false; - } - - const QByteArray dataJsonH = reply->data(); - QJsonParseError errH; - const QJsonDocument docH = QJsonDocument::fromJson( dataJsonH, &errH ); - if ( errH.error != QJsonParseError::NoError ) - { - QgsDebugMsgLevel( QStringLiteral( "QJsonParseError when reading hierarchy from file %1" ).arg( fileUrl ), 2 ); - return false; - } - - const QJsonObject rootHObj = docH.object(); - for ( auto it = rootHObj.constBegin(); it != rootHObj.constEnd(); ++it ) - { - const QString nodeIdStr = it.key(); - const int nodePointCount = it.value().toInt(); - const IndexedPointCloudNode nodeId = IndexedPointCloudNode::fromString( nodeIdStr ); - mHierarchyMutex.lock(); - if ( nodePointCount >= 0 ) - mHierarchy[nodeId] = nodePointCount; - else if ( nodePointCount == -1 ) - mHierarchyNodes.insert( nodeId ); - mHierarchyMutex.unlock(); - } - } - - mHierarchyMutex.lock(); - found = mHierarchy.contains( nodeId ); - mHierarchyMutex.unlock(); - - return found; -} - -bool QgsRemoteEptPointCloudIndex::isValid() const -{ - return mIsValid; -} - -void QgsRemoteEptPointCloudIndex::copyCommonProperties( QgsRemoteEptPointCloudIndex *destination ) const -{ - QgsEptPointCloudIndex::copyCommonProperties( destination ); - - // QgsRemoteEptPointCloudIndex specific fields - destination->mUrlDirectoryPart = mUrlDirectoryPart; - destination->mUrlFileNamePart = mUrlFileNamePart; - destination->mHierarchyNodes = mHierarchyNodes; -} - -///@endcond diff --git a/src/core/pointcloud/qgsremoteeptpointcloudindex.h b/src/core/pointcloud/qgsremoteeptpointcloudindex.h deleted file mode 100644 index 4a765ec84098..000000000000 --- a/src/core/pointcloud/qgsremoteeptpointcloudindex.h +++ /dev/null @@ -1,86 +0,0 @@ -/*************************************************************************** - qgsremoteeptpointcloudindex.h - -------------------- - begin : March 2021 - copyright : (C) 2021 by Belgacem Nedjima - email : belgacem dot nedjima at gmail dot com - ***************************************************************************/ - -/*************************************************************************** - * * - * This program is free software; you can redistribute it and/or modify * - * it under the terms of the GNU General Public License as published by * - * the Free Software Foundation; either version 2 of the License, or * - * (at your option) any later version. * - * * - ***************************************************************************/ - -#ifndef QGSREMOTEEPTPOINTCLOUDINDEX_H -#define QGSREMOTEEPTPOINTCLOUDINDEX_H - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "qgspointcloudindex.h" -#include "qgspointcloudattribute.h" -#include "qgsstatisticalsummary.h" -#include "qgis_sip.h" -#include "qgseptpointcloudindex.h" - -///@cond PRIVATE -#define SIP_NO_FILE - -class QgsCoordinateReferenceSystem; -class QgsTileDownloadManager; - -class CORE_EXPORT QgsRemoteEptPointCloudIndex: public QgsEptPointCloudIndex -{ - Q_OBJECT - public: - - explicit QgsRemoteEptPointCloudIndex(); - ~QgsRemoteEptPointCloudIndex(); - - std::unique_ptr clone() const override; - - QList nodeChildren( const IndexedPointCloudNode &n ) const override; - - void load( const QString &fileName ) override; - - std::unique_ptr nodeData( const IndexedPointCloudNode &n, const QgsPointCloudRequest &request ) override; - QgsPointCloudBlockRequest *asyncNodeData( const IndexedPointCloudNode &n, const QgsPointCloudRequest &request ) override; - - bool hasNode( const IndexedPointCloudNode &n ) const override; - - bool isValid() const override; - - QgsPointCloudIndex::AccessType accessType() const override { return QgsPointCloudIndex::Remote; } - - /** - * Copies common properties to the \a destination index - * \since QGIS 3.26 - */ - void copyCommonProperties( QgsRemoteEptPointCloudIndex *destination ) const; - - private: - bool loadNodeHierarchy( const IndexedPointCloudNode &nodeId ) const; - - QString mUrlDirectoryPart; - QString mUrlFileNamePart; - - QUrl mUrl; - - //! Contains the nodes that will have */ept-hierarchy/d-x-y-z.json file - mutable QSet mHierarchyNodes; -}; - -///@endcond - -#endif // QGSREMOTEEPTPOINTCLOUDINDEX_H diff --git a/src/core/providers/ept/qgseptprovider.cpp b/src/core/providers/ept/qgseptprovider.cpp index 3a3f4aaec9c2..37e25f885e06 100644 --- a/src/core/providers/ept/qgseptprovider.cpp +++ b/src/core/providers/ept/qgseptprovider.cpp @@ -19,7 +19,6 @@ #include "qgseptprovider.h" #include "moc_qgseptprovider.cpp" #include "qgseptpointcloudindex.h" -#include "qgsremoteeptpointcloudindex.h" #include "qgsruntimeprofiler.h" #include "qgsapplication.h" #include "qgsprovidersublayerdetails.h" @@ -40,10 +39,7 @@ QgsEptProvider::QgsEptProvider( Qgis::DataProviderReadFlags flags ) : QgsPointCloudDataProvider( uri, options, flags ) { - if ( uri.startsWith( QStringLiteral( "http" ), Qt::CaseSensitivity::CaseInsensitive ) ) - mIndex.reset( new QgsRemoteEptPointCloudIndex ); - else - mIndex.reset( new QgsEptPointCloudIndex ); + mIndex.reset( new QgsEptPointCloudIndex ); std::unique_ptr< QgsScopedRuntimeProfile > profile; if ( QgsApplication::profiler()->groupIsActive( QStringLiteral( "projectload" ) ) ) diff --git a/src/core/providers/ept/qgseptprovider.h b/src/core/providers/ept/qgseptprovider.h index bcb8bfd843ba..527666c7d914 100644 --- a/src/core/providers/ept/qgseptprovider.h +++ b/src/core/providers/ept/qgseptprovider.h @@ -27,7 +27,6 @@ #define SIP_NO_FILE class QgsEptPointCloudIndex; -class QgsRemoteEptPointCloudIndex; class QgsEptProvider: public QgsPointCloudDataProvider { diff --git a/src/core/providers/vpc/qgsvirtualpointcloudprovider.cpp b/src/core/providers/vpc/qgsvirtualpointcloudprovider.cpp index 3f76477399ac..212e18536f7d 100644 --- a/src/core/providers/vpc/qgsvirtualpointcloudprovider.cpp +++ b/src/core/providers/vpc/qgsvirtualpointcloudprovider.cpp @@ -22,7 +22,6 @@ #include "moc_qgsvirtualpointcloudprovider.cpp" #include "qgscopcpointcloudindex.h" #include "qgseptpointcloudindex.h" -#include "qgsremoteeptpointcloudindex.h" #include "qgspointcloudsubindex.h" #include "qgspointcloudclassifiedrenderer.h" #include "qgspointcloudextentrenderer.h" @@ -391,20 +390,10 @@ void QgsVirtualPointCloudProvider::loadSubIndex( int i ) if ( sl.index() ) return; - if ( sl.uri().startsWith( QStringLiteral( "http" ), Qt::CaseSensitivity::CaseInsensitive ) ) - { - if ( sl.uri().endsWith( QStringLiteral( "copc.laz" ), Qt::CaseSensitivity::CaseInsensitive ) ) - sl.setIndex( new QgsCopcPointCloudIndex() ); - else if ( sl.uri().endsWith( QStringLiteral( "ept.json" ), Qt::CaseSensitivity::CaseInsensitive ) ) - sl.setIndex( new QgsRemoteEptPointCloudIndex() ); - } - else - { - if ( sl.uri().endsWith( QStringLiteral( "copc.laz" ), Qt::CaseSensitivity::CaseInsensitive ) ) - sl.setIndex( new QgsCopcPointCloudIndex() ); - else if ( sl.uri().endsWith( QStringLiteral( "ept.json" ), Qt::CaseSensitivity::CaseInsensitive ) ) - sl.setIndex( new QgsEptPointCloudIndex() ); - } + if ( sl.uri().endsWith( QStringLiteral( "copc.laz" ), Qt::CaseSensitivity::CaseInsensitive ) ) + sl.setIndex( new QgsCopcPointCloudIndex() ); + else if ( sl.uri().endsWith( QStringLiteral( "ept.json" ), Qt::CaseSensitivity::CaseInsensitive ) ) + sl.setIndex( new QgsEptPointCloudIndex() ); if ( !sl.index() ) return;