diff --git a/src/axom/core/ArrayBase.hpp b/src/axom/core/ArrayBase.hpp index a24d910af6..4bab2d8fb7 100644 --- a/src/axom/core/ArrayBase.hpp +++ b/src/axom/core/ArrayBase.hpp @@ -387,15 +387,26 @@ class ArrayBase updateStrides(); } - /// \brief Set the shape and stride - AXOM_HOST_DEVICE void setShapeAndStride(const StackArray& shape, - const StackArray& stride) + /*! + \brief Set the shape and stride + + \param [in] shape + \param [in] stride + \param [in] orderPref Preference for resolving non-unique strides. + \a ROW means to advance left index first. \a COLUMN means to advance + right index first. Indexing is correct regardless of the + preference, but the index ordering is dependent on the choice. + */ + AXOM_HOST_DEVICE void setShapeAndStride( + const StackArray& shape, + const StackArray& stride, + axom::ArrayStrideOrder orderPref = axom::ArrayStrideOrder::ROW) { #ifdef AXOM_DEBUG validateShapeAndStride(shape, stride); #endif m_shape = shape; - m_mapping.initializeStrides(stride); + m_mapping.initializeStrides(stride, orderPref); m_minStride = m_mapping.fastestStrideLength(); } diff --git a/src/axom/core/MDMapping.hpp b/src/axom/core/MDMapping.hpp index 74995c8b83..19b303189d 100644 --- a/src/axom/core/MDMapping.hpp +++ b/src/axom/core/MDMapping.hpp @@ -232,10 +232,13 @@ class MDMapping os << strides[d] << ","; } os << strides[DIM - 1] << ")"; - std::cerr << "ERROR: MDMapping: Non-unique strides " << os.str() << ".\n" - << "Likely, multi-dim array shape is 1 in some direction.\n" - << "Impossible to compute index ordering.\n" - << "Please use a different MDMapping initializer.\n"; + std::cerr + << "ERROR: MDMapping: Non-unique strides " << os.str() << ".\n" + << "Caused by multi-dim array shape of 1 in some direction.\n" + << "It is impossible to compute index ordering.\n" + << "Use initializeStrides() with the stride order preference to fix.\n" + << "The resulting slowestDirs() depends on the preference\n" + << "but the array mapping will still be correct.\n"; #endif utilities::processAbort(); } diff --git a/src/axom/quest/DistributedClosestPoint.cpp b/src/axom/quest/DistributedClosestPoint.cpp index 9bf6063a19..63553d8c2b 100644 --- a/src/axom/quest/DistributedClosestPoint.cpp +++ b/src/axom/quest/DistributedClosestPoint.cpp @@ -320,5 +320,15 @@ void DistributedClosestPoint::verifyTopologyName(const conduit::Node& meshNode, } } +axom::IndexType DistributedClosestPoint::searchCount() const +{ + return m_impl->searchCount(); +} + +double DistributedClosestPoint::effectiveDistanceThreshold() const +{ + return m_impl->effectiveDistanceThreshold(); +} + } // end namespace quest } // end namespace axom diff --git a/src/axom/quest/DistributedClosestPoint.hpp b/src/axom/quest/DistributedClosestPoint.hpp index dca36f580b..ca91f1b000 100644 --- a/src/axom/quest/DistributedClosestPoint.hpp +++ b/src/axom/quest/DistributedClosestPoint.hpp @@ -8,6 +8,7 @@ #include "axom/config.hpp" #include "axom/core/execution/runtime_policy.hpp" +#include "axom/core/Types.hpp" #include "conduit_node.hpp" @@ -173,6 +174,23 @@ class DistributedClosestPoint void computeClosestPoints(conduit::Node& query_node, const std::string& topology); + /*! + @brief Return the number of searches done on the last query + mesh's local partition. + + This count includes 1 by the owner rank plus however many remote + ranks searched the partition. + */ + axom::IndexType searchCount() const; + /*! + @brief Return the effective distance threshold used for the + last query mesh's local partition. + + Due to optimizations, this may be smaller than the value set + in setDistanceThreshold(). + */ + double effectiveDistanceThreshold() const; + private: /*! @brief Allocate the DistributedClosestPointImpl object, which actually does the work. diff --git a/src/axom/quest/MeshViewUtil.hpp b/src/axom/quest/MeshViewUtil.hpp index 7c3098c6e5..75be596aad 100644 --- a/src/axom/quest/MeshViewUtil.hpp +++ b/src/axom/quest/MeshViewUtil.hpp @@ -124,7 +124,9 @@ static void shapesToStridesAndOffsets( @param offsets [i] Blueprint-style index offsets. @param strides [i] Blueprint-style strides. @param valuesCount [i] Number of values in - ghost-padded data. + ghost-padded data. If this is too small (results in + a negative \a hiPads for the slowest stride + direction), that padding will be bumped to zero. @param paddedShape [o] \a realShape + \a loPads + \a hiPads @param loPads [o] Ghost padding amount on low side. @param hiPads [o] Ghost padding amount ont high side. @@ -165,7 +167,9 @@ static void stridesAndOffsetsToShapes(const axom::StackArray& realSh const int& nextDir = strideOrder[nd + 1]; paddedShape[curDir] = strides[nextDir] / strides[curDir]; } - paddedShape[strideOrder[DIM - 1]] = valuesCount / strides[strideOrder[DIM - 1]]; + const int slowestDir = strideOrder[DIM - 1]; + paddedShape[slowestDir] = std::max(valuesCount / strides[slowestDir], + realShape[slowestDir] + offsets[slowestDir]); for(int d = 0; d < DIM; ++d) { @@ -500,7 +504,7 @@ class MeshViewUtil /*! @brief Return view to a scalar field variable. - WARNING: The view returned has an allocator id determined by + WARNING: The view returned claims an allocator id determined by \a MemSpace, regardless of the memory type. WARNING: Assuming, without checking, that the field contains @@ -647,8 +651,9 @@ class MeshViewUtil @param [in] fieldName @param [in] association "vertex" or "element" - @param [in] dtype Conduit data type to put in the field. Must be at least - big enough for the strides and offsets specified. + @param [in] dtype Conduit data type to put in the field. If this is not + big enough for the strides and offsets specified, a minimally + sufficient size will be used to avoid negative ghost padding. @param [in] strides Data strides. Set to zero for no ghosts and default strides. @param [in] offsets Data index offsets. Set to zero for no ghosts. diff --git a/src/axom/quest/detail/DistributedClosestPointImpl.hpp b/src/axom/quest/detail/DistributedClosestPointImpl.hpp index 13d157db6c..401bfcc1f3 100644 --- a/src/axom/quest/detail/DistributedClosestPointImpl.hpp +++ b/src/axom/quest/detail/DistributedClosestPointImpl.hpp @@ -254,7 +254,7 @@ class DistributedClosestPointImpl , m_mpiComm(MPI_COMM_NULL) , m_rank(-1) , m_nranks(-1) - , m_sqDistanceThreshold(std::numeric_limits::max()) + , m_sqUserDistanceThreshold(std::numeric_limits::max()) { } virtual ~DistributedClosestPointImpl() { } @@ -303,7 +303,7 @@ class DistributedClosestPointImpl { SLIC_ERROR_IF(sqThreshold < 0.0, "Squared distance-threshold must be non-negative."); - m_sqDistanceThreshold = sqThreshold; + m_sqUserDistanceThreshold = sqThreshold; } /*! @@ -542,6 +542,23 @@ class DistributedClosestPointImpl virtual void computeClosestPoints(conduit::Node& queryMesh, const std::string& topologyName) const = 0; + /*! + @brief Return the number of searches done on the last query + mesh's local partition. + */ + axom::IndexType searchCount() const + { + return m_searchCount; + } + /*! + @brief Return the effective distance threshold of the last query + partition. + */ + double effectiveDistanceThreshold() const + { + return m_effectiveDistanceThreshold; + } + protected: int m_allocatorID; bool m_isVerbose; @@ -550,7 +567,8 @@ class DistributedClosestPointImpl int m_rank; int m_nranks; - double m_sqDistanceThreshold; + //!@brief Distance threshold specified by user. + double m_sqUserDistanceThreshold; bool m_outputRank = true; bool m_outputIndex = true; @@ -569,6 +587,14 @@ class DistributedClosestPointImpl /// MPI rank of closest element int rank {-1}; }; + + //@{ + //!@name Diagnostic data + //!@brief Number of searches conducted on the last query node. + mutable axom::IndexType m_searchCount = -1; + //!@brief Effective distance threshold of the last query partition. + mutable double m_effectiveDistanceThreshold = 0.0; + //@} }; /*! @@ -639,7 +665,7 @@ class DistributedClosestPointExec : public DistributedClosestPointImpl } // Copy points to internal memory - PointArray coords(ptCount, ptCount); + m_objectPtCoords = PointArray(ptCount, ptCount, m_allocatorID); axom::Array domIds(ptCount, ptCount); std::size_t copiedCount = 0; conduit::Node tmpValues; @@ -673,7 +699,7 @@ class DistributedClosestPointExec : public DistributedClosestPointImpl const int N = internal::extractSize(copySrc); const std::size_t nBytes = sizeof(double) * DIM * N; - axom::copy(coords.data() + copiedCount, + axom::copy(m_objectPtCoords.data() + copiedCount, copySrc.fetch_existing("x").data_ptr(), nBytes); tmpValues.reset(); @@ -682,16 +708,41 @@ class DistributedClosestPointExec : public DistributedClosestPointImpl copiedCount += N; } + + // Compute bounding box. +#if defined(AXOM_USE_RAJA) + // Coordinates may be on device but should be compatible with ExecSpace. + axom::ArrayView coordsView = m_objectPtCoords.view(); + PointType minPt, maxPt; + for(int d = 0; d < DIM; ++d) + { + RAJA::ReduceMin minCoord( + std::numeric_limits::max()); + RAJA::ReduceMax maxCoord( + -std::numeric_limits::max()); + RAJA::forall( + RAJA::RangeSegment(0, ptCount), + AXOM_LAMBDA(RAJA::Index_type n) { + minCoord.min(coordsView[n][d]); + maxCoord.max(coordsView[n][d]); + }); + minPt[d] = minCoord.get(); + maxPt[d] = maxCoord.get(); + } + m_objectBb = BoxType(minPt, maxPt); +#else + m_objectBb = + axom::primal::BoundingBox {m_objectPtCoords.data(), + m_objectPtCoords.size()}; +#endif + gatherBoundingBoxes(m_objectBb, m_objectPartitionBbs); + // copy computed data to ExecSpace - m_objectPtCoords = PointArray(coords, m_allocatorID); m_objectPtDomainIds = axom::Array(domIds, m_allocatorID); } bool generateBVHTree() override { - // Delegates to generateBVHTreeImpl<> which uses - // the execution space templated bvh tree - SLIC_ASSERT_MSG(!m_bvh, "BVH tree already initialized"); // In case user changed the allocator after setObjectMesh, @@ -703,18 +754,20 @@ class DistributedClosestPointExec : public DistributedClosestPointImpl } m_bvh = std::make_unique(); - return generateBVHTreeImpl(m_bvh.get()); - } + const int npts = m_objectPtCoords.size(); + axom::Array boxesArray(npts, npts, m_allocatorID); + auto boxesView = boxesArray.view(); + auto pointsView = m_objectPtCoords.view(); - /// Get local copy of all ranks BVH root bounding boxes. - void gatherBVHRoots() - { - SLIC_ASSERT_MSG( - m_bvh, - "BVH tree must be initialized before calling 'gatherBVHRoots"); + axom::for_all( + npts, + AXOM_LAMBDA(axom::IndexType i) { boxesView[i] = BoxType {pointsView[i]}; }); + + // Build bounding volume hierarchy + m_bvh->setAllocatorID(m_allocatorID); + int result = m_bvh->initialize(boxesView, npts); - BoxType local_bb = m_bvh->getBounds(); - gatherBoundingBoxes(local_bb, m_objectPartitionBbs); + return (result == spin::BVH_BUILD_OK); } /// Allgather one bounding box from each rank. @@ -724,7 +777,6 @@ class DistributedClosestPointExec : public DistributedClosestPointImpl aabb.getMin().to_array(&sendbuf[0]); aabb.getMax().to_array(&sendbuf[DIM]); axom::Array recvbuf(m_nranks * sendbuf.size()); - // Note: Using axom::Array may reduce clutter a tad. int errf = MPI_Allgather(sendbuf.data(), 2 * DIM, mpi_traits::type, @@ -745,6 +797,22 @@ class DistributedClosestPointExec : public DistributedClosestPointImpl } } + /// Allgather a primitive value. + template + void gatherPrimitiveValue(const T& val, axom::Array& allVals) const + { + allVals.resize(m_nranks); + int errf = MPI_Allgather(&val, + 1, + mpi_traits::type, + allVals.data(), + 1, + mpi_traits::type, + m_mpiComm); + SLIC_ASSERT(errf == MPI_SUCCESS); + AXOM_UNUSED_VAR(errf); + } + /// Compute bounding box for local part of a mesh. BoxType computeMeshBoundingBox(conduit::Node& xferNode) const { @@ -776,6 +844,19 @@ class DistributedClosestPointExec : public DistributedClosestPointImpl * The worst case could incur nranks^2 sends. To avoid excessive * buffer usage, we occasionally check the sends for completion, * using check_send_requests(). + * + * To exclude fruitless searches and communication, we check the + * distance between object partition and query partition, using + * their bounding boxes. If a query partition is too far from an + * object partition, we don't check that specific pair. + * + * TODO: The bounding box for a partition could be excessively big + * if the domains on that partition are spread out far, leading to + * some fruitless communications and checks. Consider having one + * bounding box per domain instead of one per partition. This goes + * for query mesh too. One way simple to implement this is to have + * a xferNode for each query domain and a BVH for each object + * domain. */ void computeClosestPoints(conduit::Node& queryMesh, const std::string& topologyName) const override @@ -800,23 +881,58 @@ class DistributedClosestPointExec : public DistributedClosestPointImpl BoxArray allQueryBbs; gatherBoundingBoxes(myQueryBb, allQueryBbs); + /* + Note: The two m_nranks loops below can be moved to device for the cost of + copying the bounding box arrays to device. Not sure if it's worthwhile, + because in general, we assume that m_nranks is small relative to the + number of points in the partitions. + */ + + // Compute the min of the max distance between myQueryBb and each rank's object bounding box. + double minMaxSqDist = std::numeric_limits::max(); + for(int i = 0; i < m_nranks; ++i) + { + auto maxSqDist = maxSqDistBetweenBoxes(myQueryBb, m_objectPartitionBbs[i]); + minMaxSqDist = std::min(minMaxSqDist, maxSqDist); + } + + const double sqDistanceThreshold = + std::min(m_sqUserDistanceThreshold, minMaxSqDist); + xferNodes[m_rank]->fetch("sqDistanceThreshold") = sqDistanceThreshold; + m_effectiveDistanceThreshold = sqrt(sqDistanceThreshold); + + axom::Array allSqDistanceThreshold(m_nranks); + gatherPrimitiveValue(sqDistanceThreshold, allSqDistanceThreshold); + { conduit::Node& xferNode = *xferNodes[m_rank]; computeLocalClosestPoints(xferNode); + xferNode["searchCount"].set_int32(1); + m_searchCount = 1; } + /* + Count number of remote query partitions to receive, + i.e., how many partitions close enough to myObjectBb. + */ const auto& myObjectBb = m_objectPartitionBbs[m_rank]; int remainingRecvs = 0; - for(int r = 0; r < m_nranks; ++r) + if(myObjectBb.isValid()) { - if(r != m_rank) + for(int r = 0; r < m_nranks; ++r) { - const auto& otherQueryBb = allQueryBbs[r]; - double sqDistance = - axom::primal::squared_distance(otherQueryBb, myObjectBb); - if(sqDistance <= m_sqDistanceThreshold) + if(r != m_rank) { - ++remainingRecvs; + const auto& otherQueryBb = allQueryBbs[r]; + if(otherQueryBb.isValid()) + { + double sqDistance = + axom::primal::squared_distance(otherQueryBb, myObjectBb); + if(sqDistance <= allSqDistanceThreshold[r]) + { + ++remainingRecvs; + } + } } } } @@ -879,10 +995,14 @@ class DistributedClosestPointExec : public DistributedClosestPointImpl if(homeRank == m_rank) { node_copy_xfer_to_query(xferNode, queryMesh, topologyName); + m_searchCount = xferNode["searchCount"].value(); + xferNodes.erase(m_rank); } else { computeLocalClosestPoints(xferNode); + auto tmpCount = xferNode["searchCount"].as_int32(); + xferNode["searchCount"].set_int32(1 + tmpCount); isendRequests.emplace_back(conduit::relay::mpi::Request()); auto& isendRequest = isendRequests.back(); @@ -921,6 +1041,8 @@ class DistributedClosestPointExec : public DistributedClosestPointImpl int homeRank = xferNode.fetch_existing("homeRank").value(); BoxType bb; get_bounding_box_from_conduit_node(bb, xferNode.fetch_existing("aabb")); + auto sqDistanceThreshold = + xferNode.fetch_existing("sqDistanceThreshold").as_double(); for(int i = 1; i < m_nranks; ++i) { int maybeNextRecip = (m_rank + i) % m_nranks; @@ -930,7 +1052,7 @@ class DistributedClosestPointExec : public DistributedClosestPointImpl } double sqDistance = primal::squared_distance(bb, m_objectPartitionBbs[maybeNextRecip]); - if(sqDistance <= m_sqDistanceThreshold) + if(sqDistance <= sqDistanceThreshold) { return maybeNextRecip; } @@ -940,29 +1062,6 @@ class DistributedClosestPointExec : public DistributedClosestPointImpl // Note: following should be private, but nvcc complains about lambdas in private scope public: - /// Templated implementation of generateBVHTree function - bool generateBVHTreeImpl(BVHTreeType* bvh) - { - SLIC_ASSERT(bvh != nullptr); - - const int npts = m_objectPtCoords.size(); - axom::Array boxesArray(npts, npts, m_allocatorID); - auto boxesView = boxesArray.view(); - auto pointsView = m_objectPtCoords.view(); - - axom::for_all( - npts, - AXOM_LAMBDA(axom::IndexType i) { boxesView[i] = BoxType {pointsView[i]}; }); - - // Build bounding volume hierarchy - bvh->setAllocatorID(m_allocatorID); - int result = bvh->initialize(boxesView, npts); - - gatherBVHRoots(); - - return (result == spin::BVH_BUILD_OK); - } - void computeLocalClosestPoints(conduit::Node& xferNode) const { using axom::primal::squared_distance; @@ -1063,7 +1162,7 @@ class DistributedClosestPointExec : public DistributedClosestPointImpl const int rank = m_rank; double* sqDistThresh = axom::allocate(1, m_allocatorID); - *sqDistThresh = m_sqDistanceThreshold; + *sqDistThresh = xferNode.fetch("sqDistanceThreshold").as_double(); auto ptCoordsView = m_objectPtCoords.view(); auto ptDomainIdsView = m_objectPtDomainIds.view(); @@ -1170,12 +1269,64 @@ class DistributedClosestPointExec : public DistributedClosestPointImpl axom::Array m_objectPtDomainIds; + //!@brief Bounding box for m_objectPtCoords. + BoxType m_objectBb; + /*! @brief Object partition bounding boxes, one per rank. All are in physical space, not index space. */ BoxArray m_objectPartitionBbs; std::unique_ptr m_bvh; + + /*! + @brief Compute maximum squared-distance possible between points in 2 boxes, + or std::numeric_limits::max() if either box is invalid. + */ + AXOM_HOST_DEVICE double maxSqDistBetweenBoxes(const BoxType& a, + const BoxType& b) const + { + if(!a.isValid() || !b.isValid()) + { + return std::numeric_limits::max(); + } + + double maxSqDist = 0.0; + /* + The following logic is necessary should one box nest inside the + other when projected onto one or more axis directions. + + We look at the distance between each corner of box a and the + opposite corner of b. The max distance is the max among those. + Opposite means that if we choose the lower corner in a, we must + compare with the upper corner in b. And vice versa. + */ + int numCorners = 1 << DIM; + for(int i = 0; i < numCorners; ++i) + { + PointType aCoords; // i-th corner of a. + PointType bCoords; // Corner of b opposite from i-th corner of a. + for(int d = 0; d < DIM; ++d) + { + bool upperA_lowerB = i & (1 << d); + if(upperA_lowerB) + { + aCoords[d] = a.getMin()[d]; + bCoords[d] = b.getMax()[d]; + } + else // upperB_lowerA + { + bCoords[d] = b.getMin()[d]; + aCoords[d] = a.getMax()[d]; + } + } + primal::Vector separation {aCoords, bCoords}; + double sqDist = separation.squared_norm(); + maxSqDist = std::max(maxSqDist, sqDist); + } + return maxSqDist; + } + }; // DistributedClosestPointExec } // namespace internal diff --git a/src/axom/quest/examples/quest_distributed_distance_query_example.cpp b/src/axom/quest/examples/quest_distributed_distance_query_example.cpp index 15e798e6b3..9805bd52e0 100644 --- a/src/axom/quest/examples/quest_distributed_distance_query_example.cpp +++ b/src/axom/quest/examples/quest_distributed_distance_query_example.cpp @@ -339,6 +339,9 @@ struct BlueprintParticleMesh MPI_Allreduce(MPI_IN_PLACE, &m_dimension, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD); SLIC_ASSERT(m_dimension > 0); + // For debugging, create cell-centered owner-rank field. + m_dimension == 2 ? setCellOwnerRank<2>(mdMesh) : setCellOwnerRank<3>(mdMesh); + if(domCount > 0) { // Put mdMesh into sidre Group. @@ -429,6 +432,43 @@ struct BlueprintParticleMesh } } + /*! + @brief Set the cell-centered owner rank field. + + @param mdMesh A multi-domain structured mesh. + */ + template + void setCellOwnerRank(conduit::Node& mdMesh) + { + std::string fieldName = "owner_rank"; + axom::StackArray zeroPads; + axom::StackArray strideOrder; + for(int d = 0; d < DIM; ++d) + { + zeroPads[d] = 0; + strideOrder[d] = DIM - d - 1; + } + for(conduit::Node& dom : mdMesh.children()) + { + axom::quest::MeshViewUtil domainView(dom, m_topologyName); + domainView.createField(fieldName, + "element", + conduit::DataType::uint32(), + zeroPads, + zeroPads, + strideOrder); + + auto ownerRankView = + domainView.template getFieldView(fieldName); + axom::detail::ArrayOps::fill( + ownerRankView.data(), + 0, + ownerRankView.size(), + ownerRankView.getAllocatorID(), + m_rank); + } + } + template axom::Array> getPoints(int domainIdx) { @@ -943,11 +983,6 @@ class QueryMeshWrapper SLIC_ASSERT(queryPts.size() == cpCoords.size()); SLIC_ASSERT(queryPts.size() == cpIndices.size()); - if(params.isVerbose()) - { - SLIC_INFO(axom::fmt::format("Closest points ({}):", cpCoords.size())); - } - /* Allowable slack is half the arclength between 2 adjacent object points. A query point on the object can correctly have that @@ -959,20 +994,51 @@ class QueryMeshWrapper const double longSpacing = 2 * M_PI * params.circleRadius / params.longPointCount; const double latSpacing = params.circleRadius * M_PI / 180 * - (params.latRange[1] - params.latRange[0]) / params.latPointCount; + (params.latRange[1] - params.latRange[0]) / (params.latPointCount - 1); const double avgObjectRes = DIM == 2 ? longSpacing : std::sqrt(longSpacing * longSpacing + latSpacing * latSpacing); const double allowableSlack = avgObjectRes / 2; using IndexSet = slam::PositionSet<>; + const PointType circleCenter {params.circleCenter.data(), DIM}; + double zNorth = + params.circleRadius * std::sin(params.latRange[1] * M_PI / 180); + double xyNorth = + params.circleRadius * std::cos(params.latRange[1] * M_PI / 180); + double zSouth = + params.circleRadius * std::sin(params.latRange[0] * M_PI / 180); + double xySouth = + params.circleRadius * std::cos(params.latRange[0] * M_PI / 180); for(auto i : IndexSet(queryPts.size())) { bool errf = false; + // Compute the analytical distance to sphere (or partial sphere + // if the latitude range doesn't go all the way to the poles). const auto& qPt = queryPts[i]; const auto& cpCoord = cpCoords[i]; double analyticalDist = std::fabs(sphere.computeSignedDistance(qPt)); + if(DIM == 3 && params.latRange[0] > -90 && params.latRange[1] < 90) + { + // More complicated analytical distance for partial-sphere object. + axom::primal::Vector cToQ {circleCenter, qPt}; + double z = cToQ[2]; + cToQ[2] = 0.0; + double xy = cToQ.norm(); + double qPtLat = std::atan(z / xy) * 180 / M_PI; + if(qPtLat > params.latRange[1]) + { + analyticalDist = std::sqrt((z - zNorth) * (z - zNorth) + + (xy - xyNorth) * (xy - xyNorth)); + } + else if(qPtLat < params.latRange[0]) + { + analyticalDist = std::sqrt((z - zSouth) * (z - zSouth) + + (xy - xySouth) * (xy - xySouth)); + } + } + const bool closestPointFound = (cpIndices[i] == -1); if(closestPointFound) { @@ -1032,7 +1098,7 @@ class QueryMeshWrapper { errf = true; SLIC_INFO( - axom::fmt::format("***Warning: Closest distance for {} (index " + axom::fmt::format("***Error: Closest distance for {} (index " "{}, cp {}) is {}, off by {}.", qPt, i, @@ -1061,6 +1127,7 @@ class QueryMeshWrapper /** * Generates points on a sphere, partitioned into multiple domains. + * The sphere's polar axis is in the z-direction. * Point spacing in the longitudinal direction can be random (default) or uniform. * 3D points cover the given latitude range. */ @@ -1478,6 +1545,12 @@ int main(int argc, char** argv) MPI_Allreduce(&inVal, &maxVal, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); MPI_Allreduce(&inVal, &sumVal, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); }; + auto getIntMinMax = + [](int inVal, int& minVal, int& maxVal, int& sumVal) { + MPI_Allreduce(&inVal, &minVal, 1, MPI_INT, MPI_MIN, MPI_COMM_WORLD); + MPI_Allreduce(&inVal, &maxVal, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD); + MPI_Allreduce(&inVal, &sumVal, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); + }; // Output some timing stats { @@ -1487,26 +1560,46 @@ int main(int argc, char** argv) double minQuery, maxQuery, sumQuery; getDoubleMinMax(queryTimer.elapsedTimeInSec(), minQuery, maxQuery, sumQuery); + double minFilterDist, maxFilterDist, sumFilterDist; + getDoubleMinMax(query.effectiveDistanceThreshold(), minFilterDist, maxFilterDist, sumFilterDist); + + int minSearchCount, maxSearchCount, sumSearchCount; + getIntMinMax(int(query.searchCount()), minSearchCount, maxSearchCount, sumSearchCount); + SLIC_INFO(axom::fmt::format( - "Initialization with policy {} took {{avg:{}, min:{}, max:{}}} seconds", + "Initialization with policy {} took {{avg:{:.5f}, min:{:.5f}, max:{:.5f}}} seconds", axom::runtime_policy::s_policyToName.at(params.policy), sumInit / num_ranks, minInit, maxInit)); SLIC_INFO(axom::fmt::format( - "Query with policy {} took {{avg:{}, min:{}, max:{}}} seconds", + "Query with policy {} took {{avg:{:.5f}, min:{:.5f}, max:{:.5f}}} seconds", axom::runtime_policy::s_policyToName.at(params.policy), sumQuery / num_ranks, minQuery, maxQuery)); + SLIC_INFO(axom::fmt::format( + "Search counts {{avg:{}, min:{}, max:{}}}", + sumSearchCount / num_ranks, + minSearchCount, + maxSearchCount)); + SLIC_INFO(axom::fmt::format( + "Effective distance threshold {{avg:{:.5f}, min:{:.5f}, max:{:.5f}}}", + sumFilterDist / num_ranks, + minFilterDist, + maxFilterDist)); } slic::flushStreams(); + SLIC_INFO(axom::fmt::format("Updating closest points.")); + slic::flushStreams(); queryMeshWrapper.update_closest_points(queryMeshNode); int errCount = 0; int localErrCount = 0; if(params.checkResults) { + SLIC_INFO(axom::fmt::format("Checking results.")); + slic::flushStreams(); if(spatialDim == 2) { primal::Point center(params.circleCenter.data()); @@ -1544,6 +1637,7 @@ int main(int argc, char** argv) queryMeshWrapper.saveMesh(params.distanceFile); + axom::slic::flushStreams(); if(errCount) { SLIC_INFO(axom::fmt::format(" Error exit: {} errors found.", errCount));