From 077ef7335c060ef441cb3a2b12e915a814d38063 Mon Sep 17 00:00:00 2001 From: Colby Jeffries Date: Tue, 18 Feb 2020 11:08:08 -0600 Subject: [PATCH 1/4] Update version numbers. --- .../portableglobe/cutter/cgi-bin/geecheck_tests/common.py | 2 +- earth_enterprise/src/fusion_version.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/earth_enterprise/src/fusion/portableglobe/cutter/cgi-bin/geecheck_tests/common.py b/earth_enterprise/src/fusion/portableglobe/cutter/cgi-bin/geecheck_tests/common.py index 54c7ceb565..029536b02a 100755 --- a/earth_enterprise/src/fusion/portableglobe/cutter/cgi-bin/geecheck_tests/common.py +++ b/earth_enterprise/src/fusion/portableglobe/cutter/cgi-bin/geecheck_tests/common.py @@ -20,7 +20,7 @@ import platform import socket -LATEST_VERSION = '5.3.3' +LATEST_VERSION = '5.3.3.1' SUPPORTED_OS_LIST = { 'redhat': {'min_release': '6.0', diff --git a/earth_enterprise/src/fusion_version.txt b/earth_enterprise/src/fusion_version.txt index c6ed489f04..8a544b8717 100644 --- a/earth_enterprise/src/fusion_version.txt +++ b/earth_enterprise/src/fusion_version.txt @@ -19,4 +19,4 @@ # earth_enterprise/installer/ # Format is 1 line (comment lines ignored): # full version number: e.g., 3.1.2.3, 5.2.4, etc -5.3.3 +5.3.3.1 From 050bede0728a8534b7865b3357d133690a6854f1 Mon Sep 17 00:00:00 2001 From: tst-cjeffries <44170038+tst-cjeffries@users.noreply.github.com> Date: Tue, 18 Feb 2020 10:24:45 -0600 Subject: [PATCH 2/4] #1650: WMS now uses forwarded protocol (#1651) * Add Joseph's quick fix. * Actually apply protocol to url. --- earth_enterprise/src/server/wsgi/wms/ogc/common/utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/earth_enterprise/src/server/wsgi/wms/ogc/common/utils.py b/earth_enterprise/src/server/wsgi/wms/ogc/common/utils.py index 80fd0fefd2..1c232d55c6 100644 --- a/earth_enterprise/src/server/wsgi/wms/ogc/common/utils.py +++ b/earth_enterprise/src/server/wsgi/wms/ogc/common/utils.py @@ -143,7 +143,11 @@ def GetServerURL(environ): complete_url += urllib.quote(environ.get("PATH_INFO", "")) if environ.get("HTTP_X_FORWARDED_HOST"): - proxy_url = environ["wsgi.url_scheme"] + "://" + environ["HTTP_X_FORWARDED_HOST"] + proxy_scheme = environ["wsgi.url_scheme"] + if environ.get("HTTP_X_FORWARDED_PROTO"): + proxy_scheme = environ["HTTP_X_FORWARDED_PROTO"] + + proxy_url = proxy_scheme + "://" + environ["HTTP_X_FORWARDED_HOST"] proxy_url += urllib.quote(environ.get("REDIRECT_URL", "")) proxy_url += urllib.quote(environ.get("PATH_INFO", "")) else: From be9b6b8060b1a9a1ecbdcabcb5affb9e2ea3c92d Mon Sep 17 00:00:00 2001 From: Lee Savoie Date: Mon, 24 Feb 2020 10:14:40 -0700 Subject: [PATCH 3/4] Issue #254: Handle images with nodata values that don't fit in the pixel type (#1647) * Ignore NoData when it's too large for the pixel type * Move assert to start of function * Print a warning when ignoring the nodata value * Delete unused copy functions * More useful ingest warnings - Notify when using non-zero NoData - Print NoData warnings once * Move NoData calculation to a separate function * Implement unit tests and fix bug The bug was that numeric_limits::min() returns a positive value for floats. * Explanatory comment in unit tests * Avoid unnecessarily recalculating NoData * Remove temporary file included by accident --- earth_enterprise/src/fusion/khgdal/SConscript | 4 +- .../src/fusion/khgdal/khGDALReader.cpp | 5 + .../src/fusion/khgdal/khGDALReader.h | 8 ++ .../src/fusion/khgdal/khGDALReaderImpl.h | 46 ++++++++-- .../fusion/khgdal/khGDALReader_unittest.cpp | 91 +++++++++++++++++++ earth_enterprise/src/fusion/khgdal/khgdal.h | 6 +- 6 files changed, 148 insertions(+), 12 deletions(-) create mode 100644 earth_enterprise/src/fusion/khgdal/khGDALReader_unittest.cpp diff --git a/earth_enterprise/src/fusion/khgdal/SConscript b/earth_enterprise/src/fusion/khgdal/SConscript index 469e88efb5..4aa7633919 100644 --- a/earth_enterprise/src/fusion/khgdal/SConscript +++ b/earth_enterprise/src/fusion/khgdal/SConscript @@ -54,9 +54,11 @@ getranslate = env.executable('getranslate', 'getranslate.cpp', gereproject = env.executable('gereproject', 'gereproject.cpp', LIBS=['gegdal', 'geutil', 'gdal', 'geos']) - khgdal_test = env.test('khgdal_test', ['khgdal_test.cpp'], LIBS=['gegdal', 'gecommon', 'geos']) +khGDALReader_unittest = env.test('khGDALReader_unittest', + ['khGDALReader_unittest.cpp'], + LIBS=['gegdal', 'gecommon', 'geos', 'gtest']) env.install('fusion_bin', [geinfo, gevirtualraster, gesplitkhvr, getranslate, gereproject]) diff --git a/earth_enterprise/src/fusion/khgdal/khGDALReader.cpp b/earth_enterprise/src/fusion/khgdal/khGDALReader.cpp index dc36d663e2..172f8ae520 100644 --- a/earth_enterprise/src/fusion/khgdal/khGDALReader.cpp +++ b/earth_enterprise/src/fusion/khgdal/khGDALReader.cpp @@ -25,6 +25,8 @@ khGDALReader::khGDALReader(const khGDALDataset &srcDS_, uint numbands) numBands(numbands), gdalDatatype(srcDS->GetRasterBand(1)->GetRasterDataType()), topToBottom(srcDS.normalizedGeoExtents().topToBottom()), + no_data_set(false), + sanitized_no_data(0), paletteSize(0), palette(0) { @@ -91,6 +93,9 @@ khGDALReader::khGDALReader(const khGDALDataset &srcDS_, uint numbands) } } +void khGDALReader::GetNoDataFromSrc(double & no_data, int & nodata_exists) { + no_data = srcDS->GetRasterBand(1)->GetNoDataValue(&nodata_exists); +} // **************************************************************************** // *** khGDALSimpleReader diff --git a/earth_enterprise/src/fusion/khgdal/khGDALReader.h b/earth_enterprise/src/fusion/khgdal/khGDALReader.h index de006aa7d9..9176a91d15 100644 --- a/earth_enterprise/src/fusion/khgdal/khGDALReader.h +++ b/earth_enterprise/src/fusion/khgdal/khGDALReader.h @@ -42,6 +42,9 @@ class khGDALReader template void TypedRead(const khExtents &readExtents, bool topToBottom, TileType &tile, const khOffset &tileOffset); + // virtual so it can be overridden by unit tests + virtual void GetNoDataFromSrc(double & no_data, int & nodata_exists); + protected: khGDALDataset srcDS; uint numBands; @@ -49,12 +52,17 @@ class khGDALReader GDALDataType gdalDatatype; khTypes::StorageEnum storage; bool topToBottom; + bool no_data_set; + double sanitized_no_data; uint paletteSize; const GDALColorEntry *palette; std::vector rawReadBuf; + // Protected so it can be called from unit tests + template SrcPixelType GetNoDataOrZero(); + // reads numBands worth of gdalDatatype pixels into rawReadBuf // assumes rawReadBuf has already been sized appropriately // will succeed or throw diff --git a/earth_enterprise/src/fusion/khgdal/khGDALReaderImpl.h b/earth_enterprise/src/fusion/khgdal/khGDALReaderImpl.h index 0b4a306843..a100a2f7db 100644 --- a/earth_enterprise/src/fusion/khgdal/khGDALReaderImpl.h +++ b/earth_enterprise/src/fusion/khgdal/khGDALReaderImpl.h @@ -19,6 +19,7 @@ #include "khGDALReader.h" #include "ReadHelpers.h" +#include // **************************************************************************** @@ -31,6 +32,8 @@ khGDALReader::TypedRead(const khExtents &srcExtents, bool topToBottom, TileType &tile, const khOffset &tileOffset) { assert(storage == khTypes::Helper::Storage); + // Make sure SrcPixelType has a min and max + assert(std::numeric_limits::is_specialized); if (topToBottom != this->topToBottom) { throw khException(kh::tr("topToBottom doesn't match dataset")); @@ -40,15 +43,12 @@ khGDALReader::TypedRead(const khExtents &srcExtents, bool topToBottom, const uint pixelsPerBand = srcExtents.width() * srcExtents.height(); const uint bandSize = (pixelsPerBand * sizeof(SrcPixelType)); rawReadBuf.reserve(numBands * bandSize); - // Fill buffer with 0 (or NoData if it exists) - int nodata_exists = 0; - SrcPixelType no_data = - static_cast( - srcDS->GetRasterBand(1)->GetNoDataValue(&nodata_exists)); - if (!nodata_exists) - no_data = 0; - for (uint i = 0; i < pixelsPerBand * numBands; ++i) + + // Fill buffer with NoData or zero + SrcPixelType no_data = GetNoDataOrZero(); + for (uint i = 0; i < pixelsPerBand * numBands; ++i) { ((SrcPixelType *) (&rawReadBuf[0]))[i] = no_data; + } // read pixels from all bands into rawReadBuf FetchPixels(srcExtents); @@ -108,5 +108,35 @@ khGDALReader::TypedRead(const khExtents &srcExtents, bool topToBottom, } } +template +SrcPixelType khGDALReader::GetNoDataOrZero() { + if (!no_data_set) { + double no_data; + int nodata_exists; + GetNoDataFromSrc(no_data, nodata_exists); + if (nodata_exists) { + // Check that nodata fits in the pixel type + SrcPixelType pixelMin = std::numeric_limits::lowest(); + SrcPixelType pixelMax = std::numeric_limits::max(); + if (no_data >= pixelMin && no_data <= pixelMax) { + sanitized_no_data = no_data; + } + else { + notify(NFY_WARN, "Ignoring NoData (%.2f) because it is too large for the pixel type.", no_data); + } + } + + if (sanitized_no_data != 0) { + notify(NFY_NOTICE, "Using non-zero NoData (%.2f). If Fusion produces black regions " + "around this image, consider creating your own mask and disabling " + "Auto Masking for this resource.", no_data); + } + + no_data_set = true; + } + + return static_cast(sanitized_no_data); +} + #endif /* __khGDALReaderImpl_h */ diff --git a/earth_enterprise/src/fusion/khgdal/khGDALReader_unittest.cpp b/earth_enterprise/src/fusion/khgdal/khGDALReader_unittest.cpp new file mode 100644 index 0000000000..a2aacf5649 --- /dev/null +++ b/earth_enterprise/src/fusion/khgdal/khGDALReader_unittest.cpp @@ -0,0 +1,91 @@ +// Copyright 2020 The Open GEE Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include "khGDALReaderImpl.h" + +// Point the dataset at a valid image. We won't use this, but it prevents crashes +// due to an invalid dataset. +khGDALDataset dataset("fusion/testdata/i3SF15-meter.tif"); + +class MockReader : public khGDALReader { + public: + double no_data_val; + int nodata_exists_val; + uint num_gets; + MockReader(double no_data, int nodata_exists) : + khGDALReader(dataset, 3), // Values don't matter; just making the constructor happy + no_data_val(no_data), nodata_exists_val(nodata_exists), num_gets(0) { } + + // Give callers access to the functions we want to test + template + SrcPixelType CallGetNoDataOrZero() { + return GetNoDataOrZero(); + } + private: + virtual void GetNoDataFromSrc(double & no_data, int & nodata_exists) override { + no_data = no_data_val; + nodata_exists = nodata_exists_val; + ++num_gets; + } + + // Override abstract functions so we can instantiate the class + virtual void FetchPixels(const khExtents &srcExtents) override {} +}; + +TEST(khGDALReaderTest, NoDataDoesntExist) { + MockReader reader(255, false); + int8 nodata = reader.CallGetNoDataOrZero(); + ASSERT_EQ(0, nodata); +} + +TEST(khGDALReaderTest, NoDataZero) { + MockReader reader(0, true); + uint32 nodata = reader.CallGetNoDataOrZero(); + ASSERT_EQ(0, nodata); +} + +TEST(khGDALReaderTest, NoDataNonZero) { + MockReader reader(-9999, true); + float32 nodata = reader.CallGetNoDataOrZero(); + ASSERT_EQ(-9999, nodata); +} + +TEST(khGDALReaderTest, NoDataTooBigPositive) { + MockReader reader(256, true); + uint8 nodata = reader.CallGetNoDataOrZero(); + ASSERT_EQ(0, nodata); +} + +TEST(khGDALReaderTest, NoDataTooBigNegative) { + MockReader reader(-100000, true); + int16 nodata = reader.CallGetNoDataOrZero(); + ASSERT_EQ(0, nodata); +} + +TEST(khGDALReaderTest, NoDataMultipleCalls) { + MockReader reader(16, true); + float64 nodata = reader.CallGetNoDataOrZero(); + ASSERT_EQ(16, nodata); + ASSERT_EQ(1, reader.num_gets); + nodata = reader.CallGetNoDataOrZero(); + ASSERT_EQ(16, nodata); + // Make sure we only extract the NoData value once + ASSERT_EQ(1, reader.num_gets); +} + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/earth_enterprise/src/fusion/khgdal/khgdal.h b/earth_enterprise/src/fusion/khgdal/khgdal.h index f540ffb16f..e781c93a30 100644 --- a/earth_enterprise/src/fusion/khgdal/khgdal.h +++ b/earth_enterprise/src/fusion/khgdal/khgdal.h @@ -32,9 +32,9 @@ // **************************************************************************** class CPLStrGuard { - // private and unimplemented - it would be a waste to copy the strings - CPLStrGuard(const CPLStrGuard &o); - CPLStrGuard& operator=(const CPLStrGuard &o); + // it would be a waste to copy the strings + CPLStrGuard(const CPLStrGuard &o) = delete; + CPLStrGuard& operator=(const CPLStrGuard &o) = delete; char *str; public: From 3ad395bf75bb2d3cb8ea9af609b1c683ad5cf31f Mon Sep 17 00:00:00 2001 From: tst-cjeffries <44170038+tst-cjeffries@users.noreply.github.com> Date: Tue, 3 Mar 2020 13:57:25 -0600 Subject: [PATCH 4/4] Add ius repo to rhel6 instructions. (#1653) --- earth_enterprise/BUILD_RHEL_CentOS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/earth_enterprise/BUILD_RHEL_CentOS.md b/earth_enterprise/BUILD_RHEL_CentOS.md index 21dc585fd1..427b7020b3 100644 --- a/earth_enterprise/BUILD_RHEL_CentOS.md +++ b/earth_enterprise/BUILD_RHEL_CentOS.md @@ -39,6 +39,7 @@ sudo subscription-manager repos --enable=rhel-server-dts2-6-rpms # For all RHEL 6 Editions: sudo subscription-manager repos --enable=rhel-6-server-optional-rpms sudo yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-6.noarch.rpm +sudo yum install -y https://repo.ius.io/ius-release-el6.rpm ``` ## Install Git