From 8c9ca79c95901fb2c731f7c856225c13b7115671 Mon Sep 17 00:00:00 2001 From: Felix Schott Date: Mon, 10 Mar 2025 12:17:06 +0100 Subject: [PATCH 01/11] add 'extract' command with column and spatial subsetting --- cmd/gpq/command/command.go | 9 + cmd/gpq/command/convert.go | 10 +- cmd/gpq/command/extract.go | 104 ++++++++++ cmd/gpq/command/extract_test.go | 123 +++++++++++ internal/geo/geo.go | 78 +++++++ internal/geo/geo_test.go | 137 +++++++++++++ internal/geoparquet/geoparquet.go | 147 ++++++++++++++ internal/geoparquet/geoparquet_test.go | 192 ++++++++++++++++++ internal/geoparquet/metadata.go | 12 ++ internal/geoparquet/recordreader.go | 123 ++++++++++- .../cases/example-v1.1.0-covering.parquet | Bin 0 -> 28588 bytes .../testdata/cases/example-v1.1.0.parquet | Bin 0 -> 29834 bytes 12 files changed, 921 insertions(+), 14 deletions(-) create mode 100644 cmd/gpq/command/extract.go create mode 100644 cmd/gpq/command/extract_test.go create mode 100644 internal/geo/geo_test.go create mode 100644 internal/testdata/cases/example-v1.1.0-covering.parquet create mode 100644 internal/testdata/cases/example-v1.1.0.parquet diff --git a/cmd/gpq/command/command.go b/cmd/gpq/command/command.go index 53ff7f4..65d0574 100644 --- a/cmd/gpq/command/command.go +++ b/cmd/gpq/command/command.go @@ -15,6 +15,7 @@ var CLI struct { Convert ConvertCmd `cmd:"" help:"Convert data from one format to another."` Validate ValidateCmd `cmd:"" help:"Validate a GeoParquet file."` Describe DescribeCmd `cmd:"" help:"Describe a GeoParquet file."` + Extract ExtractCmd `cmd:"" help:"Extract columns by name or rows by spatial subsetting."` Version VersionCmd `cmd:"" help:"Print the version of this program."` } @@ -49,3 +50,11 @@ func readerFromInput(input string) (storage.ReaderAtSeeker, error) { return os.Open(input) } + +func hasStdin() bool { + stats, err := os.Stdin.Stat() + if err != nil { + return false + } + return stats.Size() > 0 +} diff --git a/cmd/gpq/command/convert.go b/cmd/gpq/command/convert.go index febb425..4e6b1f8 100644 --- a/cmd/gpq/command/convert.go +++ b/cmd/gpq/command/convert.go @@ -32,7 +32,7 @@ type ConvertCmd struct { To string `help:"Output file format. Possible values: ${enum}." enum:"auto, geojson, geoparquet" default:"auto"` Min int `help:"Minimum number of features to consider when building a schema." default:"10"` Max int `help:"Maximum number of features to consider when building a schema." default:"100"` - InputPrimaryColumn string `help:"Primary geometry column name when reading Parquet withtout metadata." default:"geometry"` + InputPrimaryColumn string `help:"Primary geometry column name when reading Parquet without metadata." default:"geometry"` Compression string `help:"Parquet compression to use. Possible values: ${enum}." enum:"uncompressed, snappy, gzip, brotli, zstd" default:"zstd"` RowGroupLength int `help:"Maximum number of rows per group when writing Parquet."` } @@ -100,14 +100,6 @@ func getFormatType(resource string) FormatType { return UnknownType } -func hasStdin() bool { - stats, err := os.Stdin.Stat() - if err != nil { - return false - } - return stats.Size() > 0 -} - func (c *ConvertCmd) Run() error { inputSource := c.Input outputSource := c.Output diff --git a/cmd/gpq/command/extract.go b/cmd/gpq/command/extract.go new file mode 100644 index 0000000..fd64b07 --- /dev/null +++ b/cmd/gpq/command/extract.go @@ -0,0 +1,104 @@ +package command + +import ( + "context" + "io" + "os" + "strings" + + "github.com/planetlabs/gpq/internal/geo" + "github.com/planetlabs/gpq/internal/geoparquet" +) + +type ExtractCmd struct { + Input string `arg:"" optional:"" name:"input" help:"Input file path or URL. If not provided, input is read from stdin."` + Output string `arg:"" optional:"" name:"output" help:"Output file. If not provided, output is written to stdout." type:"path"` + Bbox string `help:"Filter features by intersection of their bounding box with the provided bounding box (in x_min,y_min,x_max,y_max format)."` + DropCols string `help:"Drop the provided columns. Provide a comma-separated string of column names to be excluded. Do not use together with --keep-only-cols."` + KeepOnlyCols string `help:"Keep only the provided columns. Provide a comma-separated string of columns to be kept. Do not use together with --drop-cols."` +} + +func (c *ExtractCmd) Run() error { + + // validate and transform inputs + + inputSource := c.Input + outputSource := c.Output + + if c.Input == "" && hasStdin() { + outputSource = inputSource + inputSource = "" + } + + input, inputErr := readerFromInput(inputSource) + if inputErr != nil { + return NewCommandError("trouble getting a reader from %q: %w", c.Input, inputErr) + } + + var output *os.File + if outputSource == "" { + output = os.Stdout + } else { + o, createErr := os.Create(outputSource) + if createErr != nil { + return NewCommandError("failed to open %q for writing: %w", outputSource, createErr) + } + defer o.Close() + output = o + } + + // prepare input reader (ignore certain columns if asked to - DropCols/KeepOnlyCols) + config := &geoparquet.ReaderConfig{Reader: input} + if c.DropCols != "" { + cols := strings.Split(c.DropCols, ",") + config.ExcludeColNames = cols + } + if c.KeepOnlyCols != "" { + cols := strings.Split(c.KeepOnlyCols, ",") + config.IncludeColNames = cols + } + + recordReader, rrErr := geoparquet.NewRecordReader(config) + if rrErr != nil { + return NewCommandError("trouble reading geoparquet: %w", rrErr) + } + defer recordReader.Close() + + // prepare output writer + recordWriter, rwErr := geoparquet.NewRecordWriter(&geoparquet.WriterConfig{ + Writer: output, + Metadata: recordReader.Metadata(), + ArrowSchema: recordReader.ArrowSchema(), + }) + if rwErr != nil { + return NewCommandError("trouble getting record writer: %w", rwErr) + } + defer recordWriter.Close() + + // parse bbox filter argument into geo.Bbox struct if applicable + inputBbox, err := geo.NewBboxFromString(c.Bbox) + if err != nil { + return NewCommandError(err.Error()) + } + + // read and write records in loop + for { + record, readErr := recordReader.Read() + if readErr == io.EOF { + break + } + if readErr != nil { + return readErr + } + + filteredRecord, err := geoparquet.FilterRecordBatchByBbox(context.Background(), recordReader, &record, inputBbox) + if err != nil { + return NewCommandError(err.Error()) + } + + if err := recordWriter.Write(*filteredRecord); err != nil { + return err + } + } + return nil +} diff --git a/cmd/gpq/command/extract_test.go b/cmd/gpq/command/extract_test.go new file mode 100644 index 0000000..345d2ea --- /dev/null +++ b/cmd/gpq/command/extract_test.go @@ -0,0 +1,123 @@ +package command_test + +import ( + "bytes" + + "github.com/apache/arrow/go/v16/parquet/file" + "github.com/planetlabs/gpq/cmd/gpq/command" + "github.com/planetlabs/gpq/internal/geoparquet" +) + +func (s *Suite) TestExtractDropCols() { + cmd := &command.ExtractCmd{ + Input: "../../../internal/testdata/cases/example-v1.0.0.parquet", + DropCols: "pop_est,iso_a3", + } + s.Require().NoError(cmd.Run()) + + data := s.readStdout() + + fileReader, err := file.NewParquetReader(bytes.NewReader(data)) + s.Require().NoError(err) + defer fileReader.Close() + + s.Equal(int64(5), fileReader.NumRows()) + + s.Require().NoError(err) + s.Equal(4, fileReader.MetaData().Schema.NumColumns()) + + recordReader, err := geoparquet.NewRecordReader(&geoparquet.ReaderConfig{ + Reader: bytes.NewReader(data), + }) + s.Require().NoError(err) + defer recordReader.Close() + + record, readErr := recordReader.Read() + s.Require().NoError(readErr) + s.Assert().Equal(int64(4), record.NumCols()) +} + +func (s *Suite) TestExtractKeepOnlyCols() { + cmd := &command.ExtractCmd{ + Input: "../../../internal/testdata/cases/example-v1.1.0.parquet", + KeepOnlyCols: "geometry,pop_est,iso_a3", + } + s.Require().NoError(cmd.Run()) + + data := s.readStdout() + + fileReader, err := file.NewParquetReader(bytes.NewReader(data)) + s.Require().NoError(err) + defer fileReader.Close() + + s.Equal(int64(5), fileReader.NumRows()) + + s.Require().NoError(err) + s.Equal(3, fileReader.MetaData().Schema.NumColumns()) + + recordReader, err := geoparquet.NewRecordReader(&geoparquet.ReaderConfig{ + Reader: bytes.NewReader(data), + }) + s.Require().NoError(err) + defer recordReader.Close() + + record, readErr := recordReader.Read() + s.Require().NoError(readErr) + s.Assert().Equal(int64(3), record.NumCols()) +} + +// Since the 1.1.0 parquet file includes a bbox column, we expect the bbox column to be used for spatial filtering. +func (s *Suite) TestExtractBbox110() { + cmd := &command.ExtractCmd{ + Input: "../../../internal/testdata/cases/example-v1.1.0.parquet", + Bbox: "34,-7,36,-6", + } + s.Require().NoError(cmd.Run()) + + data := s.readStdout() + + recordReader, err := geoparquet.NewRecordReader(&geoparquet.ReaderConfig{ + Reader: bytes.NewReader(data), + }) + s.Require().NoError(err) + defer recordReader.Close() + + // we expect only one row, namely Tanzania + s.Require().Equal(int64(1), recordReader.NumRows()) + + record, readErr := recordReader.Read() + s.Require().NoError(readErr) + s.Assert().Equal(int64(7), record.NumCols()) + s.Assert().Equal(int64(1), record.NumRows()) + + country := record.Column(recordReader.Schema().ColumnIndexByName("name")).ValueStr(0) + s.Assert().Equal("Tanzania", country) +} + +// Since the 1.0.0 parquet file doesn't have a bbox column, we expect the bbox column to be calculated on the fly. +func (s *Suite) TestExtractBbox100() { + cmd := &command.ExtractCmd{ + Input: "../../../internal/testdata/cases/example-v1.0.0.parquet", + Bbox: "34,-7,36,-6", + } + s.Require().NoError(cmd.Run()) + + data := s.readStdout() + + recordReader, err := geoparquet.NewRecordReader(&geoparquet.ReaderConfig{ + Reader: bytes.NewReader(data), + }) + s.Require().NoError(err) + defer recordReader.Close() + + // we expect only one row, namely Tanzania + s.Require().Equal(int64(1), recordReader.NumRows()) + + record, readErr := recordReader.Read() + s.Require().NoError(readErr) + s.Assert().Equal(int64(6), record.NumCols()) + s.Assert().Equal(int64(1), record.NumRows()) + + country := record.Column(recordReader.Schema().ColumnIndexByName("name")).ValueStr(0) + s.Assert().Equal("Tanzania", country) +} diff --git a/internal/geo/geo.go b/internal/geo/geo.go index f560dbe..9794fa7 100644 --- a/internal/geo/geo.go +++ b/internal/geo/geo.go @@ -2,8 +2,11 @@ package geo import ( "encoding/json" + "errors" "fmt" "math" + "strconv" + "strings" "sync" "github.com/paulmach/orb" @@ -334,3 +337,78 @@ func (i *DatasetStats) Types(name string) []string { i.readUnlock() return collection.Types() } + +type Bbox struct { + Xmin float64 + Ymin float64 + Xmax float64 + Ymax float64 +} + +// Checks whether the bbox overlaps with another axis-aligned bbox. +func (box1 *Bbox) Intersects(box2 *Bbox) bool { + // check latitude overlap + if box1.Ymax < box2.Ymin || box2.Ymax < box1.Ymin { + return false + } + + // shift all negative x coordinates to accomodate antimeridian crossings + if box1.Xmin < 0 { + box1.Xmin += 360 + } + if box1.Xmax < 0 { + box1.Xmax += 360 + } + if box2.Xmin < 0 { + box2.Xmin += 360 + } + if box2.Xmax < 0 { + box2.Xmax += 360 + } + + // check longitude overlap + if box1.Xmax < box2.Xmin || box2.Xmax < box1.Xmin { + return false + } + + return true +} + +// Create a new Bbox struct from a string of comma-separated values in format xmin,ymin,xmax,ymax. +func NewBboxFromString(bounds string) (*Bbox, error) { + inputBbox := &Bbox{} + + if bounds != "" { + bboxValues := strings.Split(bounds, ",") + if len(bboxValues) != 4 { + return nil, errors.New("please provide 4 comma-separated values (xmin,ymin,xmax,ymax) as a bbox") + } + + xminInput, err := strconv.ParseFloat(bboxValues[0], 64) + if err != nil { + return nil, fmt.Errorf("trouble parsing xmin input as float64: %w", err) + } + inputBbox.Xmin = xminInput + + yminInput, err := strconv.ParseFloat(bboxValues[1], 64) + if err != nil { + return nil, fmt.Errorf("trouble parsing ymin input as float64: %w", err) + } + inputBbox.Ymin = yminInput + + xmaxInput, err := strconv.ParseFloat(bboxValues[2], 64) + if err != nil { + return nil, fmt.Errorf("trouble parsing xmax input as float64: %w", err) + } + inputBbox.Xmax = xmaxInput + + ymaxInput, err := strconv.ParseFloat(bboxValues[3], 64) + if err != nil { + return nil, fmt.Errorf("trouble parsing ymax input as float64: %w", err) + } + inputBbox.Ymax = ymaxInput + } else { + inputBbox = nil + } + return inputBbox, nil +} diff --git a/internal/geo/geo_test.go b/internal/geo/geo_test.go new file mode 100644 index 0000000..f912b5a --- /dev/null +++ b/internal/geo/geo_test.go @@ -0,0 +1,137 @@ +package geo + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestBboxIntersectsTrue(t *testing.T) { + box1 := &Bbox{ + Xmin: 10, + Ymin: 20, + Xmax: 30, + Ymax: 40, + } + + box2 := &Bbox{ + Xmin: 25, + Ymin: 35, + Xmax: 45, + Ymax: 55, + } + + require.Equal(t, box1.Intersects(box2), true) +} + +func TestBboxIntersectsFalse(t *testing.T) { + box1 := &Bbox{ + Xmin: -10, + Ymin: 20, + Xmax: -5, + Ymax: 40, + } + + box2 := &Bbox{ + Xmin: -1, + Ymin: 50, + Xmax: 0, + Ymax: 70, + } + + require.Equal(t, box1.Intersects(box2), false) +} + +func TestBboxIntersectsTouches(t *testing.T) { + box1 := &Bbox{ + Xmin: 10, + Ymin: 20, + Xmax: 30, + Ymax: 40, + } + + box2 := &Bbox{ + Xmin: 30, + Ymin: 20, + Xmax: 40, + Ymax: 40, + } + + require.Equal(t, box1.Intersects(box2), true) +} + +func TestBboxIntersectsContains(t *testing.T) { + box1 := &Bbox{ + Xmin: 10, + Ymin: 10, + Xmax: 30, + Ymax: 30, + } + + box2 := &Bbox{ + Xmin: 0, + Ymin: 0, + Xmax: 40, + Ymax: 40, + } + + require.Equal(t, box1.Intersects(box2), true) +} + +func TestBboxIntersectsTrueAntimeridian(t *testing.T) { + box1 := &Bbox{ + Xmin: 170, + Ymin: -10, + Xmax: -165, + Ymax: 10, + } + + box2 := &Bbox{ + Xmin: -180, + Ymin: -5, + Xmax: -170, + Ymax: 15, + } + + require.Equal(t, box1.Intersects(box2), true) +} + +func TestBboxIntersectsFalseAntimeridian(t *testing.T) { + box1 := &Bbox{ + Xmin: 170, + Ymin: -10, + Xmax: 180, + Ymax: 10, + } + + box2 := &Bbox{ + Xmin: -160, + Ymin: -5, + Xmax: -150, + Ymax: 15, + } + + require.Equal(t, box1.Intersects(box2), false) +} + +func TestNewBboxFromString(t *testing.T) { + bbox, err := NewBboxFromString("-160,-5,-150,15") + assert.NoError(t, err) + assert.Equal(t, -160.0, bbox.Xmin) + assert.Equal(t, -5.0, bbox.Ymin) + assert.Equal(t, -150.0, bbox.Xmax) + assert.Equal(t, 15.0, bbox.Ymax) +} + +func TestNewBboxFromStringErrNotEnoughValues(t *testing.T) { + bbox, err := NewBboxFromString("-160,-5,-150") + assert.ErrorContains(t, err, "please provide 4") + assert.Nil(t, bbox) +} + +func TestNewBboxFromStringErrWrongType(t *testing.T) { + bbox, err := NewBboxFromString("foo,-5,-150,15") + assert.ErrorContains(t, err, "float") + assert.Nil(t, bbox) +} diff --git a/internal/geoparquet/geoparquet.go b/internal/geoparquet/geoparquet.go index faaa6a3..1eb6486 100644 --- a/internal/geoparquet/geoparquet.go +++ b/internal/geoparquet/geoparquet.go @@ -1,6 +1,7 @@ package geoparquet import ( + "context" "encoding/json" "errors" "fmt" @@ -8,6 +9,7 @@ import ( "github.com/apache/arrow/go/v16/arrow" "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/compute" "github.com/apache/arrow/go/v16/arrow/memory" "github.com/apache/arrow/go/v16/parquet" "github.com/apache/arrow/go/v16/parquet/compress" @@ -189,3 +191,148 @@ func FromParquet(input parquet.ReaderAtSeeker, output io.Writer, convertOptions return pqutil.TransformByColumn(config) } + +// Returns the index of the bbox column, -1 means not found. +// If there is no match for the standard name "bbox", the covering metadata is consulted. +func GetBboxColumnIndex(schema *schema.Schema, metadata *Metadata) int { + // try standard name first + bboxColIdx := schema.Root().FieldIndexByName("bbox") + // if no match, check covering metadata + if bboxColIdx == -1 && metadata.Columns[metadata.PrimaryColumn].Covering != nil && len(metadata.Columns[metadata.PrimaryColumn].Covering.Bbox.Xmin) == 2 { + bboxColName := metadata.Columns[metadata.PrimaryColumn].Covering.Bbox.Xmin[0] + bboxColIdx = schema.Root().FieldIndexByName(bboxColName) + } + return bboxColIdx +} + +func FilterRecordBatchByBbox(ctx context.Context, recordReader *RecordReader, record *arrow.Record, inputBbox *geo.Bbox) (*arrow.Record, error) { + + metadata := recordReader.Metadata() + schema := recordReader.Schema() + + bboxColIdx := -1 // -1 means no column found + if inputBbox != nil { + bboxColIdx = GetBboxColumnIndex(schema, metadata) + } + + var filteredRecord *arrow.Record + + if inputBbox != nil && bboxColIdx != -1 { // bbox argument has been provided and there is a bbox column we can use for filtering + col := (*record).Column(bboxColIdx).(*array.Struct) + defer col.Release() + + // we build a boolean mask and pass it to compute.FilterRecordBatch later + maskBuilder := array.NewBooleanBuilder(memory.DefaultAllocator) + defer maskBuilder.Release() + + var xminName string + var yminName string + var xmaxName string + var ymaxName string + + // loop over individual bbox values per record + for idx := range col.Len() { + var bbox map[string]json.RawMessage + if err := json.Unmarshal([]byte(col.ValueStr(idx)), &bbox); err != nil { + return nil, fmt.Errorf("trouble unmarshalling bbox struct: %w", err) + } + + // infer bbox field names from the first element + if idx == 0 { + // check standard name first, if no match, check covering metadata + if _, ok := bbox["xmin"]; ok { + xminName = "xmin" + } else if metadata.Columns[metadata.PrimaryColumn].Covering != nil { + xminName = "xmin" // DEBUG metadata.Columns[metadata.PrimaryColumn].Covering.Xmin[1] + } else { + return nil, fmt.Errorf("can not infer bbox field name for 'xmin'") + } + + if _, ok := bbox["ymin"]; ok { + yminName = "ymin" + } else if metadata.Columns[metadata.PrimaryColumn].Covering != nil { + yminName = metadata.Columns[metadata.PrimaryColumn].Covering.Bbox.Ymin[1] + } else { + return nil, fmt.Errorf("can not infer bbox field name for 'ymin'") + } + + if _, ok := bbox["xmax"]; ok { // check standard name first + xmaxName = "xmax" + } else if metadata.Columns[metadata.PrimaryColumn].Covering != nil { + xmaxName = metadata.Columns[metadata.PrimaryColumn].Covering.Bbox.Xmax[1] + } else { + return nil, fmt.Errorf("can not infer bbox field name for 'xmax'") + } + + if _, ok := bbox["ymax"]; ok { + ymaxName = "ymax" + } else if metadata.Columns[metadata.PrimaryColumn].Covering != nil { + ymaxName = metadata.Columns[metadata.PrimaryColumn].Covering.Bbox.Ymax[1] + } else { + return nil, fmt.Errorf("can not infer bbox field name for 'ymax'") + } + } + + bboxValue := &geo.Bbox{} // create empty struct to hold bbox values of this record + + if err := json.Unmarshal(bbox[xminName], &bboxValue.Xmin); err != nil { + return nil, fmt.Errorf("trouble parsing bbox.%v field: %w", xminName, err) + } + if err := json.Unmarshal(bbox[yminName], &bboxValue.Ymin); err != nil { + return nil, fmt.Errorf("trouble parsing bbox.%v field: %w", yminName, err) + } + if err := json.Unmarshal(bbox[xmaxName], &bboxValue.Xmax); err != nil { + return nil, fmt.Errorf("trouble parsing bbox.%v field: %w", xmaxName, err) + } + if err := json.Unmarshal(bbox[ymaxName], &bboxValue.Ymax); err != nil { + return nil, fmt.Errorf("trouble parsing bbox.%v field: %w", ymaxName, err) + } + + // check whether the bbox passed to this function + // intersects with the bbox of the record + maskBuilder.Append(inputBbox.Intersects(bboxValue)) + } + + r, filterErr := compute.FilterRecordBatch(ctx, *record, maskBuilder.NewBooleanArray(), &compute.FilterOptions{NullSelection: 0}) // TODO check what this is doing + if filterErr != nil { + return nil, fmt.Errorf("trouble filtering record batch: %w", filterErr) + } + filteredRecord = &r + } else if inputBbox != nil && bboxColIdx == -1 { + // bbox filter passed to function but there is no bbox col. + // this means we have to compute the bbox of the records ourselves + primaryColIdx := schema.ColumnIndexByName(metadata.PrimaryColumn) + col := (*record).Column(primaryColIdx) + defer col.Release() + + maskBuilder := array.NewBooleanBuilder(memory.DefaultAllocator) + defer maskBuilder.Release() + + for idx := range col.Len() { + value := col.GetOneForMarshal(idx) + g, decodeErr := geo.DecodeGeometry(value, metadata.Columns[metadata.PrimaryColumn].Encoding) + if decodeErr != nil { + return nil, fmt.Errorf("trouble decoding geometry: %w", decodeErr) + } + bounds := g.Coordinates.Bound() + bboxValue := &geo.Bbox{ + Xmin: bounds.Min.X(), + Ymin: bounds.Min.Y(), + Xmax: bounds.Max.X(), + Ymax: bounds.Max.Y(), + } + + // now that we've computed the bbox, same logic as above + maskBuilder.Append(inputBbox.Intersects(bboxValue)) + } + filter := maskBuilder.NewBooleanArray() + r, filterErr := compute.FilterRecordBatch(ctx, *record, filter, &compute.FilterOptions{NullSelection: 0}) // TODO check what this is doing + if filterErr != nil { + return nil, fmt.Errorf("trouble filtering record batch with computed bbox: %w (%v vs. %v)", filterErr, (*record).NumRows(), filter.Len()) + } + filteredRecord = &r + } else { + filteredRecord = record + } + return filteredRecord, nil +} diff --git a/internal/geoparquet/geoparquet_test.go b/internal/geoparquet/geoparquet_test.go index f6e5a5a..024a6fd 100644 --- a/internal/geoparquet/geoparquet_test.go +++ b/internal/geoparquet/geoparquet_test.go @@ -137,6 +137,7 @@ func TestRowReaderV100Beta1(t *testing.T) { require.NoError(t, err) numRows := 0 + var numCols int for { record, err := reader.Read() if err == io.EOF { @@ -144,9 +145,72 @@ func TestRowReaderV100Beta1(t *testing.T) { } require.NoError(t, err) numRows += int(record.NumRows()) + numCols = int(record.NumCols()) } assert.Equal(t, 5, numRows) + assert.Equal(t, 6, numCols) +} + +func TestRecordReaderV100ExcludeCols(t *testing.T) { + fixturePath := "../testdata/cases/example-v1.0.0.parquet" + input, openErr := os.Open(fixturePath) + require.NoError(t, openErr) + + reader, err := geoparquet.NewRecordReader(&geoparquet.ReaderConfig{ + Reader: input, + ExcludeColNames: []string{"continent", "gdp_md_est"}, + }) + require.NoError(t, err) + + record, err := reader.Read() + require.NoError(t, err) + assert.Equal(t, record.NumCols(), int64(4)) + + fields := record.Schema().Fields() + colNames := make([]string, len(fields)) + for idx, field := range fields { + colNames[idx] = field.Name + } + + assert.ElementsMatch(t, colNames, []string{"geometry", "pop_est", "iso_a3", "name"}) +} + +func TestRecordReaderV110IncludeCols(t *testing.T) { + fixturePath := "../testdata/cases/example-v1.0.0.parquet" + input, openErr := os.Open(fixturePath) + require.NoError(t, openErr) + + reader, err := geoparquet.NewRecordReader(&geoparquet.ReaderConfig{ + Reader: input, + IncludeColNames: []string{"geometry", "continent", "gdp_md_est"}, + }) + require.NoError(t, err) + + record, err := reader.Read() + require.NoError(t, err) + assert.Equal(t, record.NumCols(), int64(3)) + + fields := record.Schema().Fields() + colNames := make([]string, len(fields)) + for idx, field := range fields { + colNames[idx] = field.Name + } + + assert.ElementsMatch(t, colNames, []string{"geometry", "continent", "gdp_md_est"}) +} + +func TestRecordReaderV110NoGeomColError(t *testing.T) { + fixturePath := "../testdata/cases/example-v1.1.0.parquet" + input, openErr := os.Open(fixturePath) + require.NoError(t, openErr) + + reader, err := geoparquet.NewRecordReader(&geoparquet.ReaderConfig{ + Reader: input, + IncludeColNames: []string{"continent", "gdp_md_est"}, + }) + require.ErrorContains(t, err, "geometry column") + require.Nil(t, reader) } func toWKB(t *testing.T, geometry orb.Geometry) []byte { @@ -394,3 +458,131 @@ func TestRecordReading(t *testing.T) { assert.Equal(t, reader.NumRows(), int64(numRows)) } + +func TestGetBboxColumnIdxV100(t *testing.T) { + f, fileErr := os.Open("../testdata/cases/example-v1.0.0.parquet") + require.NoError(t, fileErr) + reader, readerErr := file.NewParquetReader(f) + require.NoError(t, readerErr) + defer reader.Close() + + metadata, err := geoparquet.GetMetadata(reader.MetaData().KeyValueMetadata()) + require.NoError(t, err) + + // no bbox col in the file, we expect -1 + colIdx := geoparquet.GetBboxColumnIndex(reader.MetaData().Schema, metadata) + require.Equal(t, -1, colIdx) +} + +func TestGetBboxColumnIdxV110(t *testing.T) { + f, fileErr := os.Open("../testdata/cases/example-v1.1.0.parquet") + require.NoError(t, fileErr) + reader, readerErr := file.NewParquetReader(f) + require.NoError(t, readerErr) + defer reader.Close() + + metadata, err := geoparquet.GetMetadata(reader.MetaData().KeyValueMetadata()) + require.NoError(t, err) + + // there is a bbox col in the file, we expect index 6 + colIdx := geoparquet.GetBboxColumnIndex(reader.MetaData().Schema, metadata) + require.Equal(t, 6, colIdx) +} + +func TestGetBboxColumnIdxV110NonStandardBboxCol(t *testing.T) { + f, fileErr := os.Open("../testdata/cases/example-v1.1.0-covering.parquet") + require.NoError(t, fileErr) + reader, readerErr := file.NewParquetReader(f) + require.NoError(t, readerErr) + defer reader.Close() + + metadata, err := geoparquet.GetMetadata(reader.MetaData().KeyValueMetadata()) + require.NoError(t, err) + + // there is a bbox col in the file with the non-standard name "geometry_bbox", + // we expect index 6 + colIdx := geoparquet.GetBboxColumnIndex(reader.MetaData().Schema, metadata) + assert.Equal(t, 6, colIdx) + assert.Equal(t, 6, reader.MetaData().Schema.Root().FieldIndexByName("geometry_bbox")) +} + +func TestFilterByBboxV100(t *testing.T) { + fileReader, fileErr := os.Open("../testdata/cases/example-v1.0.0.parquet") + require.NoError(t, fileErr) + defer fileReader.Close() + + recordReader, err := geoparquet.NewRecordReader(&geoparquet.ReaderConfig{Reader: fileReader}) + require.NoError(t, err) + defer recordReader.Close() + + record, readErr := recordReader.Read() + require.NoError(t, readErr) + assert.Equal(t, int64(6), record.NumCols()) + assert.Equal(t, int64(5), record.NumRows()) + + inputBbox := &geo.Bbox{Xmin: 34.0, Ymin: -7.0, Xmax: 36.0, Ymax: -6.0} + + filteredRecord, err := geoparquet.FilterRecordBatchByBbox(context.Background(), recordReader, &record, inputBbox) + require.NoError(t, err) + + // we expect only one row, namely Tanzania + assert.Equal(t, int64(6), (*filteredRecord).NumCols()) + assert.Equal(t, int64(1), (*filteredRecord).NumRows()) + + country := (*filteredRecord).Column(recordReader.Schema().ColumnIndexByName("name")).ValueStr(0) + assert.Equal(t, "Tanzania", country) +} + +func TestFilterByBboxV110(t *testing.T) { + fileReader, fileErr := os.Open("../testdata/cases/example-v1.1.0.parquet") + require.NoError(t, fileErr) + defer fileReader.Close() + + recordReader, err := geoparquet.NewRecordReader(&geoparquet.ReaderConfig{Reader: fileReader}) + require.NoError(t, err) + defer recordReader.Close() + + record, readErr := recordReader.Read() + require.NoError(t, readErr) + assert.Equal(t, int64(7), record.NumCols()) + assert.Equal(t, int64(5), record.NumRows()) + + inputBbox := &geo.Bbox{Xmin: 34.0, Ymin: -7.0, Xmax: 36.0, Ymax: -6.0} + + filteredRecord, err := geoparquet.FilterRecordBatchByBbox(context.Background(), recordReader, &record, inputBbox) + require.NoError(t, err) + + // we expect only one row, namely Tanzania + assert.Equal(t, int64(7), (*filteredRecord).NumCols()) + assert.Equal(t, int64(1), (*filteredRecord).NumRows()) + + country := (*filteredRecord).Column(recordReader.Schema().ColumnIndexByName("name")).ValueStr(0) + assert.Equal(t, "Tanzania", country) +} + +func TestFilterByBboxV110NonStandardBboxCol(t *testing.T) { + fileReader, fileErr := os.Open("../testdata/cases/example-v1.1.0-covering.parquet") + require.NoError(t, fileErr) + defer fileReader.Close() + + recordReader, err := geoparquet.NewRecordReader(&geoparquet.ReaderConfig{Reader: fileReader}) + require.NoError(t, err) + defer recordReader.Close() + + record, readErr := recordReader.Read() + require.NoError(t, readErr) + assert.Equal(t, int64(7), record.NumCols()) + assert.Equal(t, int64(5), record.NumRows()) + + inputBbox := &geo.Bbox{Xmin: 34.0, Ymin: -7.0, Xmax: 36.0, Ymax: -6.0} + + filteredRecord, err := geoparquet.FilterRecordBatchByBbox(context.Background(), recordReader, &record, inputBbox) + require.NoError(t, err) + + // we expect only one row, namely Tanzania + assert.Equal(t, int64(7), (*filteredRecord).NumCols()) + assert.Equal(t, int64(1), (*filteredRecord).NumRows()) + + country := (*filteredRecord).Column(recordReader.Schema().ColumnIndexByName("name")).ValueStr(0) + assert.Equal(t, "Tanzania", country) +} diff --git a/internal/geoparquet/metadata.go b/internal/geoparquet/metadata.go index 506aa09..4d1ce27 100644 --- a/internal/geoparquet/metadata.go +++ b/internal/geoparquet/metadata.go @@ -79,6 +79,17 @@ func (p *Proj) String() string { return id } +type coveringBbox struct { + Xmin []string + Ymin []string + Xmax []string + Ymax []string +} + +type Covering struct { + Bbox coveringBbox +} + type GeometryColumn struct { Encoding string `json:"encoding"` GeometryType any `json:"geometry_type,omitempty"` @@ -88,6 +99,7 @@ type GeometryColumn struct { Orientation string `json:"orientation,omitempty"` Bounds []float64 `json:"bbox,omitempty"` Epoch float64 `json:"epoch,omitempty"` + Covering *Covering `json:"covering,omitempty"` } func (g *GeometryColumn) clone() *GeometryColumn { diff --git a/internal/geoparquet/recordreader.go b/internal/geoparquet/recordreader.go index 78979c8..e910fa9 100644 --- a/internal/geoparquet/recordreader.go +++ b/internal/geoparquet/recordreader.go @@ -3,6 +3,8 @@ package geoparquet import ( "context" "errors" + "fmt" + "slices" "github.com/apache/arrow/go/v16/arrow" "github.com/apache/arrow/go/v16/arrow/memory" @@ -17,10 +19,12 @@ const ( ) type ReaderConfig struct { - BatchSize int - Reader parquet.ReaderAtSeeker - File *file.Reader - Context context.Context + BatchSize int + Reader parquet.ReaderAtSeeker + File *file.Reader + Context context.Context + ExcludeColNames []string + IncludeColNames []string } type RecordReader struct { @@ -29,6 +33,68 @@ type RecordReader struct { recordReader pqarrow.RecordReader } +// A Set type based on map, to hold arrow column indices. +// Implements common Set methods such as Difference() and Contains(). +// To instantiate, use the constructor newIndicesSet() followed by either +// Add() if you want to build the Set sequentially or the convenience function +// FromColNames(). +type indicesSet map[int]struct{} + +func newIndicesSet(size int) *indicesSet { + var s indicesSet = make(map[int]struct{}, size) + return &s +} + +func (s *indicesSet) Add(col int) *indicesSet { + (*s)[col] = struct{}{} + return s +} + +func (s *indicesSet) FromColNames(cols []string, schema *arrow.Schema) *indicesSet { + for _, col := range cols { + if indicesForColumn := schema.FieldIndices(col); indicesForColumn != nil { + for _, colIdx := range indicesForColumn { + s.Add(colIdx) + } + } + } + return s +} + +func (s *indicesSet) Contains(col int) bool { + _, ok := (*s)[col] + return ok +} + +func (s *indicesSet) Difference(other *indicesSet) *indicesSet { + sSize := s.Size() + otherSize := s.Size() + var newSet *indicesSet + if sSize < otherSize { + newSet = newIndicesSet(otherSize - sSize) + } else { + newSet = newIndicesSet(sSize - otherSize) + } + for key := range *s { + if !other.Contains(key) { + newSet.Add(key) + } + } + return newSet +} + +func (s *indicesSet) Size() int { + return len(*s) +} + +func (s *indicesSet) List() []int { + keys := make([]int, 0, len(*s)) + for k := range *s { + keys = append(keys, k) + } + return keys +} + func NewRecordReader(config *ReaderConfig) (*RecordReader, error) { batchSize := config.BatchSize if batchSize == 0 { @@ -62,7 +128,46 @@ func NewRecordReader(config *ReaderConfig) (*RecordReader, error) { return nil, arrowErr } - recordReader, recordErr := arrowReader.GetRecordReader(ctx, nil, nil) + var recordReader pqarrow.RecordReader + var recordErr error + + excludeColNamesProvided := len(config.ExcludeColNames) > 0 + includeColNamesProvided := len(config.IncludeColNames) > 0 + if excludeColNamesProvided || includeColNamesProvided { + if excludeColNamesProvided == includeColNamesProvided { + return nil, errors.New("config must only contain one of ExcludeColNames/IncludeColNames") + } + + schema, schemaErr := arrowReader.Schema() + if schemaErr != nil { + return nil, schemaErr + } + + if excludeColNamesProvided { + if slices.Contains(config.ExcludeColNames, geoMetadata.PrimaryColumn) { + return nil, fmt.Errorf("can't exclude primary geometry column '%v'", geoMetadata.PrimaryColumn) + } + + // generate indices from col names and compute the indices to include + indicesToExclude := newIndicesSet(schema.NumFields()-len(config.ExcludeColNames)).FromColNames(config.ExcludeColNames, schema) + allIndices := newIndicesSet(schema.NumFields()) + for i := range schema.NumFields() { + allIndices.Add(i) + } + indices := allIndices.Difference(indicesToExclude) + recordReader, recordErr = arrowReader.GetRecordReader(ctx, indices.List(), nil) + } else { + if !slices.Contains(config.IncludeColNames, geoMetadata.PrimaryColumn) { + return nil, fmt.Errorf("column names must include primary geometry column '%v'", geoMetadata.PrimaryColumn) + } + + // generate indices from col names + indices := newIndicesSet(len(config.IncludeColNames)).FromColNames(config.IncludeColNames, schema) + recordReader, recordErr = arrowReader.GetRecordReader(ctx, indices.List(), nil) + } + } else { + recordReader, recordErr = arrowReader.GetRecordReader(ctx, nil, nil) + } if recordErr != nil { return nil, recordErr } @@ -87,6 +192,14 @@ func (r *RecordReader) Schema() *schema.Schema { return r.fileReader.MetaData().Schema } +func (r *RecordReader) ArrowSchema() *arrow.Schema { + return r.recordReader.Schema() +} + +func (r *RecordReader) NumRows() int64 { + return r.fileReader.NumRows() +} + func (r *RecordReader) Close() error { r.recordReader.Release() return r.fileReader.Close() diff --git a/internal/testdata/cases/example-v1.1.0-covering.parquet b/internal/testdata/cases/example-v1.1.0-covering.parquet new file mode 100644 index 0000000000000000000000000000000000000000..f8311806948b4adcf5ab57d8d4ea26cd909e7e58 GIT binary patch literal 28588 zcmZs@cRebX)#$6It?-{afB$1%@zG>i z{y(nLbNwy*&|f9CoLosDi~0{^`tSR3sDE`0?f>rIg=N|AzuwaB`|lm53)6oc^^sfIhx%E<)>z5@&rE}Q0y*?Z zVD_kNW|WeYF-1(Diu8Z9t)Vz2D`ni@0r;hiNl8!9rpwYX*%L;l8;U2SOc*)daBRl^ z7|Z`}Eodx?|ECGCxZEs zivO<_r(WdfvbG#O^8aXD@ZXJFDimn2#}JDIlF^71P5rNB~V zTNY=)xSI~^hN;@2yjqWp$Q>{Jb{Q|R!+U*a^R|u7fHqeCLbEnI%$U4yuVZuuEM9xo zN<+>b*LFg08XHRFX)rzG?{uaXW}xj^qlGeg zI~?h$q!l+U14o4^-`0!JEVr?b!X(xC;9n9J_ z_(A9CpffAK8e-PU+lDL7_<^DByrm^>#;9UyIlc6oA0*!yG&RM+6qh?RKbvdm4l#xZ9V7Cbah?SP8(G-XS65W+E=Br z^t2Y%uint1Ki3mj7UEC5JsQ~J#?w%W_JppKD_ej3QAIc&Z}%KMp;)J9i1%ex?0L{+ zc23_5@ZQx?{CE}Yywa>XdZrf?2e$4UU&=+niO(08Bhl>JukSxp!p^n6FPu*ho#`#| z4^%)w)`bT3`9wEOEo4vXi`h|5B_0J{us7CKV@0GKmc3LAWoXv*WZij0^Ir^FV0hXK zdN%&P7aZ0Hvs3Mczisyd???TjBBh-oTnZP)Rkf=cJdN9II{Svt{>s53}{?0Cgs{TeL`wkj3qp*S~fIl(Z{5 zEF$_L=de!K>JiX0dC=DEL?d*q*grLL@0KCB_R5Mc&+3MQigAtYbe0tg)|*G0pLGS+ zr=U$I2UsF&;oV!$`?&+Za%u6=u@-po)@R(J>kjC@q4#HtIVPt(R|K1%ATweHeK;Ped2(5En|1C2K?XpX63Y~-D z(Z?o*M|YEO_WinC;izCZYVVn{;7<~cJLxB^yc-OgT2|DI{+xuB!!@?sS%-k}&NxpC z(`2mgdpM*nIRtzQ4(6Y+O2+s{3AT^Vg+S5bgv~{QWNhBX(!OjOO2X=Mn}skLS;3XH zm=Ow{RhI1xE%*ND^ixDLLNwkx88!9Xz}Jag~P>uC_2ap4$X@nls&`6C>-C;MRs@*b{Wg76aSBe!n|yPONd$zh z5VgPAl!E3jD_VzJM8Icedx&=Nxjv0mEP(V847ZB#cdYgw)k@+zJ{IYS`K_t(-a;62zfG*Kl7hcBkq~hcLe0|~ zZhd=`(P8DlH`Wp%Z2qWw%)c!e|E_YD5bn|N8aL4DMz5L{oh*bsYo{J)%T31J!_@AK zn<|9H!X18N+>-GpX~dDGLRc6xY~DDNWc0BMTKX_w2vP3?pD*j5jNx}r?!Q_r1oxNI zo{Z)sW9wk9$Cr&lxFmDhu;E@3#*E2!HZ2#zQ!%SFGcO6v&uzGRipDRmHDr|Cpd@_z z8r@75Qu`}@=r2?w?%JF%cV&zaOxJ5Rb~PnpVPEB61Ji}zcCcmAiJghqw)Z*f)FdG! zx3OorB_?8P%&lv@@j{4~i!+n`Ou%~^4&Akt2|;d}apdxh1e7$Nyj%}LkT}yq^Ad2J z5x;kcyt5FBWVOz1m*cVK+`HWGv|yq6(%X{@?T8|9k- z`d=*#xRw-;bB2X&Rq7UiVDXwUY6s$Qzg2X>nvVjIjW14KGBplmoIi1`zXhO2s#@wA zhfFisw*(OPdeNN2A7b(P`|_FRzX~98x^q7Tr#S4V@Vx8&Kq2U*FJE>*DGr%#Zd(bV z?IL{|WAX2|jSwo<(q};|I{y6#p;R}lsYe=%`jbQYy*3fTc;;AY#NzMk1uxCnLfDxs z5m{V{!E>R*7x?rQ!h>hVCk7_OAbY-GOU6q9Fg8WQHyO_C{`7V!jeph1lC*^NGCX&* zesSLq0(kw{MHj1O_&L?m;ts9HLrKl)Tcc#CPAWseObE65zbkh2lcCsqthHzWjsMcm zQ9K11ju3Y>HeVG08Gctq52Nww*ra}!Ok5Hwg%bevqH|1t1&A*ALQwfH=WLecDOZ7*Wz zTKQq$m2{)Ae9_m4i#bA2d3j{7Ty+$pk!1VrBq1rKwLPIkOMgu8$S@EBYwS$U+3+ab zG&oRUsXz#GeK~r*>?kxCG_xSVLI_(AtT-i9kHS-4?h~JVq4k_KW?1%3DZcd6x_;!M z0AkcCx2|3x#f_#3j^Dluz%G5_&KK!Y{5jTago}d^mOi}Oo0%fTq{QNB6AFaTFNlnX zzEZ5JBJE8K?!cJV+aySa_FKbILWn%Lv+RRVf@)&p1&JX-P~CF;lvacUNBxM7s<5W< zrLzatPm^HCyFp1)6oim(XxhJAAi+HwS3DU0TmY)fM#&`TOP0OXIRR95rUoeAlwc!C z)2IpooF+?Mb(j?Q2}V~;E)zg^wK&=cUV=p09M*v`@HwM6u*Dy(_L*ZfZf6YSGo+M5Jdj-q*}_Fbrwomqz1+z?Gl-j*oyhzT-Cp)<)x6l8=LU5pe&|oT=@^e=g6I-i<2^ z2i-qjPfmwNqeiplEWPw_m^Znh!_KHN`&+#xh=%g%}OXtIxe#8wStn#qKh(7<1j$Jl@ea9ESh8y;QwQjD2Rg z75&Tzhe=!83l~eps61FzZ2Tr1&U&u)nvgHXd^#U}QX*gwo$s6T#AwYNc*{uG$5`93 z;(tD5U?fas7Pwo4#IEZ%n?%A7##`GVLf>f@*4t`Df)VjQP88vcAMSaVTqEH|9H|U< z(LaVzj)X1tu|d+;LR2Yo+3D330bS0M7QE^ZqCCm-CCeh9v*Al#>qsHqr*7}fkAUTj z%FGmE1LOVaMS!i*;NBdO5U+TjI=iqa96E=VReseHA~T%@G{1LCzwB7`Qh-_`U0O^B zg@YBlrT2E9QUPXuXmc5!N(=C7*8RFj0p^%L%o+bK9KM(Yq|3|%xS(7uxYs%Yl8y#* z`@D$68x_y>x7CM3bC$j|93nBRYg1>cMmU6QC^kR6KLS7cT+X;LB@8Y^?wIEOJOcTG zpW5Sxguw?!@r;PT`!UDf_&y1R$IG8Yyle?aL(_p0ha4Fhd@i> zyWa&Jp}5erNaw6t2#g+eJ2uHF45u>fJ`RTYXK07Z!f@d6($t1S!EnG7!iT1XBWv08 zAMPc=u#|CoI1#wUEkSeO;9#&ntn#$-UIdO2X3P##3WkLq!ACRRMq&U70B^HkxHfQg z0YuY!-gq%;h>wa#ZJxADh#Aa&O%M8~k*$bkKIVKF$Z%A( z07qjjO2zM$1XxCVpqRvmx|O7Ig9P|xV&{5QMLwvQ z%n2IgD8R&CjT)U9LGa{ZX1uF`04-bwcg!;gf?CFsULA?#%sH1o2!y6#pFVCq9f56Y zw2}wE4+J$AkK-cW2s9=ICO0z(teAngMPNjLb?H#kpnnGW^KhKAv2DlYPl0gf(cV_w zxe<82D&X+PszAtQG)HA5b}{a(G!XQelRH3wdzCew{4@##MaBlD^|i=2u6_P&06bOK zZ+$kO70YeyRGhlr4eqY&LZ}Q-EQ-Q%XI4 z)ZI`;f%rm;07;Fms#q2Xs*GjwL4Z|TSMUCGI}koOEPS?_=yi|!PMuy71iiznRM-QA z$d`;&&edHVc zsT%r1)G5TrFIVm7&I|-88AQHOBAhOIbM1X`AWX<258#Lh>&;7!4j_6P$+Lv(BD|MH zwnG-R&wRASxWt4+H#ZR2ymIB9I72a>o|$tmzE2=*oj4UhHB`Z4ZE!AERcu zjTNJF$H}L2h(5EgTSaG&7~8KIzwJZx$m_fVKhKM?;oyco`;G-bJ~L1xiAavq(#-)d zMgCULOffRkojWA}bbpPSWza|xJ!|llg<}Fhufb=bt_drv<(0~&xvcB$)cz@ zshV&~Hvrln@D26XOVFvJ_* zC1~#0t^eewKk%7-IVVB0EWT2}zthqnL1xjFh6jLrIhVAV7u%?J9eY{;lsEq}x=iZK zF0^8#PHO-pr?|h|uu6i+NLSZDnBz4u#A0{UP$m_%Fan&ctJ31d~H~tZ~skqz8d_rQ(lUBiVd&8 zB#1(-`BSwOrD*8cy`jt}2)K-WtRY3#g^(2@ry$6SnzrY#xfCt84{Y~$4T6=&7M#0i zEXA)mvnoFN2LVf4N9mzGwWnUTq*NFLWglYpsk+kpca7gVj}LLEMU#b5 z*m-%+zy*Eyko8yUL}3rDaS)pif0_DF)W5wq=7SZvm!G=G++|{Ls(iS`EW1TCmN9OG z8z0WhEVD_@ipH#z$Pquo`LOsS9j{f<=sDo+B{IDJy-(&-w-3~g<^!Jz5gv?2L!b9) z7x(euX~fIqU#-ztaAK40Fru%$?zFc(AB{d8ygU_H{L49}IzomGi~IIyQmA3Q4~t_J zV_c>JhPitCH05GZJET1O#|%C=@jAGirdUlFz$#W&;mnTHP?48$AYMa7yNu+9hFpS% zih2VX4u5=fROIbgEOVBsmx`)!zkxj?nRE!(hZj=*M}kw^55#s zyq942{lV{~WHfMnT#{KD(^&l#w}rvB_hO{T3Z^=WfWBF3t`hbgg-(%FUPU1}(OF)F zYnYHDqD#f)nsf;^<q)-{L1u^fV5Z zFz=jlp}&TzYni`;T;=l=(BL6+y?B_sp-Uypz(<{@uTx*KlDrFb2}+|$_bNrg-b?m| zQ+|k%ESKmw`$&)%eqr+;k>uF0fxE;uQnN^__KFx8i!zcXc~$?+CG}Tg9NtMGo+}X$ z^XG+KQeO%FH8Jl-0B?+v%}^}~a>%9D>k|oHc`S#;te*VDVT;p;XGWv+P;OxT2@xc% z{&-$#VKlD#8Nq(JMFd`jV>b0ZkH$|$tmnITh=8|L&tjpI4390{Gc@s-2&N7Z4=+%Y zVXLSn%J!BB&i1pvG5&Bg-hL)0m-A8tom##bE2a|<7NqU{c2xw4!(8%M8jF1-Jt>;o zIO|VOjK(LsS8onV5ks>l+xzwMXmq=?)N)*s7!K$^Tz^+dhL3K3ct0;q3>RMQ%A1Jp^3@dy$I$wAajd4x>JI+>!A+dMXVv|qNs6Y66cg<`uY_JhezAl!b z@|3h;d6UF&Vz_l&?|d0vu;I@SpDTvxI$MUNk|!aDK@9V@oF6l;R)*7@W*uxh zB8H?cp5rR13?E*Xj_^JthOKTcOb&!MQIq1av$~!|ZBmTF;hvwD3|cD!^Oh#3_jyrh zq11HucclnS?_Cd292AY;X1ZV7a7YBJ-#lAA#6B8db}X$uMQhx)Zm4T7xxo#3*K?+Q zCBE(2ne9{0SA!6{4R8wgoH>%JmzuGTQ41Fe#8GJ0A+RJ%&eUH5uR3GyW)IX2HhNDWJ z*N9Fk(aS8_8->^GXE`-f`??#>zvZ2XLQmJmsz#0&w#i>wv}r;V?$x{A?em#<4Lc{?`bmmwx;^teo{B)9yJfF`xfElDKKu3Xfe4;1AL_cmL5g?t ztxlip7J;F{ma(pN5{zVb4|n(=g7+R1Z@cj&I6VE(i+yiIAo)GYcaNL|M{pLrU2;PN zV|qSYB$1nA)mA@8op`jUxIsTmUxHU;5ByY#hd1Z;zu7QVfJ~bX^ zPHrx^H%1IL6HKa-Y~!&t<4~qQ@!ru(#J3L9SUoB7n;SOlIv$6Gweg>(91ug6((&(k zqv9}|(|u&#Rxt=izZ{=gABW-HmEV+zi(miYSMD>8$GwNz2Mi*bGuO$X)+ioFYy9GS zREweDN%9a@XB@uFFAdE&EC%__e@{s7z&=JK`Ys~se6gxF7zW*bu3@Pf zhB+5H-o1+shP?VS;~$O;Lx(-@j(&T=2ae>2pHM9fc}{n^!v_X~0;ltCOk*gPoMc^W z`N9W{=dpSIFGI1osIi~L1wNS0T!#B6hhoIEm=h|sq`o$4?yB(!#R&(kYJJY}fhRX@ z`JcuRy!2*u%acufu&xp4tbZDUQ`%JToB5J0-5HzLcTos>t?~J&DC0wkYxvKDMImTA z{d2FrIv*yF(SP163PIf;+isVh34#Z+FATj?5`qV3+NHh_ldb&PUj5|h5F8!x%hoh9 z2<+IMaW6DOvE`2O+wjaFu)b?J+u%nCdcIoS(ziVbrY4LvXfX)IzHa;LLOO$B>uT?k zumhoZ&{3l(C_f1H96mFC_aB;{sA6Qp#2{E$pRyxxODGN&{Ib2jF9=R;*0j+z48vK= zYCOJBdsb7A>Q!wgU#T$Wgs_!-k%79c+|A*5_(oT;ds`4R28RFKcPJb!st<)5eI#Re z%=Sf}M~35NgY!S^4f!zZUhu;$8^Td#sMpOev>x#%Q+Aj;gk$ouetid)(Rz*EzUbxd zF#PNrnb{K;4CC%R*C|3Id8IaUzXW$HR?JYPFwj@6!>E}p z!6_{t-W%!*;PJukve9W0%h+ zdyz2s(~1W<-V$scc;!&_!AMBf6Mx~5U1&0>fA#2VkzlQ5>#< zKZS+_osERg^BOj+SS`hbqtmrQDNMCiM`vOB5(?X_|F(07lK_70EcjejAVsZTW@h}c z0yy|>#RE}@6kqJy_EBY&0PG*^d^yfbilIyAUU^Mnv0pi3lJ@IM(Jx+Pa3NX%E8Oka z&f%bZ$n@LwJo1csDTj&Km-?E^}`)>aAO)+K)DlJDd}6Wumn2x@AJYd9wa=vmfv zrz=zlZ;$*kn(rvYQ?1ftTDC%XrEb0M{CWXijyseY#-X6T_{*F{#ZQ{hd6y+;db%7I3>ibx6(A0Pl$vaPh;{`X#e^W zGj`K>npb(X)_)bE@6u;a6gwiJv>|zji%f*k^FyCzb14V$c7;LaArU&?ulzP~hyZ#| zRmf5gim)mA(2M6ZztWcBk$Jgd^y!m)wT>qMyU=+%R#sEYj}GS!Qvq!4oTZ#wD8`{D z2VH6v(|&Y+&K$xPqqpN{-W!Pkl*_$>bEk({W$N`xak?54V06~M@y-DUnGL^$l=wvTh)3ZVF7+oJbk z5#GPNv^JmkC)JXD@+*yB`GK6=49aQvyH1SRG+c~VwmHW5QI6&VYs;|BJTb;Z`Y2Bf z7Q(pb@BZl`G0tV1n*k-IruX%Ietw}C65WDX99k16GCzM z(1MPFD4d&J8tQdP2%T3eWJZgl(7P&rVt+cn6UL2s*b^6pR?9+7gZX5P#F$hW>PF!{ z?-gfrQpsN+G5fRSkJ}?hX~f^pRI;-wu=D zP;-9JXE_mY|Cjt>%kjnxCMcNNN#&kRkLxwwjHrypUv=$~n@dEH|E00$k#018tZToV zF;4`SbDYLiRz~4B+gS&%PZB}1x%dmPqOs%3hxa!h z0j|rUG2h4T=W@0P+;>%}jkptynJc>Pe4@czNilh3*++)!HlCb5)I$WzduN3YVaxD7 z4gPnW2y$0OsUM>RlfJO#ru}5{J}Oy{Z~C;sA^ zL)-*>efCjuShg7E`-s0x(@4M(M)@X&u42d+rS`cG0K=8`eG_L~Tnx!18Y+IYbOdN%3iPe`p_ zos2Ejlll&p4+T-L#j8J)DPLQ6C_H{ZD8xEnj$So08S&1ZfeO<@;O6&~BS%LjWAX3Y z9Br)-c#uAF<+O3hSn@l!JDO-E#ha?lbCa>`Pj2@kVF)xHJ-T4+vSdtt5E{}}8UhNk z_em$KlCk#uhCVwFhCtr<+mfBzC>w5@>Cn?10;?5ot~tFW87*Z^r%q{vGOO!woYk}G zQr5g3Gm~Pm!Fs14{F)f-O9Pr7K8nHC@9OHS&WpjjX@A;frx;91c+lidW~k1#^R{>7 zVor(`|Qc&VSI??A3pjzI|*g!JY1X22kpIj zRR+_NuzlPtU8OaASf0Wz%s8Bc^$vHr9Zh_=)zyNMZ0J`$hs(S42D3B+5JS6A~!s2W!XgW?26S_QkTXiVV!^XhPB`LaN_>{`)>%a z(XwRJL$_Tz-W}kq)3}z`p1a$<4P`{LbolN}6NCX~CG*v$lwHcH|uTVoJzVlDX+tOQ< zNI&&B@A=t8WZ4rUq=Afz3JvayMEuD!Hti9Sk<#0o`E2}@B>Z}M`kL}DLiqWIK1oEI z?8`W)CW7mfcG}^Xgf*ozHs$-!$QfYB&qVBH!kVMWLu8N)v@B1V@8u*1SZo&Cz)r0v zPLr6xuw&lB{N7j$?f3eR9{HEkm$k1Re=HXJQug;MOAI^SUky(!i2aw=|1E-4jj`zl zxv~FHEaYDvUqW zJ1qiv-b}TU+ywj-5;M>Bk_gT)U=Pg%{F|%gieWVWUfRps@u>DkPl3dX+B+S-Sv*e8 zd~>+Jq8L8BqXfpmIOP2{>5-Z}5y6MFiwU9rarmu-GDtL+?-}a`UE36gyUtT&e2^F{ z-zcB38ySa360-JOb0C+I#;9i!hrSe3%N`;I1-Z+|=PSoy0mc5hsC&Z>7XjNjH*YAd z>n(ztjH`;pb|r4ieSZ<`n!5VMshC)t%HGi9H$()*U9-Ngwx@M`MA7nYQpHRfUq2R| zXxU3o3SqsVq}XqN3{GJ3q??2=he;B7#bBpp;lw2~g}`Ayw+V>BawaQ2l9-5hU8l!y zPYiY&>E8Dtyh{7&jg!94jlo?%iGhQJP|hSB*2Un;_p}OjLMUU}f2Enx+qX9EY5MnkOPlQrm%Y z1^p%Q*pgEg9B52^-!j{AWKulV4cYXpkgz4$?h5fJV}rktU<^}irgimAKousq79!L% z)#OSO&|ht(cF$=c2nZWtJ0t-O38?8ihYW}g{sxX#_ zp#8|jm-`pSWAZ%Ln`eVaj4(-^9r1W?{M8@LRwBq@((8KhDA}qMXhuc>!zFG_aacZo zZtOKZ3g-|A<3&*%>V;=o%%=NU^G}+OhB*95IolFTx?f66%Z11kngGuzv}-wAo= zw8*-tj8QDMF{D5j6Y<{|gY$U0gA|-e>~*p0@qX%XGq%R!+#{R}k>>9~TtO^a z@jmOYveXW0k%w*=i&J=g^-N<$@HcZUi&e~3u%p1B1=o_`MrzXsDI`#90+vQDOIi1o z5{FeeRJZ4_tr#*%dJG&Z!zF~{8?7RS&mXrMYvjssJOkN!CxZUWQClcOT3QN{(_RsB z?DR(&UZbrV+aLl4*t6I*1|umHzq3gMgDDE_K0F2!nY=cIe;Htoab*nlODYf3pitj8 zO4Hic#E>iAaA%>081_z{(HNZ+gG@eu(LnLPeClr*5-@FZf|VFH_9IDHBf}vKw8l*Q zH$-oXERFju8Z;m)CM`+k<0`^dmzqmJ=^Ftmu2Teg27zGRB@niN4%v!mv?e&$A}@)i zwprKSXx#CGRpSyafj4H0ueg)3D^~9sSSXM{>n~ow%$6wJvfP~=ZY%kx7UH9D%%1#X z<%SXn@aBG$qDDZLsRSM~D)N*R&73)}#F{dbo*iV;GiaivH^eY%R_zbgMJYbtWA=3R zX)zQQZL#t5jY2|be@j^|{)d`85QRRU3GlaA49E8*)eWL}t|qCeZ)6|c6WCvRI5iqO zDOapgCx&xr1SX#njVEdd&Tvi)r405xI2uPWaGhIXn7Bfl6LlgQiz^7<(IJNVqKKl0 z9N9l){u?obG1=9jw8_h(UuY{yzzf*U349+~z<_zu#%0f0Dn zg{=}$yD?bdf&_h@bBk8egYUQ%lu_i3G~gVO1a~L|Qe?=~&_C#u1R^QpwCS1{*DsW4 z$FPhX5fkHi#ldfm)kYSa$@n8B9X0 z1e^n3*N*xSf#FptHfxA!2wRt|A;569Ldz5hpyFHw*E#_PG1!zb68Ou>uP7eyl|)Ul z1Olg0?mUrF+zfPaqceLJVM7(Zb%2BSc<~KHu>pu`&Z? zh!o+%I#Q=y;sq8xaRnmM0=#qc?~36sgv$NDpgz~d(Er^y)rXHnxP*ZkT@*v-9Kx!R z+nlqCpsJ_DV6d%=Z+lyW?nPvPREc5HaEGNoDKAxIK}^ z_!k4`BZeQwlsc#OHZg@1loSwE-Qb%Lw26Sl8U4lZWiMf9R7Z;_9HP`7Y ztde3teGW~Y;rp^*QY1y(cwvngVzWg`M_Z-n#Ix1s(g9n-GjvdpNl|ULO;7MiF-YyK zw+{@I;!Z!Rs<eI!4bk_&gN}d|kZP^8iSl2(9g9~pvdK^MG6LH63w$V$pR1pFGqkQj7-zX}zBb z;4f0lQH&KyguPW1LbTswgV1jxOk)6FYC_Pwed&w%oCtduHO3MG71?}R-z@s4zUY1G z|8H>*FHEIpQ<>^>-$E4`&N>}Z8&7b{9VYkJjei-93(x9gJPi~=!?$}kyj5jbIV>|~ zJlS)ZhoXvSXUcGO-tq+3$ z4j(GyIOBU?MPO>xO{aG?e4yeFmh+1UR3})}xW#<8js<;XTk*IdnwaYmO z&Q4CfxriP45AICxgb0<1gNq3p*1#sbDkli`_g3|PH8v78MTA!Q5eVxTjGQ8go-O_P z4nYu1SU^J^0qSONZ`4l+0Bn0Wb_c%_-Jp1RLA@v1gaQ&!^CMlTq{8O`^7 zVxmB>0|z`IdcVqjX0Q?d-<8OZn>wKd(AfI%<*!mHuq{W=ae_QJTd3J z?9ZBOEoZ^1Fe9b?g2T#knQh@!ZK3`TOk;k)LV;j54o@st z0(Ut}FB40(2t{i@%d>B-(=5GaTY6PndNo;kJ+ajEQZ$#d8VS}WR-A3^E`C;CX;xmd ztym`R)m8!Z)=gF%)@+w2R%%C;Sv$<-te>rBtuQyS_VTm#O0!m*uVOvh+N;`{^U1-b z$=d6QwU^uwC})o4aRJ)3KJ%{041{)`o|#P_Au=%WN~=Jqb|F??HGW&Vh`?TN6znVx5?G1>9TJ%6BAd-6vVb?NKi4_{ z{CQgaIh%xFs>=FhbI>PPv(cob-ow2G;ULh<3c`JZ`N& zOl6mF)kKKwWd3G<=(%(!SId;1v3DkoQjx6W^lx1H<{VG~>0@+i8V#mKRx z{=nt=DRX}cv6C(1((PTmAM z{TPx{;0Ip`5_s|iHO`z%Dn0Dkyw|Ypq6kZPaW4E#e$Zm&zWf5IbXOX~rIUWZey37I z(6VBA#V)EdOzP+iT`Cdy!a*nZGEbo^19 zpVXL9LCZQ{Fl3O#eZ*Kx$fjGBzR<~WQgtD7sy&X-K3{x+%h-ZsKEQr*sx19rOI=T) zrWoy*LWC85U^Awg-AK4!K7$;Jqbf$G(15H789Dg1N`B-m^9|!kt#s<#<<=YcL0<3I zZRV#L@&PQn5oAE?>#VbYx0V(dBG zzir}YfMnyGf+EU!xDrSY-U5V8%Hce2BGBC%szf;nP_9P>6>VZ-A(EUc0iNoUwKPzI zU+3j6a_|Aj=~lZDH(WxQYBJP008YQBHP4kw{yF@4-cUdaIxUHSX$&m)s25~0UXw(E zbqtj^6_5j4qIf}Ily{#)PydM+6UcM7lRK0!-k1WlXSdc``-&TMI&8a;s!jJ#+i-*sZ*fA&(-EuyMfb8ZG~Ji!$Q6931x6&t% zbJShar@zf^ddsB`RcDQWZl5lj!-Se*=JL zsa_C9Vavs#1k<-8^svAS40lYGut~F&Nxmn1vh;!irYbF#p3iRH{PL+Mbn}8Vdnza= zlQ-wuEUjr&(58iSPub z4B^>=C~Ri}hXypCepH2BPjn=?X6ByIX>r3#pA=p(Q}MOi6AJcHp~>KA{EG%&Ohu+l zpdo@BG)C;SctQ??cAi8IUjDWTb%#Bns-J@8u4t=DZjmQA@pvjW1h|4*%6B!7djN|! zTiwQ-j3%4pK1U)vz>sav<;&1;=2y=35!;ayEz>Mk(gKEzPvSf;R3U%(VrJYOQ zBNd*a635wH2Y9WDT-yCG-U3BVip&aG9YQ-Q>#6i{RTN5FL-g|2=vSj+@!^*kV6dBGGL{TM7`ioy4LK-urwY-SpP<7Un z0dZJH@uYRp-lQZRs&aklK5vkB3DLcih)+(%p`q}&bDh*1TBh7MKSvyol}!1FJb*mr z9E?ZpO?{F*NKgw#eTE@>D8BbI+|oZDh}(?xdLOg@#A6S0Y$+pbI5x<2C_e#Pn3`@X z>2@7mQ_~}&fLrj;x>s`nOqsgEsTA=meYR`PWdO_VXCi_s60j>$QU4~@RJ)4a^#)WW zAS`W?Es^tqzW~j%3Fx{g=fw}IfuEB(;ETpRYEO0C^9mauxYy6{&;qSQET(EKcQ+p} zonhB=&ms{^evSO{G~5T6xLNfuqM16pl|ImGb9wf$TuM4H;zylo$Wzy!RIE>|>*$?s;zD>mX33b0O7y3ctgJ16xzfg9- zmUPl%KX7`q=6j(IC0!WkDAlx6_$7Z_|0K+-8+3T+a#HE7O}@L8DG5W>UA=$&L4gTC z8dK7QDF;do0M7lytJ;c5$YScq?gW5#oWeK(I|)lt8@Akt2n5U9`&3K5C8DJxG0^Ei zU_5ed`6R4mYUalT!5mUrBI>{897P!Kk(te6!H*%-AB$DIPK(dbtP!S3NRB~qg&hH7 zNs8||N9)6&tmO&ndSYb6nk}?G&J@-v4T8UAxT_LTZN4sN4~Z5f1zH?6RN!dwHD~CO zt$V=Ecmt0YWuP}=@j=OhoHSIELC+3#Cs<&$%zs#B8X8i)nagiS81iF}!tazcO!)2h z{morRxXz;zq?j~(%Y>$8IRcfHWvq)$!$?y0N)sIadHwYMBnQ`!L{CRJq)R{Eke`Mp zm=)Kc_es3TPfA1WZqjV890*J>TJhVoG~7r9luiR2;csOn(M;J5MmfS7rXF@y8s6tp zX#0R8xRM2PsyGdK%>#P;^4c9?3iERVQ_|4+P|ogQ-yLB#wVzE5RGZMRNZ2^RE=G~l z{eJXo2bum(;6U!>;KDR~%2Wn;J28nHXMJ9}3hN#3iV9~&nx;%WtjP)Nr}~F2pnhLb zQ0H!q6BO{AwK$8@G=J*!h?YA+r2{G8kTJ3GlU^In(bK+ncuBD>7LXop(AFZ&%X;?yy z?(=hoaE;QCSM_Pwhc|zOe<0C0UVD_!q@nb4g6hvmXRG;HEor!j!eI&vo&ObZw$gG> zq*~XNG=#qs(X*X8luqn{*J)@|L8iuV7w|RLaqCaC;x4_|^dB^}VFo=OH{TK`?ISPo zu`@iR`i~9YXi3M-o!p?}0wc~9cYpelhFhpYEyctI`ZUU`seepEsfX62!DGz8Aem1YqjYLA{>vOgWW_xg-4>ZKB}72!5R zn$vL$ismfq3G)H#_IALa2;koRi$OG6(=#*L zF%E?mY52L>%Cg_jFtC$5+oYzJhS$=PzovW)1Dk8ML3221xUzE8R_?=cM1a0&;EqNfjEiPM$c3%x$oX^z&!$nrY-qc zo0f_NKAdci{0Kt+JIon6KNagwi{_X{MSzUw_IpKfDynRkN8eFEcL#8bkem{v`aQ$WVAON=3ZV_*ZnyaEt^Z7@4f zQ}Y$GsMb^7+WW2NtpIfjDwzN<&fH>P9x zwg=n2fG-UmM?LB>bRMnM}frfyz(w1Zs7TuWADHcrTGsyt{d~YVn zd?fRkWWEsMypPPhWQPZg>Xt z_3)8(<==J@Z*17x+x;jl?L7XF`r)jLcnkD3{}s@m*tGGO+eMTg+%@&SqmWkyCzT(E z=Z9wt|_iox@04;P;>KT<3=qXWvwI3nOh_dFPA^&|ajwA-#`e zq-jfsHgFD|Rqa;mgL`e>~Ykbp6HqKKBsF`+jDy!sj8@9$!68 zwFc;~+r4u48V~XNL%TZuu>kA0*z;c1Mh|h}@lUsFkUf|_4sKlOA&$I%OCex~_UVjf zF&wmQFT1C4h7RoK>#vzYyFBnT>>qn}Rs+A@Rj*(B4z!;Yw?272Bk_wr?6SbwMU6#2 zajlk-Hklr1-CyG+0tNN`?*jfZvvyfQ?fsZB3DRp})zM0Z@fYzeGK`DgZn(7*W~%-8P$`@D7X`aSDs zdWka!uRFit2#n`x@yr}Ap^tU7O@Ums;*PK8x3zkSHv-n@S5080@^<3yL7monQUrqr1FB zwy1`fBkrHqmNj8^MijRz#pc)*z)8wIQG-|+|-v} zhj=snaQFNJ&-sYYA6|Ujk6_<+?~{ff{|tcS1Yt zg&w)}-0CfGbmGX_-yU87{x|uyHRpEvh%L^+#XHMs>BQde{LA&v`-po!s5_{;3FO>d)Yb#;`_{n)z6%UqY}UPPHyv0(v-C2+Vy*$ zy$|H`EoY}UQc`^Tl$*E0+2{F#$95&Jqok8hC(3WV9nN-U7rb=hOjPPP`^>BR;0J`S zU1Kiu?~6(Y7Cry1e+GT*JH6e63b>$(ElL5Gk?k^E-wn0^=!Bwq&l4|lc~%;{`>EWG z|K%aHZ>>%~L9^08!@r+8dSSbRh-p{`2A$G3bR{0wf3kQ2uhwFEEh)KtS z?_OKQN;A@*ZJj>oA+|rh_-EUzS?S4JUj6hN>pVpH!jp6Uq-CY|BnYOXG^=G{dwI7Gq%CA^KSl`XP2;28=R8=x6R`JeAS zv)w~nD(|I5@fualMQ=ZpWKG=_zUW=oKU(g4nvCxfL2YZnOU<%qncScrN*QxpH?#@bfvm# zO4VP=s#d5q8BN7_inZ3vg}SM#O0^0u)8Gv~n>@2}YKi`^8Zd_gWmQMZG-uVCyVWxr z$0*49#l*1oFrBsPs)yB@d%vW4a{{6>Mlsbdrc_cpru9Nu)vhwlo8_AGUqZBoOQw!Z zVq$4#BbG@~-uW`knhMRfNua*bWl)i6jfSemW}>KXXz4MF*WXrG{du2Sv-=ln&ERAZ z_V3x|Z@yyT#o10S86SA&CD6+l-(}fx z!U%Rh{FPes*e?;g(K!RL8x0*FI~2P{uv_z*TC?dWVmCTNAa=fbe!?#{FbE_vn^sc`5D@*HgY%0x(LfyRDI$gc4 zURyV}wl10C646w#hZgy40(U8~9`(x5G>lkUHv~rwR-i?O6Jm%~u-$@imHwVgG0CDK z3+uchUyK1m=yhfJ45zdgA<`+C>?}E9fxr+xP)NW?&?XFBctvApccr32n#*AJ9D`X# zvzfS<;xa`fWUIH+hC~CkB7tVB#=U$m$vu#ZV?UhqD>Bm92xydjN%7$@J*6B>+ z5!yH&(v3~>LSK=lO{VdXZfewzLm}f?M7_>nzGBh_t$F!!WI zoa;$M`@~dFw7?;YZ8S7B>kLiM#|bDTusyweG?QTk*`M|1=6b!UNv}7U8Vz8Aby1qm zrlK@9Q@tEDu0XGi20flUShSdePyyBHd(zmOu<@s}l;XHUGl=p?;F6Rh{A$P>c~OXQ znJjn+!lS4Zo8zLHQb_BN!vVvkSXzOx(R2pkW}!wt@-l{vrC7G^BF7r04x}ziJc{XD zL|j0X|Hq=f2x%=%i*j6Yi<#PfLCg#FSn3oTMUn(joWT&ZrIuA9LO|$&rI4kVS9_CL z3xx8pz3&;GIe>M|2#(p0M90`@#us4&G7@L374z7vNTgE8sl)NH4*5bI%fxU9uS@dq zcLq`jt0d+^#JwK)gK+4r)M_@=HyTa4MVK5Im265&l~GIU3h+??X1Q=UO<;n6k9*KS z^l4#i6x@SQ&?kq6;r|e4wB51Fb>`^5!wP^Xu~TH_?9Mz z5aDUTouu?e&cl(MtJ7HM=p(rfE6LGzXOarG71@BfkJ1-(R8Yo==mR+_7|_x}KgJ8| z&1oj6%hE+hqr;x&M)rF+;E@Pq0jED2GzpGAG3MZz4j<=8JFHG25;SKbAwS}wWkVJo z(`ZX`-8OTY4Kg`O#&a{lf;Jkkf43;9L7qgVXDRdZ0I3ZV#=_9jl&YY(c<|R?V4SY-kVSSvZf~{G(jpf5KVVV0G zp3*bLaL6J?gYK4Y-i7B*CE79(nPio>GLfK;i-ep5NWP$9 zy!1#hV0XqrrX*vP*D)IOfeoPbYSS^cc#cl+T2`K0TX&n=?%=S_b5Ji6*-F9H71sBL zv}v^F@U2yl&iY+E1R@q0xA;7npi_wtZB-?ku*R7=$f9}UN+Ja;bn0vw3 zK-O$HsKfGr|AK84`W>yvC;U8PZ0{7?y7bD|FA7b z!8Oo-mAZE2Jt3W0jxT{UW#jX--aMqYk#=74cR5z-1DR;RoCyb$Ihr@eSsRc2(%t7W zcv@Nq+(qO&e3nTBinKn^8%Y%65TkesU=b8ITO1~j)mU_Q(M4CEOXu!$!+MikIOy(U zLe6;DAlrwFGv=?xhCq&@TR=7hH1%d!KYtSDqKyR11En}4`@Z1Tg$2KfmVKOy1PkeC z$TC3b`TlS)FWW~bo|NNs<|2C>X@``}oQ>#tEfWMA#&M&K3ky!3N$_#77Zle~oOYpD zj%141_+ljJ1{*eKu{|aNaoXU+eN4~tgIPD73v?r-VOW1R-u1{mn0^d?~hZ5V68cJKx9-s{)O`3>dmQhq}@5Au)` z#{x9+;__rAyJmfm7(}eHw{4OU8 zp9zOzTnOSwFZ{>bEz$*V7TE7JlV}eHn=~s)w&1-QkG65P7}}PJXzw-$b<3MLu_4#c z01Zn^CkEn?^8O8N@~mkD?M0VX+(o0?7Sv@MHE#e~;QNe*hN!oxp#h=$TzIEwgWD26 ze23}Vu?}%NkSpQGtlf${WC3@D(N&fQw_zeA3A&B5xM{VP@U5h)WllpwGeBEgFcNHT zQ3Ipl3lhbG*BLk4n@ac&vgwNCjU%1oq52_t$uQblBf@C&3@@yo5%R*mm?VeSydO2m zPA?47cEJV9YsErZv8qPMD*`60r`^&^1^vCqPXU1U+pw^(x0Yo=NO+{9MWNR=_WY45 zElG$6z2PpfmG%L^pCiIWmfsKEBsr$OaZbWaTDlRw7Ris-w_9e7hvHj&ho@qLJ+FXr zKp@MM`av&PG`Msl`C*A*9@wsY!;4&6KbR!EM(6{n){=*KsOSQX2YrOmyZnlgH>;{^ wlhM?YEStHBP9|@Prcz?Qw)Ev7eAQL2t=H9TfuE1Q;GgOTGgK-s{GGo4170q8q5uE@ literal 0 HcmV?d00001 diff --git a/internal/testdata/cases/example-v1.1.0.parquet b/internal/testdata/cases/example-v1.1.0.parquet new file mode 100644 index 0000000000000000000000000000000000000000..c025304efa5c1975ddcf6bcad2e444fb4d842e40 GIT binary patch literal 29834 zcmb4rcR*87(r*F*0Ye0&3rKGw(gmapJ@ixvy-HC)uwuuKii#bvfuLf?E{ZyeBB&_# zuGmpgQL(o-S9jlj-|oIYUj86VZtl6~%$YOwHxc-W9JM%F0xdy=mWmcjgT=CTXVL4V z^#DJ5J9MmWK3j>)dYNO&R&ppT?V84XS~ItNub&o6OIu6dj_y*^Qq)p3&{g{9l|TP6 zxAgo0>98s7ubkt}qIc*i~A8hRs zIm^+OO8-Bj>)FwQSXy!{`lCi~EEdrb%mNkvEYNaUFZwJxV`y4IZki(f>z6e$HzQ4z z{)-wha&(TdU;a?$1G-&POZQ(3bYoff`o~?mihu8+TU5;du0&3LnVe;%f`JzI9}7{= z7*Q}fw_s?&-=(y$wcFX22RN2zkh(t_O4Lowm25e{CJjNBRE_ zp{w|J0J{Hw@6uJJe=Uwvkd{C6&lW9{(_SZM`LdVg0YzqA|7D#2y0T!c;D4^mOjn+s z6&x1I?xiG&_ha{hATgi$D8Mg@d6SC&L)Pd)^7OH;JbmRq=2_eOU-MMV9W^2)&H2wv z53sclvn|i7SiV$dX8NxI{_6w<|C*q+Qn40Gfz4_Cq|i%=HD6v?$?L@weR?j7HLrIF zn|c4w!|=~5x=QnzMbJx|4#R(bN*De&n>X+bv^f8eA(iaR!j$~X|9Kuhu`L&>T5jjk ziTKaK{vVI{uM2FyC8zcNu-rd)p>sRirJUBnW=;e%W43~V8oep8RN2-gSupat)9OL$ z4yd5fEhBR4i-7H>3mx#T;tbyA;aSkeDw=1}=73q_cJFeI&4LB1PTOe7JL0Mih}08j zfwK2%>G~Q6e06DNo=!#<#Pt5WQg4IRF0p#S2;~DRlQJaLHBG%rY8` z^*^dDuyjJ_)|R-N-dW(bbi{#%P$w+VseI*PmjzaurX6awPPpZS3%9>b7Ch{|bj$Px zM{GKhcxs$$7L+;l%UqJ?i1d7?5|-B5QU#h!_o&+)>EpA|{f zB2@2=E^%=5WTB(yz%Ul7zw`kXCsN*fzKaXp_bmL|h3y{zva{Ii9uxUD$4h4(SQD=7 z2Q1mQc7>1K$5Rz%Z!fsz4<*rsyEil|;r>nI&)7Wjhe=1jKR!B46$>JwWsg<`z@5#H z9p(3HU`72Y`LoFZfal({Hf_*Buu-^|oJDjW7}oc>9v(89^-6e%=!GXgS||3y!bSAn z7yyvPybSPWKCz)909fmiZO*kDq5FjgGySXR`!|rKbYGSbz4SQV&y9Gq;{n_B8-fZ}*#&=46gboSL7`vJQmzU%qdM-(ZgIMmY+z ziH80eZy362^(o`}lL1ij;LC02a~8-EZ?QY<765A93Mz_AED;K3_v?P)4^`irZ>*b3 z^fvW2i;$@`p}LLKPX#rOAS=4u&U5o<%XFbr2b)yIl8~Plbrtr0MDKrY?EPz zd_4n;UH1V>RHJt-yJm_G2cNk)rvX$&1!$^|HN|d`VXsBH03DuA^0K=oSbWj>YTiO$ zn6!f%IAFF3p4IcqOtD`G{D8FZTW}Sc|+YH&h06h`ly>Rtaj#bZ@9U;T5Zuu9XzyR?G3|O z-oUaFf8_1d!WIvnmP)KQbf#U}^!>Lw!qG&BXXp(jdffwjFREkr{U(buhCYCIFAw7< zs$s{aX7%CIe4r$_b=#;iE((r)I==*oX5V^s?|}+-tnz>EdW`67Ur}JN5(;w8HE7Nu zx@ppQ_E<&Cjd3aUD)xb0@$Onnqvf&ug>pDUv#zG<&nBAxyx&~olRnVB?$_PW2szA6 zcNqMp-3NRhw$4WT@5f6x)uY3P)AtY93CgYBAIGWF=WCrNditX?6+^m@w>R4hkKXeI z)|9>~>V@6MeH`RjZPT7EzT0zvZ9oU8BcNzp@xK!@Gho+l|LXm@t{v$BUCz;}JSJ+|8l!>4jwyWe;~j(geT zI}fZ7HV@8unlc#Pne_Uy>bMQAS(AF|C4Vr~$KOb6^Rh+FhNrsBsK8y1!P@E?fXOPUsB-F1aS2A`gJ z71`mM@po=!MZ1AOfxq6jWdN?awDj}S`oW-PT5CUrWrKn>ma&$n-GTKnWc~3z*2tQ7 z=jO9sp1`kKRC0KP6`sHO2{-C{0tT+_`O#vDscDZitRHzoPRem@UwsQaG=AXXjXyoX zdgnOTAF1Z3YQO72)?H7iTi*J5O0n7hrBkQ$+cf@)DY8PoC2i6k0=j*4mA|hr!SQOg zJx`5>fSQ8e(8C-4wLi39PW5cgAm_FNEF=2^iVhlNQ@&UyE;_kC8P%$X_T;il`0y&> z+32`q$+$Zt&st3M{*Is&KhG3gGX6;)#b0~~uXkVbH9G|za?7p@T|?pFhbED+dg;v#JRycH2TemZ` z{JXeD7}yjsS)E zO{-E?rsCbZ73=D6M!@vqq~odEQt_*v4sWJQB%J@y@n(BVDjJ)3-F7dC1n!NRLht*j zSkZQ7fAxk)SkAlt+v9C2?j|-|dlCs1&rjTo>yw71%s59PL6ez>eHzv>+HF)MXgFFd z=?qK56{$t;AI?NV?&8;zeACieuXYEsqQLCV*D1qF(r{woxa$RGQ4qdV)c$&X8d|=n zY#nSB1)rGhA==IF>SR`l05XR>h+aUnN&U#2#rZ-gtbFXsR^##=8`IcoJ$&{^!O4g_ z8EM$mt?p=%EQHS!KA)NBn1*{RZca+{6+*$iHnE{v8va~QO2k75wNI{jDE6eH)3Uy= zZ6!k3@In7bU|TBw;c|`;?$Y!c*AjPQmrssO6~fL{llHddr{a!58n;JI5<=tntpNod zsrZ8|;?Oc7%!?Z|d!$(^`ZUn0LX?7WYoY$UDdPTrLrU=ZndYhjUV~wLjPE z^Ex41kh!c~dp8B+3UXb|D}?Yw%qq(+OhL;tYcHRm`77uQ80OF~1>d|v53_kR{?hM; z^E61hHYCkj7AFMrHQJ4xP02W3QT1ovOd)veYZ-fNTQauodd502RtTwW?CBoK$=Diq z^9pa25EA7REMz~D@b23EckE?Cke_TCy(B9MCC$e#9s(gqT#3=VBphkN?-`)rDufBL zI@h*~iCBB)ZT>f6Sa^~2*0}MB_-W<6rE~OzV0}m?oN*-)51%9-<*NXCUoH!}l9Grs z2SsdB=@Nio!O8-Sy$QI-CboFx2LZ@Nm8345lz=kM?}XN00x%#`Ep<;oW|-WY0!Vl@ zf98Sr@p$!J#k8|u1du(&wU?4h0`^jR*7>fl5DYSxEZ(bL`|9J_aOh2NjTN;msxBRqpQ7Qb45xK{e6xt=zkFzEM$#G?o;iGIf#Q1syn5uO zk2NymDE1v;E9yh@zwl!iPf3PD#GQ@J zmjytM-(}H*SiC$UrPl=o5=iFQY>P#&s=TvW93kuECGwQ?N*%-+%y#L{OJTid@Sw#A~2@0L$5 zz6qg+3JqCeh*_%G$U8nhXlG#VbC`f{aCD+ z|0U{to)FYt9NZ;e6N6|X*>WpINJeQ@cR10~@1wo4jD)}%F^zLNG6vW84^~G_ zNpZJec;&cq0d&=fV||LG*t-IE9H|k&@W>Af#WSV&M=oaw0OovsFLzRk_x!pdsv`xk z%;CzXU00>}?Y&%AjiUf|2>V>>G>SnG1vePdW3Ya^i;Y5l1bCgyJuzP{2Jfmz^-lJR07hRQr}g(yJAdi%?{JtHnz3&o@r!nE z?54Zn@W;%76lai6c4SjHY^jZczBeSOw8(9tY<)Ov-*Gc#>|zPhZdm?U5)R+0viH7R zEx`ezv^*0spXS#eiCLu*97@|frX~z7eilF87$?D68EIV2FfceU@NC{UG0uCrW7>|& zP*9^-#i~_|9s`Rn#hwa+2XavZlf9B!qIjW!AvBf~F;SK$!spKxKLsTebZR5zss zM!*Orin0=;aYfCUx2q!H=^?SFOf1HHkCl(MDn`QKpSKoiR*O+?y2pecS&=YyQ~US@ zQZcIbR~MVUj)c?RD||*5iLr>zhg@0|^rQ2AeYO~FnFDVf4Z9g{J6`-Zh768|NeqL# zL`dSgW`kKYd}pGyts?ZFd~S`sPBfU1{NorAPW|p#c)>jyt|gGk@D%+O2-RrV=olX& zeI-P-32xhbnxdf7b?n@iH-xA_`h4NyDClVTT-Z8Pi1+BRcjrXG62@d^3$cNT{tTkP z-lTs|o=Aw7d{3O7Hz5)_29{TS(GeoEoVm2Vw~Ib+UH(FVIz!!B%=<-x4ZEf1mRy+t zv){M54NfNp{G5KTK3afzmJjkqy^Vy=W|2g1Y z<04SYwPN^j2nq(&d>)f>W~CHd{I;v3;O&wPU3 zX^k)#KI~R}ic17eV#a+G3Uf};4wpw@-=k&e4f{i3uQ@~x%!owR;wj%fOG9B16ZUYT zaHU6*cHjP?;CMjoN!8sbED&bRh)@ZId0wH1v))8w5Gep(i%_`IcSSM85s9`8{=YQMV|DpgHg+Jj0|F1h(O`GIC3z8I=`ZeVT*P_Q;zS}w#aX1}I{{I$q7L^Cf-KJ;ZIsz!ihSBEPv=EF7) zd87dXyuEBdPI91}!iV~0WO73U_<2mn8g*qpsF}?S>E|rKP!8Y!4_t+*cyO@lsu*mN=q+g9qN_J0=)8g5=k zMgCD}N(M}RS_s%M6Y+?`s36<2f#xB99r9<9ICEXw){7s5;r7E_t@^W~@N9L^fe+Qe zkjq$(s%Y$F!dYoB7&0ffj{tY6YCZm85)8_W4@&&C$~vlh_H+F9u1JViojW90yILcnAFY0o=Hj%C!SHkjJ$9M^BX}oNx&!EO1C<5hb1ecS zGrGKTaWJSep2>RwR&QFdfOeOr&#Kh;VS9zQ*QY5V2Fc6K)BSaa?Mug2AkM;B!%j5Ffo* zzK1(47^LJ7`NxQGisNyt|2V!LVt})WV4_V*GNxlQpS52wJ`mo9;0}jIKA1KbcAN zsoh;_dOO9~e#P{S9MMCs^7j5XE5?R>Yvpzy34$VKqDUH%0;feAf?%S;&F*PpWR^Q? zVi4&695&skku-Wv|4Z`O590V23KTR%@IdcfF9IDqE1gUAB zFV-%XATrw3Js4*CjEVEFkzncD{Je?VgF*7Lf8e{P5)_1{ep%ER3^zY7HB!AVLH81e zN5bF1U_5zheZg}Ho=@90_A@U8292tVko)>Cy5nX4Cmjk>EL3iI31%UbYAu?it1Lxh z@2<7wej&hR{9`RCvd)Dq6}f~!Va(*62P~y%y`^t^pnC``J2Lmoc~dEV$(vsJAut43 zx_T-P9BDkw@`Yu>5Ga2iw_Dwv?!ROD#&uK(6b(Eh#9S#p;l-FZtO)_xxu1TE*Gf@w zY~z_L%^~18`{N>uyVAe8#4jPR`^HrTXI*laZgjGmNxA&^&^Te7Fa|p=?(92Pjt@D1 zv`!3m6OTjKeE1{O`(ys@y(u4TD7^gGN$xI_gHz|jO@{4Ou~^Q85gvRvHLct(H76Ex z(xQj_h~&e94|Kei$D((iHy6n9`g=dQPhEacH=GZACPla}7LEPhWt`v5hbK`lQh&C_ zV)3!{`h$qR@~Xqp{%kDz-QX3f!GfQzaWzpgY*?V!txc(hHGV9PO`K`D8W`sr?ADf# zN8PZB-0xHQ;KIAXLmyU&sv2iTf|iCai zz+zdl)P2;{O?&n29?DddxPH8_ir*5P)V??FqenDI`v%_Y)h@wD=P6Dc6AhO=w^kY# zOYmZpVo2+#Xc)zu(eV;o`MZ3|_xNbAB2$}jN`gQ^OqymieBRfTvh*tor(wfZbt-Lt`Hop@iWmYiVSp@XX(r}lsZ!2|(qzYZ_zvmMl+Ay)(%@oL6eVp0fdnM;BdjFnhAYV^0{u?u}0`%9R6PPoxuWA#u}MM2!}=ME`~68z(0 z-iZQUfr{Ng9SL$Mq&AR?2A@Kf(*jm^QSzV#nS--qQMx}r_|P#Cq^$UGR%Ko+F8>k5 zez8#mKI044_dJWmj}utWc5D>^Z;^r3JQo=rS+sLt@(~eC8Xz8AtRcfzQEiOfmpotR9-&sg$O!y{Iix$AsH;r*!A_Y2$Bc66|%Gz_({6cv^R6soE#I2k9Vxt z5RxW_W^cCdt0l4MaeI;V$P_W`HGHt@~_^BpMiobB0H zezzEw`mb|6_c#_4ngX|;t`tLZ&-4XmA7jz5|JAPA8Ddy#CmwfIEJM|a8G{PPis9H` z+k~DuGCXI;pA$Jt3{&(r4oas;TDr%1m2j;XW^X)OFtSdDlU=6oYda{0lun-Wa;Xd- zT$K*--7khs9&Stxgf~W;^0Cu9pT?|Lj={m+pBDC8B?8NqCYN`GF=(aIbmv!<2+Z$Z z4N~qGi(jXCURk?e1S?)YT`|Bh7GK<0RCj`S+_rk4dk=-d4F*^9CVwHh?cA2@GCmdu zA6A)MpeY7}g$BElF?g$S^9Sy8 z5!@VfM!4yt6jye67kWJrfgyL}uD}W@#tnS>^TB-)JXtc(eXf%f?-bdbJl-V&W2KEF z-0LM6&F&iP^j-w-yvE$};7f3D=KklqUyDHUYncB|c?k~T%zd-)ng|NIKUt+vm}Ap+ zXr?B~=!B96!w5qOUXtApP$wDQkl*`y!z2j`H7k9EYeazEagLiVNbvg_QzgAxIs?Nm z)p^a9;_>$Z@Q?j+t#@tnd)}RP_`;r=uJ#0cU1h3tcCHvwSBVE5e_w&J)8WFY&=(`9z#}zTX9hcw#R*w8}9(5vPr7F1}kJ2D{N_ z)hYIg*qXIJJCJ1W@CA}vr^&4DG=&WfYquXw!0~m7A1Ce=LypSPZ-v7WFqhMHaP}rK z2#3EIm3=4yBfF};s*n`F`pvJ}ZJLO?_P6)xM>J=ai&LFRA`aL3$@i)eL-FI(0j!P$ zd{I;uo^?PB3LE~Mke!EMwN86`tyegX-e*(icZLr<`N>OuH-_Pb*DG2c zujhkptw3+hlQ5jvrhd=DpM2?#_(H|`Vd%5c?}M_852fyrKlV)sL;ES8dJHxBFs{Jx zS&t|T^}lbvRdy-_?$0d+NhFLSWMh-_nS^;keIPYeGm- z2<$v?YSfP3v_4Vg(1tM~Fz-;>*5Hld*kAC|{@(5oII%(7PTx2Jr!TJc`b^{5Ogd~( zy@g_>@o~q5trUy&)oqoiM74oF*FO_K5|5{CwRDQa)Wy9N`<4^GhHshwVn+mi@{i8$P6&mO zx1VX1B}bsW%A@A9wV}{et!+2yZ8*Nk-!`_hJrwS?jy$5XFC6Q4ie^Tf3x$8*f2m=Q z3tN)}W9cp4he=-~ctkkyn|z1>j$~h|%ibfw9mYErOmcdup0Ti}CBvR|mfi62bM9OV-WLM0lobXMRN=5%gP_y|H?s z2)8W?f1>+N2xpg7i)6D!xC0y}*dr%zhmJyilB*ZnOp@4Vd^E`&D+f11p3 z7UGFk=@A`!A-vSIU43?q052x&&yL_wQeSe(+F>w#Uolm~=aT>~EnN(m2cq%Bf=>Os zBLaBy_Gp9Z>nLm*;_$=npa8aJR(yM;8I1#6J}vZ{CjhUP&z3Lx7>%#;G6uc$r5tjF z*6Z1!0u&On0uv}F>G@)m^-clyKM=U}T7>{2Ph3^_#1f*gcjeS}%26KL5Pe%kS%~M} zM@_TH6F~pWpr*#t0(?Dl=j-lZ%3*eV%D!D7Ko8->2~%wZ@ar7EYO+Ltvm);|P0*0EAh9b6jiO&!x!^T&&E;PHMJTE(;{E7EkA^_D2pV0itA`I^wcTYb_0PD^b{kX71gf}*~_vslWfQopRk>e+caQPki z868XMuzc;+u-!(4L%bX&xm^~(&~07ifkQ+%Xy4`!v)%}xilFFo&%lbs8<*a{yY4Llv6F4T@`za6JJnO1<0yiYw+HueUmS}?ehxpD zutngxy;@_)?O4oS+IjmUP3BUX*+XkN8LnP;e9AyC5iIGM9yx$5!+SLOUkM_}UlyZz zgbGZC!rJSO<0$&5V(pJl{ zVwmG6{ybSL35S>znHjr_p-7xr+8UgMk>7W#beV_&T^|^%n4W|WE*zS9&PWV>_2P#w z{F;Qjb~)e5mKVb{{rFeEQ<5>CoY7TZMX=bjo~6~s3+>arUcYE``k@u6*itiAvA;q% zhV(1l%%RIB zk4(kVU-@0JM5`!YS8tw`isirayB-R|pz-kGxvLhZV(R_yu(q-=P?Eh%IbNNLb!XSg zZQB*Xby$d;Nu+*;}Wj#AAc) zHbLYSF*ue5H9dG3hppcMa-C_El# zd>WS&TOo#pOOGG7d;{!I(Tn_{nI>~;?d z8M#OVGVQ*r7iA>jm)hH#AI}$odyBbQwguJ8hx~Aj8!rMiBg)itB>pX5CySt*;0qRx zN!Y_+mZC&ZPhbem3rX0)$f>B*;yzEtA3Rg@ZV@>tJw@FDIw0toSU1AHV69LbTcLtbH0HxJqTG ztuYMa)d^u%t`-)LV;!%;Rzn58J*!u2rWO{M@-?IKM5u|I4$TZ52|BGUw`10st zg2yb2M@m=4QZ@}PFlG6?1k}*5jW{5J-#aqT1oILxg-RW}DE#B_Qglm4Bx3t}uR}v9 zT(65g>vU>dBJLqPrQS{v{8F}$x%xf<6(HiQ2!7n((Imsl{Uvyw5nF6dss>UrVWiL~Hq$wYuMx^$EEBEM>;~iNX4{>M@6*33xCm zXXh0s3K?mR24)H9PdT;R0b)>+zj$q)$!P_ z!i~EZD1z;iRy;ou7mt(JYr6vmh@hl%`nMI1#K(t}E$<>z%#`sBoDTyPLVIfEYR6OU?4zS&aE>z|G%Y)P)CQX-d!lF2k!(Vuy!O^={5_X#t4C#sxlo<3xE}i0{c?+Lh7DC?D0GGqJlJMrZpzh-o zFxVL=waV2d;UTJ>@9iss9A1P`cjGu(9+eF3$%EiB)%x>^SVo`(v z&UU|kI)u~+Q`Fg-h<8U_{@!dOf;^_YZjgwQO)9|_k;avMEUOFAW}^Q%tgX5et{+XxqRkEEr1v8FkO+VMOvo^UINZ<>Wj2u3 zqVZLqlOCs?re7LMN3-+l@{8JW`1Q7PxEy&iFZU48>Z}X{o>N&kM+89}u1tnSyodcb zg`zDwsgyEfG@mA^N|QL`6wHvJ-E`6pzN)EQQyJFsGz_?ABG|>YRlxC8G)l zaaYQ48}E*`-6bKs3;)@;<*E#udGD+PuM6P?P2oK&4$HFJ+kM{(d1rLUyQzv(F0nJF zL>H6sUl)h7dHVg7TuJS9u(W>iHahTr=y9_)#pA4loGg*{uYO!XJlgO+>9KM&_UTZB zZXAyjd5Q++@gn$Bxt7H%=PNl-V$h0fO>iTPDgBg^Xfy#!E1#vR|5Am+sv4-@eZXD} z*`z)Cj*#I(!to7P6T_zun@qLxWjKm~Y`qmhZ|11YlOeH|lH{~kgd98hL55dot41`4 zfC2U_aF4@iO2uz$5WX{fQqY>L7qw=SWgK=%%wxNG!|_M&NbghqOEJuxhod8erMIX zMM~hc#ez$o&U&P%bj%&dDWx%3R0Xwfw>44Yo}opoM{&vsfonQ>AK<0ovi3-FIY zLTGcc#?$qkOJ5nW?YjAKeu=UU)Dm7CWd`tX40EGZ_RX zpBRhBY6;G8Mhs;P_B}KfhcR%Sn_?KVRF@NTEEY>D3Ey!;42LE}O?beO{YB=#7DEJ6 zT^&f9yd?Iyu8IVFfbE*Z_ag=d&6YMUej0;4Ed=o!FVQ~as5F}x@S=2<1a^_S+Bzf# ze>HC5&zd8FIKsT?9+TqQoLG~mi-{fnfvHlwvC8h&u`Lq#0|`=-hJIGsBms?U{guv1 z(C-;{!ZP~eTW%#)6nR69I0q%6H0@gluA!Yz=FY8x{7c;BM=uPuyl0Wlhst{tsP`ofA6dWcpP3HiwghS zW1GITHA}!qo47MtfYT9ydI5$o*pvbZ{E_6Bln?kqswPze!IP+Vo=hcf z20BZ$#`k_o9|fq9ZwMX|SaoDYd7oq<1~H;&<>A}`O7@BD#8nSnAyi*Q~&nbS`3 zT&wPcVi8#Z-kCXf#PA0~<@pb&&s8z>emheA!9x))WS~ap#n3U6uxb=G=Pf6w>IpFz zZSLgT-x8tc1ad&C#V~)c)1n_#mzrQi=*jt%gyzjvq6dia+cg8@it%EYWnm<6TPeoP zff`54C{9v2e)Zz&Zh`{UZ5cW|QVh4f$II%Ti*YoUFdsf*@K9T#5HUfFe=u-=c5Gc?Z@EA^*qu)NcEN>=>+gv~|y0FEN}HzJF=(PKx63Lr&&~ zi{Z2vM=i#tgEL;N{nR+~g%sIK-y7*p5d#5}<0ny4?%j>d1ryT6+N%wfmP;|{P#!Iw zk^AzWQY1s%cy6T_;&VkRhg+rS!m~Hz(g9n@Gj>vvNl{~mU3chlF-RS3xAYB`;su_z(I}P(*2BjIzr!P@Of$y__AP^_g=0PZ;!M;Emf7kcEyY8*0&|O z0E!YYNST-wI#z;jpKaka80I@DQRec*d?h0biS}u`)MrxsK3=?7#Y6%R-)z4(mK@TT zbS9eIBHKA-F|DzDQcQW&z%&0w3?9Bc@8{4AI=?K=a>KvPJ$nvc8+Ex&gakBRS#VbX z+sofC9rjs-yI$#Z&2AOIm0PVH*1oiB{0%FG%>pQQ95>J;T8w*xiQi8J@CPa8EXK+d z!rm$iAvWNVQTSI8W-x#+4IyaXy6{(XEW5)Mw}1y$24<&_Tt{ ztcdW`&3$kD{JcPh6K$R~4Vo(in{89?YJ^cOvgUN73Z;UY0`1V8;7 zM;(xDPA=_OosNx_V{^FnXJJEgJuzo`MN*P|Z~;jz*2k?wzh7aC%(&_4(}R zzi?-QCq$`@>0d(Fum(2aRXHKBr>DC2%MsD2Eh4na_h4AfVC0lZ^=$0TcM5?}!U7uW z2~a6t+I1>%@ot;@KygN!pYj~U&tYBy7(^G5%Onov;eP);{IM~*hYFsh3 zZ8B9*V>MYmHuaG=+jzj%%*-di%qPQ)^~7_AnNN+GhN=puE_QAKXE@*gv6&LV4xHr8 zpSp1wC<1Sr8OMXK4A!0h!YRzsX6?7kFxR-j)y?{gZs5%@7yLW=;SV~(w#nS!pRfsb zw7ZZ{6aNEMp&@UfM4$^NGYhppSPa_$3!e-NE`!^!pJ8F}PdtZFjfKHKu^vWE7F-4g zV*l7eiSQv#@|H>j8*wtT^a-$J-So_`^qFDFnd#$JW9idm$*E{|du+*j(VI2PR^EzL zX;N%!W~H%3opVN_oz7Jv#EVac6^E7MHp9xN#!B-qn8tFil@h^hoE}@T1fKHNK4#V$ zQOdRf)~DatW?1{ou=c63_Gz;Ad2FriqiiW}GZbviY&e_S-2!ZUGHiTi*s#nzYixoJ z**4j5STo!n+h`nCWo@;Tw|%;Twbatg)+fN$C&N}_j+*TZTb~+R&POM=CR?A!wm$L$ zpn^G;M+NB6`qZ~JI~dw|1{QX5LS$g#RW`v;#hbw61`xc8-B9{72)6GImyDPrL~F)z zxIuWF&%MS+ZxLeEWz#oh^9k(rTFJp`AAzM9&>?||&a>HEB8z#0dUI`qAdsiio3mc1 zZQq;wJP>9EEmjg@9Xm(qaUf`HWbXH6hjROjW3@aEYKiT2|HA!1BRTOnoFa~M6gC=j^309Eb} zA$G83T)MoSx1>)5!MlnXRFxnQI0NQr-fyRdQeQr*-*U|_EK zcriygRdSS)*H^_F|=o`66^6h+*sP04QZlZi5JAyzc|@iUZ&a zK?08-qrsVTNv((78+IADofly#FTst!J^)&5JeQm!lkQG)xNtlG*l*P)5VWjBLAjIq z3{!4&SDd1S&AeQ#ml*(EWE-lRXy!~b=Ntg$gqXL!NegALKI{PK893^&-47bfq_}0Z zKNvGeVmUF^5whuKl|OWFT-4pjooY`Yw9jXM;4;1-xeu_1f-36(*jV2^UR#U~Oe4b5 z0I(~lVK)-)m(L)F5~z!jX*3{jLPi08ok{>j%Y5TRGAkW=cesa)0-&(x%hAZbVoVJ< z?LPIRKh)17UXU-BlYjcxo{9coN3dDPUm~<-4;yxCq(9U?ul>A$%v1{_Fv0%H4peER z*Qa{^HVg#He<{z?X!&kt%hz8^bth8VjK^llsT2_V%p zuXqC0JlqMS2X6o(#^!MzH4*6UHFcsK2dFThhKe>Zi4bYdWdKhM$y@3x!7sD(=R5fU zzr_1a^TxSm`O{-U;$K$j)b4vg*~zpx8n?=?kaO5m+po9X z4!X-tj=HmkK$l;q-2p;PF>8D@1S(FHaJNXXI}Me}yg;FZdxX}@){W5fhRi2+UAeF6 z8WRk@C7S1^(d|WUO*gYV9UrJAc>kwhDPkOjc^kap6yi=^0YQnYB}E^o+T_N$11 zbc|PPa=|KkiN}3;?x}w>;n?=-i}fUg*Ai{2I)Q!L21hc z;RMrnAoQ@n2aLB)lCa6LluN!PeYEz0Vx}uCp1z;kyy3+YZ|LHMXm?jqO{Q?>mFYT@ zsmb#yHJn7opc=XMGsC>0@#3LEW4a;LaHR8HYi|&&c5ir4L_j^JQ6x=bgpKW70w1aIlvOyc zo_fG*Rp!#}NATtpZ?y)I4r~WbvZpqxaIVO<6tZrp z8Y?}Np~i6??z|yz)laFjR~(iwiCP0MNK~CW(ataq%b8~IJzh}0Wx6obI}VY$V8^C; z!@j{^PG$~^!|v1hE+uEY2^q>+7uTxBIq9ukuEIGJr!DWG^u-I-1TC$!uyCmze`oR87+ zrIn3Ux1&ZXHRXBzGqQY1&+qJGT}9UiQb*wlU(jB!o0EVYtsOc(L~*D)Yhs@SET??Z z>R4Yg5)ah5{&bz!&$pE5p0Ok+Clb(Dc+|CC>I*FsubrJKPQ)sveMA930rU1HqV9UR zR4-E0f?=Ovz)s5V{fM*d?B9vl%^X{*2pf+GaUaM}!WO2dn_9Zvhu7A2 zizwk1Ife#;rkFy19Mo5h0f1L%1yR>^7|K0~vHn5Q5G1|^ja1dJsuzVi(6 zhe25@5Y+Y9(5RIgi9fEC)+!5uKW(_nlTl-iK4&MX78NC8js_}mbokm+^~u-W>tMQ; z$BQvC7_wlWkRJX z!JH_`Kwfj7?tsE}XPC(R+`z;Pblsn~W6(Eem_g&`(f~DP^eYl}F0h?36mUl^wGc`k7rWw)X0*;dcBj(a`Us6)%POS?R z^IUZ}3o^8S=yi*hxImQ?8Q>)uxQ6y`WQz;zyj9vnXK^_EI}kBiN8%k%2F$wT4+j%DD7qWn7D&8}cFk$!VxUk0wCrn>PWS8X|+hyxi|N`vkWaD_;% zvapwjGEj~;XGmZ$(Rn^QRZnH0^iz`hk7!q$IXNvEIG@sCO7mR*HgL8QyT?$k>oS_c zpM@CMPZ~%kcJHeUw5udnW3U_eTk3iACR%yBL44+STG}9^Zm;WaNRoC_l=#RM9#H?s z+ONdYk+a4%sJX$AGbLRgKWE@Z>QGBFa|5|X1r5y)8EE?DLD*O?HyG&o4-6vrj=j~F z3z?W9)S6)D7zOfv%U>_spNYo6d9lnQ3PkO(GfMYlV%ILeQ4@No1#D@g-GJsy+=!x? zi#sAg>ru`!i#wUvsO0i{>X%6H*yUQi_)RAM(5|}`)GrFScm7}yO;&bKi*-℘ZQc zYPPZN^&IckPlGN-}WI0L;+8eU#m z7y<9>O!nG927a4LP@3cj_%f?+%4|UfZpx_~sxdVJCR#o}T`bK&oUm=I(vb+*)V{br z$2|kn8bvc_bw)s}^B<6|+I-Rng`Nq4J`)2qf^}KzEmsG!!g)uHZI1-bnqhk5g z;k;=s3_LChG(7tS@usS9nO1TQ0Z)xO$t#|_B?KA#v;W*^V$f5W?dl-@zc;K?L%Y+R zAnX6p=dSTRNbOG>yluZ=pCQ4#^-A^$!JLk2w~4_K|K47&7Qe8a8p?|YSFfjraxbNQ zyB;eXPp5YM%*vt9R)mA~*s()DOi9J|oWTYMkB39@oH^R8QUVYZ6)7hUih!Gw>%YM6 zR22MP(eiXf1dMgNIC<(x0uW5)@4!CPKY2V>(c>Wj62=rM_tK4obTboeJwhm28n5Qu zNV}Ydh2M{xUsQ>LN$yToyMCo15dHLbB ztC?%U4$}QM*UE`+N5UfCi-*TvNk@lN%lRg|B4O6W#r0jE($TuP?A^Pik>GH?XMD+b zdcH>QnuG_Dz`mz2+~75Rzihhx^F)%>dai~3xpZu=ZL5fw83jgir_DNOJpQC2-?*!nHIt&?g2(b5(@22b*E^&?=@kXLI`8tNi zmgNg|XM}^t|7q_@z@wZZ_&m_$uRozMYNUET4i&-#UW6?P-AD^5|zTpFO&=fp~XN`Mxa% zR($A|UmSn@H1M~3^ROY;13#w}C$AX*?H-=lHrQAU`|rN{z-)aZvGm3RXFhtHfg2tM z@rv1vgz|QqW7}^SvE+kQH~e~WBe8Xy?Nr%DMl32l^7`4GjfAOn?C%z@1$`~uw{Y+e zDdO$L+uB>6W5i7do+)0Mrig=}uZgdN{$kZj_S-39@UG3n9(x1w%Cc()AAs)RXfBNmd*&y?V(H zpObARxJPY*sQT{QbhXg`JX=NWW`b{XIlLyir6`fK3LfW`f6IYM)?tppE`a- z^%N`m{!4LklEO}G*s@~rsFzsr)7Jld<}6_+s)xQdZ~~0iOkc#M=h=xj20r3B@+(%X zsekU-BGFEmR_ULcwUHHb*XizmdWeH){%_|e{4>DsC#hv6E(fvT!2A(KPXPXfugrbr z2?z1fp3RMalY{e{=J*g;O z?}PmU6FWWLNoYdN^+O?7&3fPm*)?@e;%$#kX@&#D4GL6W3%}F&@67IJv_~w7nc#wfiwv)E(G0 zD)BLV-`@Oh6K1Ytp`U?e$1BCq58z3vG4ornpN8iTd^s8#fDC-(i$@{%-gj!`f*;qo zh!vV!wYy;dy{cDkT?jW!*ym;}90&B_-bm5wBHsLRy=KQ$RvbA+b>H!f3o3z)t4B>{ z#l~?KbIViEpWs^h;3qc&{f$5UeAyBgF>Ufwe;~jgMr^5laRl7<)AaJN*WQA7Gjeas z#GNm@h;NooyZN7>ZhQ1a?b9E^huvGge%LdfW<;U#Z^x&1xro4`ZS4aapcZ=ep0o2; z!mShg&irogB=El>_daoUlZ#ksTQ+UeU`9N&?MMH7%LW(m$jQ-%{$=B)Ki5od zWECxj*AJb%3hsTLxNQIC_|3F=Co#{!h9y%Qq8_%rWy&XP4xc*wh0QdHw zxO3`;zx#X8$MzHJ^{9ah>ewPTaOpW7fcv{a34rcUG;Ce5h0k*0vWH&E-0^{fP#v5f zU%_xmWAGUo+|*JqI`T7VWLr=pe4y?>#zvG$)Qt ze!XhsG6%8#g=xQBU&@IuPTBp{Z3`X5;MT+AKT~ny$07vNnGRxxX3nHWD<@9i<~1LM z_16#Y`FchRjK5Kw9J>a-J8u`(uA9z@^>CN`&z6I;{=Q;|GtY^$g-%F01eFc@f2zHAsA|CqA_7U{Z;fn~$M|%jm^FNA} z2j4@2_CFi?{ zk@WnL&Ve}kQL*w*A0kQnqPqlIGmE*drT_Tuht&qbq#!RUO; z$*ds@kNF=dmc3<+*!X-N0?n{^* zJ#@@XV^;p3x2FDXTShCtyB^KT@1Tvc@~ZSQR(@X&{J-ii`fcTa@+xIxB(&qCl!Xw5Q>l1zLSaeg$<_Ckv1-x(JCTwlJti9#UiC{Fzr!aY*#Uz92R5+I4 zQ+dQ>owLb|SOc{phSch+s+lwIygbIr&DC1fXfH1_d_Gw?%>- zu|!)i$0Lia(pJ}~wbii3M^Q-N+S-L+D#b-5f7TdkDm40PjYg}l(t-&pf((;R1Q~3m z8YyaAhF+_*8Z10mbSwd(0)8Z;Es4De8-FrQ%Z}SEK~O*p7r5-l21r5N6or|5A!)!t<6+(^z$43S71qH{3a0pk#1^Bs> z1cEBC*@~!F0DWK%{Z|;(Xe+98di7L{4vb1Nr5WijJr}H?*B6SZGtr}{bQtM&JG>L5JW;x7OeK2gNE=C#F8D)_29nf~#t>Og zk|EL*!gEGruk|dNtm|GkW-Zbfo-r1@ZY9Yq3H#uGgrrsnIMJ9S9U(JZRa>J0fr;AV zv_{7}c#@}@bh*Y3l5aGTJY%uNX>Wa=^B6j4O-@aFC76K5lcBvH6%*~maDI&;$$Hgk zCU32>T9SOvbq5bLBAztBbOycpsI?;&vI=aYi?=4NCR;S%HKYPQH=?29d`1D|s88}O zW!JUfsVe*uL%pZw=OM>F@KmR%yTCz&>A-H z_ZeeBuf4WKps?(8q&^jpX!_i0D&(kjg&OMYO)Xs^RYMcT(dma3+G7yts2%Xzk{BNJ z)N?QJmkgMU7SKm0=p*jW8=EdmZ3V79gz-Q}1{CTzg#Ye_G z7F!s2inAu^9D`mL*Z?}OdNpefXPAhf;w0JXTk7o=E01-af$p@(R&rFcU(@bWCDEDp zPZl!f?qa&AE2i6$zjC@EEtS7gd0V=!oNist-b}_N9SWW3%(%QQ%=+9hIQuwj>BRo$ z_86Ppd2Bb}b8Z2_QO$K~yG3niG@-N6hiNa?Kkv~P^0e6%a)Unx%pO(HONCmrLKh2u zV6CmwOSZ`5>>Z43i}8R-@=?DJ;LV17(C5}m_U&=Cs2JIwJF$({BAz{lcJMXeHSPDR zF+bqHU>mtkYaQ|lx4@c(4r{$#&uHBO-9+Xb%^{W56t+;!jE3T`Sl9mx+p^}UuFfme zHI;Sv)CMWOc#^bP$TAv3x88c%dBNYMSgG-(f*wQ4?~P{|!4T%m0`^OL2c>n?)^*wQ z$ajP^8}Z~Bji)^j$%P?C2{gK!Is-rmgQsSZkQ@36yplf2(+?_hnluwN_L z2gU2MS7JlJN8TPqHUu~|1~@-=9Ar@iJch19oRNG#YFGQCZapLUI3Mull0l!bi`EF8 zes5N?4_`Pg#pzTpd+ce4wAqjjXap7O1sle3qn`IiZ2}t+!eB2buA?|jp;(T13Ydj_ zz-tE^Hl(pVMm%9g>%wcSR}V1?!*>R}EWow!eQXlP=YlPoLAT~_r|tJi`1{hFYdf=8r{4N57S?EuKH38}>@{LL_%rd|=~hYk4dv}ZenUAA@{kS31IX!} zkms?DTcgIL->VNZQm*Fx5rYn$p*N5DOjD+8sIeJ%>HwZ=>vWV!hxpS`d7TC_X|9mV zGsrjO8Uf->Hy*gw(Ov=Dg;$4v-{ZRp5L;j+z5gSiyL)dmWuR2ciHWulF zGpoZ`?@(X(r87V}2Hf>Js{nb-WYgno%zJ@8rX!^Mz7X$&xX=!NsC*(x3jT~xztoUM zRBAc1P*$0AsP^hWWtkb5Ss^s8LEwd=tTIzs2?Jwo6P}ErXIzirbyFuAv*@V~7jZ4< z60e)FOlU~b&g#laO~?4kN`$$t7N`LbE)wx28cOtxAdis7!+N>^ZzTn^AMxy>FcByO z(EvZACrr=~b?t_v_F5&ZOS{#!1s zZ!^y87^xb;8*1SRpKm=@Di%`Guf!1&#RxTm9%7~*YD0_~KwZ-LfspXx0r>YLIgqiU z&GI1|3+IPVsfRwWY8kTu^dad2?FZ5z7Zw$;$SRpUxu~=(9!yM6bE&(Sc>JzlA`#1$ g72d=vQ^QN~3e}jE@IlU>`ByY%Y*CRDeueaZ0F8^^H2?qr literal 0 HcmV?d00001 From 7e5278bf443deeb45012ea3d2f6d767900df13c6 Mon Sep 17 00:00:00 2001 From: Felix Schott Date: Mon, 10 Mar 2025 13:12:58 +0100 Subject: [PATCH 02/11] linter complaining because range-over-int is still experimental in go1.22 - use c-style loop instead --- internal/geoparquet/geoparquet.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/geoparquet/geoparquet.go b/internal/geoparquet/geoparquet.go index 1eb6486..428f43c 100644 --- a/internal/geoparquet/geoparquet.go +++ b/internal/geoparquet/geoparquet.go @@ -231,7 +231,7 @@ func FilterRecordBatchByBbox(ctx context.Context, recordReader *RecordReader, re var ymaxName string // loop over individual bbox values per record - for idx := range col.Len() { + for idx := 0; idx < col.Len(); idx++ { var bbox map[string]json.RawMessage if err := json.Unmarshal([]byte(col.ValueStr(idx)), &bbox); err != nil { return nil, fmt.Errorf("trouble unmarshalling bbox struct: %w", err) @@ -308,7 +308,7 @@ func FilterRecordBatchByBbox(ctx context.Context, recordReader *RecordReader, re maskBuilder := array.NewBooleanBuilder(memory.DefaultAllocator) defer maskBuilder.Release() - for idx := range col.Len() { + for idx := 0; idx < col.Len(); idx++ { value := col.GetOneForMarshal(idx) g, decodeErr := geo.DecodeGeometry(value, metadata.Columns[metadata.PrimaryColumn].Encoding) if decodeErr != nil { From ac7cffcf74dbf2efd7f7d2a332c8d752f8b57df9 Mon Sep 17 00:00:00 2001 From: Felix Schott Date: Mon, 10 Mar 2025 13:16:17 +0100 Subject: [PATCH 03/11] missed one range-over-int --- internal/geoparquet/recordreader.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/geoparquet/recordreader.go b/internal/geoparquet/recordreader.go index e910fa9..427de53 100644 --- a/internal/geoparquet/recordreader.go +++ b/internal/geoparquet/recordreader.go @@ -151,7 +151,7 @@ func NewRecordReader(config *ReaderConfig) (*RecordReader, error) { // generate indices from col names and compute the indices to include indicesToExclude := newIndicesSet(schema.NumFields()-len(config.ExcludeColNames)).FromColNames(config.ExcludeColNames, schema) allIndices := newIndicesSet(schema.NumFields()) - for i := range schema.NumFields() { + for i := 0; i < schema.NumFields(); i++ { allIndices.Add(i) } indices := allIndices.Difference(indicesToExclude) From d41d55be04ef5a118f8f2881944912ea43387b3f Mon Sep 17 00:00:00 2001 From: Felix Schott Date: Tue, 11 Mar 2025 11:01:21 +0100 Subject: [PATCH 04/11] simplify bbox struct name logic as per tschaub's comment --- internal/geoparquet/geoparquet.go | 52 +++++++++---------------------- 1 file changed, 15 insertions(+), 37 deletions(-) diff --git a/internal/geoparquet/geoparquet.go b/internal/geoparquet/geoparquet.go index 428f43c..81a32be 100644 --- a/internal/geoparquet/geoparquet.go +++ b/internal/geoparquet/geoparquet.go @@ -225,11 +225,25 @@ func FilterRecordBatchByBbox(ctx context.Context, recordReader *RecordReader, re maskBuilder := array.NewBooleanBuilder(memory.DefaultAllocator) defer maskBuilder.Release() + // infer bbox struct field names var xminName string var yminName string var xmaxName string var ymaxName string + if metadata.Columns[metadata.PrimaryColumn].Covering != nil { + xminName = metadata.Columns[metadata.PrimaryColumn].Covering.Bbox.Xmin[1] + yminName = metadata.Columns[metadata.PrimaryColumn].Covering.Bbox.Ymin[1] + xmaxName = metadata.Columns[metadata.PrimaryColumn].Covering.Bbox.Xmax[1] + ymaxName = metadata.Columns[metadata.PrimaryColumn].Covering.Bbox.Ymax[1] + } else { + // fallback to standard names + xminName = "xmin" + yminName = "ymin" + xmaxName = "xmax" + ymaxName = "ymax" + } + // loop over individual bbox values per record for idx := 0; idx < col.Len(); idx++ { var bbox map[string]json.RawMessage @@ -237,43 +251,7 @@ func FilterRecordBatchByBbox(ctx context.Context, recordReader *RecordReader, re return nil, fmt.Errorf("trouble unmarshalling bbox struct: %w", err) } - // infer bbox field names from the first element - if idx == 0 { - // check standard name first, if no match, check covering metadata - if _, ok := bbox["xmin"]; ok { - xminName = "xmin" - } else if metadata.Columns[metadata.PrimaryColumn].Covering != nil { - xminName = "xmin" // DEBUG metadata.Columns[metadata.PrimaryColumn].Covering.Xmin[1] - } else { - return nil, fmt.Errorf("can not infer bbox field name for 'xmin'") - } - - if _, ok := bbox["ymin"]; ok { - yminName = "ymin" - } else if metadata.Columns[metadata.PrimaryColumn].Covering != nil { - yminName = metadata.Columns[metadata.PrimaryColumn].Covering.Bbox.Ymin[1] - } else { - return nil, fmt.Errorf("can not infer bbox field name for 'ymin'") - } - - if _, ok := bbox["xmax"]; ok { // check standard name first - xmaxName = "xmax" - } else if metadata.Columns[metadata.PrimaryColumn].Covering != nil { - xmaxName = metadata.Columns[metadata.PrimaryColumn].Covering.Bbox.Xmax[1] - } else { - return nil, fmt.Errorf("can not infer bbox field name for 'xmax'") - } - - if _, ok := bbox["ymax"]; ok { - ymaxName = "ymax" - } else if metadata.Columns[metadata.PrimaryColumn].Covering != nil { - ymaxName = metadata.Columns[metadata.PrimaryColumn].Covering.Bbox.Ymax[1] - } else { - return nil, fmt.Errorf("can not infer bbox field name for 'ymax'") - } - } - - bboxValue := &geo.Bbox{} // create empty struct to hold bbox values of this record + bboxValue := &geo.Bbox{} // create empty struct to hold bbox values of this row if err := json.Unmarshal(bbox[xminName], &bboxValue.Xmin); err != nil { return nil, fmt.Errorf("trouble parsing bbox.%v field: %w", xminName, err) From 99bc610ffb0f18788e13b79c7c4fe0c477ae515e Mon Sep 17 00:00:00 2001 From: Felix Schott Date: Tue, 11 Mar 2025 11:24:03 +0100 Subject: [PATCH 05/11] proper error wrapping --- cmd/gpq/command/extract.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/gpq/command/extract.go b/cmd/gpq/command/extract.go index fd64b07..abf9200 100644 --- a/cmd/gpq/command/extract.go +++ b/cmd/gpq/command/extract.go @@ -78,7 +78,7 @@ func (c *ExtractCmd) Run() error { // parse bbox filter argument into geo.Bbox struct if applicable inputBbox, err := geo.NewBboxFromString(c.Bbox) if err != nil { - return NewCommandError(err.Error()) + return NewCommandError("trouble getting bbox from input string: %w", err) } // read and write records in loop @@ -93,7 +93,7 @@ func (c *ExtractCmd) Run() error { filteredRecord, err := geoparquet.FilterRecordBatchByBbox(context.Background(), recordReader, &record, inputBbox) if err != nil { - return NewCommandError(err.Error()) + return NewCommandError("trouble filtering record batch: %w", err) } if err := recordWriter.Write(*filteredRecord); err != nil { From d05df57802520a23b98bfecc62268f3f4cd5f6fb Mon Sep 17 00:00:00 2001 From: Felix Schott Date: Fri, 21 Mar 2025 11:26:03 +0100 Subject: [PATCH 06/11] add pushdown filtering based on row group stats, restructure/refactor code --- cmd/gpq/command/extract.go | 109 ++++++- cmd/gpq/command/extract_test.go | 37 ++- internal/geojson/geojson.go | 2 +- internal/geoparquet/filter.go | 279 ++++++++++++++++++ internal/geoparquet/filter_test.go | 108 +++++++ internal/geoparquet/geoparquet.go | 160 +++------- internal/geoparquet/geoparquet_test.go | 251 ++++++++++++++-- internal/geoparquet/metadata.go | 5 + internal/geoparquet/recordreader.go | 185 +++++------- .../cases/example-v1.1.0-partitioned.parquet | Bin 0 -> 30566 bytes internal/validator/validator.go | 2 +- 11 files changed, 861 insertions(+), 277 deletions(-) create mode 100644 internal/geoparquet/filter.go create mode 100644 internal/geoparquet/filter_test.go create mode 100644 internal/testdata/cases/example-v1.1.0-partitioned.parquet diff --git a/cmd/gpq/command/extract.go b/cmd/gpq/command/extract.go index abf9200..129af7c 100644 --- a/cmd/gpq/command/extract.go +++ b/cmd/gpq/command/extract.go @@ -6,6 +6,7 @@ import ( "os" "strings" + "github.com/apache/arrow/go/v16/arrow" "github.com/planetlabs/gpq/internal/geo" "github.com/planetlabs/gpq/internal/geoparquet" ) @@ -49,18 +50,93 @@ func (c *ExtractCmd) Run() error { // prepare input reader (ignore certain columns if asked to - DropCols/KeepOnlyCols) config := &geoparquet.ReaderConfig{Reader: input} + + parquetFileReader, err := geoparquet.NewParquetFileReader(config) + if err != nil { + return NewCommandError("could not get ParquetFileReader: %w", err) + } + + arrowFileReader, err := geoparquet.NewArrowFileReader(config, parquetFileReader) + if err != nil { + return NewCommandError("could not get ArrowFileReader: %w", err) + } + + geoMetadata, err := geoparquet.GetMetadataFromFileReader(parquetFileReader) + if err != nil { + return NewCommandError("could not get geo metadata from file reader: %w", err) + } + + arrowSchema, schemaErr := arrowFileReader.Schema() + if schemaErr != nil { + return NewCommandError("trouble getting arrow schema: %w", schemaErr) + } + + // projection pushdown - column filtering + var columnIndices []int = nil + + var includeColumns []string + var excludeColumns []string if c.DropCols != "" { - cols := strings.Split(c.DropCols, ",") - config.ExcludeColNames = cols + excludeColumns = strings.Split(c.DropCols, ",") } if c.KeepOnlyCols != "" { - cols := strings.Split(c.KeepOnlyCols, ",") - config.IncludeColNames = cols + includeColumns = strings.Split(c.KeepOnlyCols, ",") } - recordReader, rrErr := geoparquet.NewRecordReader(config) - if rrErr != nil { - return NewCommandError("trouble reading geoparquet: %w", rrErr) + excludeColNamesProvided := len(excludeColumns) > 0 + includeColNamesProvided := len(includeColumns) > 0 + + if excludeColNamesProvided || includeColNamesProvided { + if excludeColNamesProvided == includeColNamesProvided { + return NewCommandError("please pass only one of DropColumns/KeepOnlyColumns") + } + + if includeColNamesProvided { + columnIndices, err = geoparquet.GetColumnIndices(includeColumns, arrowSchema) + if err != nil { + return NewCommandError("trouble inferring column names (positive selection): %w", err) + } + } + + if excludeColNamesProvided { + columnIndices, err = geoparquet.GetColumnIndicesByDifference(excludeColumns, arrowSchema) + if err != nil { + return NewCommandError("trouble inferring column names (negative selection): %w", err) + } + } + } + config.Columns = columnIndices + + // predicate pushdown - spatial row filtering + var rowGroups []int = nil + + // parse bbox filter argument into geo.Bbox struct if applicable + inputBbox, err := geo.NewBboxFromString(c.Bbox) + if err != nil { + return NewCommandError("trouble getting bbox from input string: %w", err) + } + var bboxCol *geoparquet.BboxColumn + if inputBbox != nil { + bboxCol = geoparquet.GetBboxColumn(parquetFileReader.MetaData().Schema, geoMetadata) + + if bboxCol.Name != "" { // if there is a bbox col in the file + rowGroups, err = geoparquet.GetRowGroupsByBbox(parquetFileReader, bboxCol, inputBbox) + if err != nil { + return NewCommandError("trouble scanning row group metadata: %w", err) + } + } + } + + config.RowGroups = rowGroups + + // create new record reader - based on the config values for + // Columns and RowGroups it will only read a subset of + // columns and row groups + ctx := context.Background() + + recordReader, err := geoparquet.NewRecordReader(ctx, arrowFileReader, geoMetadata, columnIndices, rowGroups) + if err != nil { + return NewCommandError("trouble creating geoparquet record reader: %w", err) } defer recordReader.Close() @@ -75,12 +151,6 @@ func (c *ExtractCmd) Run() error { } defer recordWriter.Close() - // parse bbox filter argument into geo.Bbox struct if applicable - inputBbox, err := geo.NewBboxFromString(c.Bbox) - if err != nil { - return NewCommandError("trouble getting bbox from input string: %w", err) - } - // read and write records in loop for { record, readErr := recordReader.Read() @@ -91,9 +161,16 @@ func (c *ExtractCmd) Run() error { return readErr } - filteredRecord, err := geoparquet.FilterRecordBatchByBbox(context.Background(), recordReader, &record, inputBbox) - if err != nil { - return NewCommandError("trouble filtering record batch: %w", err) + // filter by bbox if asked to + var filteredRecord *arrow.Record + if inputBbox != nil && bboxCol != nil { + var filterErr error + filteredRecord, filterErr = geoparquet.FilterRecordBatchByBbox(ctx, &record, inputBbox, bboxCol) + if filterErr != nil { + return NewCommandError("trouble filtering record batch by bbox: %w", filterErr) + } + } else { + filteredRecord = &record } if err := recordWriter.Write(*filteredRecord); err != nil { diff --git a/cmd/gpq/command/extract_test.go b/cmd/gpq/command/extract_test.go index 345d2ea..5f3d459 100644 --- a/cmd/gpq/command/extract_test.go +++ b/cmd/gpq/command/extract_test.go @@ -26,7 +26,7 @@ func (s *Suite) TestExtractDropCols() { s.Require().NoError(err) s.Equal(4, fileReader.MetaData().Schema.NumColumns()) - recordReader, err := geoparquet.NewRecordReader(&geoparquet.ReaderConfig{ + recordReader, err := geoparquet.NewRecordReaderFromConfig(&geoparquet.ReaderConfig{ Reader: bytes.NewReader(data), }) s.Require().NoError(err) @@ -55,7 +55,7 @@ func (s *Suite) TestExtractKeepOnlyCols() { s.Require().NoError(err) s.Equal(3, fileReader.MetaData().Schema.NumColumns()) - recordReader, err := geoparquet.NewRecordReader(&geoparquet.ReaderConfig{ + recordReader, err := geoparquet.NewRecordReaderFromConfig(&geoparquet.ReaderConfig{ Reader: bytes.NewReader(data), }) s.Require().NoError(err) @@ -76,7 +76,7 @@ func (s *Suite) TestExtractBbox110() { data := s.readStdout() - recordReader, err := geoparquet.NewRecordReader(&geoparquet.ReaderConfig{ + recordReader, err := geoparquet.NewRecordReaderFromConfig(&geoparquet.ReaderConfig{ Reader: bytes.NewReader(data), }) s.Require().NoError(err) @@ -94,6 +94,35 @@ func (s *Suite) TestExtractBbox110() { s.Assert().Equal("Tanzania", country) } +// Since the 1.1.0 parquet file includes a bbox column and is partitioned into spatially ordered row groups, +// we expect the bbox column row group statistic to be used for spatial pushdown filtering. +func (s *Suite) TestExtractBbox110Partitioned() { + cmd := &command.ExtractCmd{ + Input: "../../../internal/testdata/cases/example-v1.1.0-partitioned.parquet", + Bbox: "34,-7,36,-6", + } + s.Require().NoError(cmd.Run()) + + data := s.readStdout() + + recordReader, err := geoparquet.NewRecordReaderFromConfig(&geoparquet.ReaderConfig{ + Reader: bytes.NewReader(data), + }) + s.Require().NoError(err) + defer recordReader.Close() + + // we expect only one row, namely Tanzania + s.Require().Equal(int64(1), recordReader.NumRows()) + + record, readErr := recordReader.Read() + s.Require().NoError(readErr) + s.Assert().Equal(int64(8), record.NumCols()) + s.Assert().Equal(int64(1), record.NumRows()) + + country := record.Column(recordReader.Schema().ColumnIndexByName("name")).ValueStr(0) + s.Assert().Equal("Tanzania", country) +} + // Since the 1.0.0 parquet file doesn't have a bbox column, we expect the bbox column to be calculated on the fly. func (s *Suite) TestExtractBbox100() { cmd := &command.ExtractCmd{ @@ -104,7 +133,7 @@ func (s *Suite) TestExtractBbox100() { data := s.readStdout() - recordReader, err := geoparquet.NewRecordReader(&geoparquet.ReaderConfig{ + recordReader, err := geoparquet.NewRecordReaderFromConfig(&geoparquet.ReaderConfig{ Reader: bytes.NewReader(data), }) s.Require().NoError(err) diff --git a/internal/geojson/geojson.go b/internal/geojson/geojson.go index 73978d1..be16c0d 100644 --- a/internal/geojson/geojson.go +++ b/internal/geojson/geojson.go @@ -26,7 +26,7 @@ func GetDefaultMetadata() *geoparquet.Metadata { } func FromParquet(reader parquet.ReaderAtSeeker, writer io.Writer) error { - recordReader, rrErr := geoparquet.NewRecordReader(&geoparquet.ReaderConfig{ + recordReader, rrErr := geoparquet.NewRecordReaderFromConfig(&geoparquet.ReaderConfig{ Reader: reader, }) if rrErr != nil { diff --git a/internal/geoparquet/filter.go b/internal/geoparquet/filter.go new file mode 100644 index 0000000..621c82f --- /dev/null +++ b/internal/geoparquet/filter.go @@ -0,0 +1,279 @@ +package geoparquet + +import ( + "context" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "math" + "slices" + + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/compute" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/parquet/file" + "github.com/apache/arrow/go/v16/parquet/metadata" + "github.com/planetlabs/gpq/internal/geo" +) + +// PROJECTION PUSHDOWN - COLUMN FILTERING UTILS + +// A Set type based on map, to hold arrow column indices. +// Implements common Set methods such as Difference() and Contains(). +// To instantiate, use the constructor newIndicesSet() followed by either +// Add() if you want to build the Set sequentially or the convenience function +// FromColNames(). +type indicesSet map[int]struct{} + +func newIndicesSet(size int) *indicesSet { + var s indicesSet = make(map[int]struct{}, size) + return &s +} + +func (s *indicesSet) Add(col int) *indicesSet { + (*s)[col] = struct{}{} + return s +} + +func (s *indicesSet) FromColNames(cols []string, schema *arrow.Schema) *indicesSet { + for _, col := range cols { + if indicesForColumn := schema.FieldIndices(col); indicesForColumn != nil { + for _, colIdx := range indicesForColumn { + s.Add(colIdx) + } + } + } + return s +} + +func (s *indicesSet) Contains(col int) bool { + _, ok := (*s)[col] + return ok +} + +func (s *indicesSet) Difference(other *indicesSet) *indicesSet { + sSize := s.Size() + otherSize := s.Size() + var newSet *indicesSet + if sSize < otherSize { + newSet = newIndicesSet(otherSize - sSize) + } else { + newSet = newIndicesSet(sSize - otherSize) + } + for key := range *s { + if !other.Contains(key) { + newSet.Add(key) + } + } + return newSet +} + +func (s *indicesSet) Size() int { + return len(*s) +} + +func (s *indicesSet) List() []int { + keys := make([]int, 0, len(*s)) + for k := range *s { + keys = append(keys, k) + } + slices.Sort(keys) + return keys +} + +// Given a list of columns names to include, return the corresponding columns indices. +func GetColumnIndices(includeColumns []string, arrowSchema *arrow.Schema) ([]int, error) { + // generate indices from col names + indices := newIndicesSet(len(includeColumns)).FromColNames(includeColumns, arrowSchema).List() + + return indices, nil +} + +// Given a list of column names to exclude, return a list of the remaining columns indices. +func GetColumnIndicesByDifference(excludeColumns []string, arrowSchema *arrow.Schema) ([]int, error) { + // generate indices from col names and compute the indices to include + indicesToExclude := newIndicesSet(arrowSchema.NumFields()-len(excludeColumns)).FromColNames(excludeColumns, arrowSchema) + allIndices := newIndicesSet(arrowSchema.NumFields()) + for i := 0; i < arrowSchema.NumFields(); i++ { + allIndices.Add(i) + } + return allIndices.Difference(indicesToExclude).List(), nil +} + +// PREDICATE PUSHDOWN - ROW FILTERING UTILS + +// Get row group indices that intersect with the input bbox. Uses the bbox column row group +// stats to calculate intersection. +func GetRowGroupsByBbox(fileReader *file.Reader, bboxCol *BboxColumn, inputBbox *geo.Bbox) ([]int, error) { + numRowGroups := fileReader.NumRowGroups() + intersectingRowGroups := make([]int, 0, numRowGroups) + for i := 0; i < numRowGroups; i += 1 { + intersects, err := RowGroupIntersects(fileReader.MetaData(), bboxCol, i, inputBbox) + if err != nil { + return nil, err + } + if intersects { + intersectingRowGroups = append(intersectingRowGroups, i) + } + } + return intersectingRowGroups, nil +} + +// Return min/max statistics for a given column and RowGroup. +// For nested structures, use `.`. +func GetColumnMinMax(fileMetadata *metadata.FileMetaData, rowGroup int, columnPath string) (min float64, max float64, err error) { + rowGroupMetadata := fileMetadata.RowGroup(rowGroup) + if rowGroupMetadata == nil { + return 0, 0, fmt.Errorf("metadata for RowGroup %v is nil", rowGroup) + } + + rowGroupSchema := rowGroupMetadata.Schema + if rowGroupSchema == nil { + return 0, 0, fmt.Errorf("schema for RowGroup %v is nil", rowGroup) + } + + columnIdx := rowGroupSchema.ColumnIndexByName(columnPath) + if columnIdx == -1 { + return 0, 0, fmt.Errorf("column %v not found", columnPath) + } + + fieldMetadata, err := rowGroupMetadata.ColumnChunk(columnIdx) + if err != nil { + return 0, 0, fmt.Errorf("couldn't get ColumnChunkMetadata for RowGroup %v/Column %v: %w", rowGroup, columnPath, err) + } + fieldStats, err := fieldMetadata.Statistics() + if err != nil { + return 0, 0, fmt.Errorf("couldn't get ColumnChunkMetadata stats: %w", err) + } + if !fieldStats.HasMinMax() { + return 0, 0, fmt.Errorf("no min/max statistics available for ") + } + + bitsMin := binary.LittleEndian.Uint64(fieldStats.EncodeMin()) + min = math.Float64frombits(bitsMin) + bitsMax := binary.LittleEndian.Uint64(fieldStats.EncodeMax()) + max = math.Float64frombits(bitsMax) + + return min, max, nil +} + +// Check whether the bbox features in a row group intersect with the input bbox, based on the row group min/max stats. +func RowGroupIntersects(fileMetadata *metadata.FileMetaData, bboxCol *BboxColumn, rowGroup int, inputBbox *geo.Bbox) (bool, error) { + if bboxCol.Name == "" { + return false, errors.New("bboxCol.Name is empty") + } + xminPath := fmt.Sprintf("%v.%v", bboxCol.Name, bboxCol.Xmin) + xmin, _, err := GetColumnMinMax(fileMetadata, rowGroup, xminPath) + if err != nil { + return false, fmt.Errorf("could not get min/max statistics for %v: %w", xminPath, err) + } + + yminPath := fmt.Sprintf("%v.%v", bboxCol.Name, bboxCol.Ymin) + ymin, _, err := GetColumnMinMax(fileMetadata, rowGroup, yminPath) + if err != nil { + return false, fmt.Errorf("could not get min/max statistics for %v: %w", yminPath, err) + } + + xmaxPath := fmt.Sprintf("%v.%v", bboxCol.Name, bboxCol.Xmax) + _, xmax, err := GetColumnMinMax(fileMetadata, rowGroup, xmaxPath) + if err != nil { + return false, fmt.Errorf("could not get min/max statistics for %v: %w", xmaxPath, err) + } + + ymaxPath := fmt.Sprintf("%v.%v", bboxCol.Name, bboxCol.Ymax) + _, ymax, err := GetColumnMinMax(fileMetadata, rowGroup, ymaxPath) + if err != nil { + return false, fmt.Errorf("could not get min/max statistics for %v: %w", ymaxPath, err) + } + + rowGroupBbox := &geo.Bbox{Xmin: xmin, Ymin: ymin, Xmax: xmax, Ymax: ymax} + return rowGroupBbox.Intersects(inputBbox), nil +} + +func filterRecord(ctx context.Context, record *arrow.Record, predicate func(int64) (bool, error)) (*arrow.Record, error) { + // we build a boolean mask and pass it to compute.FilterRecordBatch later + maskBuilder := array.NewBooleanBuilder(memory.DefaultAllocator) + defer maskBuilder.Release() + + // loop over individual bbox values per record + for idx := int64(0); idx < (*record).NumRows(); idx++ { + p, err := predicate(idx) + if err != nil { + return nil, err + } + maskBuilder.Append(p) + } + + r, filterErr := compute.FilterRecordBatch(ctx, *record, maskBuilder.NewBooleanArray(), &compute.FilterOptions{NullSelection: 0}) // TODO check what this is doing + if filterErr != nil { + return nil, fmt.Errorf("trouble filtering record batch: %w", filterErr) + } + return &r, nil +} + +// Filter rows in an arrow.Record by intersection of the feature bounding boxes with an input bbox. +// If there is a bbox column, it will be used to compute intersection. If not, the bbox will be computed +// on the fly. +func FilterRecordBatchByBbox(ctx context.Context, record *arrow.Record, inputBbox *geo.Bbox, bboxCol *BboxColumn) (*arrow.Record, error) { + var filteredRecord *arrow.Record + var filterErr error + + if inputBbox != nil && bboxCol.Index != -1 { // bbox argument has been provided and there is a bbox column we can use for filtering + col := (*record).Column(bboxCol.Index).(*array.Struct) + defer col.Release() + + filteredRecord, filterErr = filterRecord(ctx, record, func(idx int64) (bool, error) { + var bbox map[string]json.RawMessage + if err := json.Unmarshal([]byte(col.ValueStr(int(idx))), &bbox); err != nil { + return false, fmt.Errorf("trouble unmarshalling bbox struct: %w", err) + } + + bboxValue := &geo.Bbox{} // create empty struct to hold bbox values of this row + + if err := json.Unmarshal(bbox[bboxCol.Xmin], &bboxValue.Xmin); err != nil { + return false, fmt.Errorf("trouble parsing bbox.%v field: %w", bboxCol.Xmin, err) + } + if err := json.Unmarshal(bbox[bboxCol.Ymin], &bboxValue.Ymin); err != nil { + return false, fmt.Errorf("trouble parsing bbox.%v field: %w", bboxCol.Ymin, err) + } + if err := json.Unmarshal(bbox[bboxCol.Xmax], &bboxValue.Xmax); err != nil { + return false, fmt.Errorf("trouble parsing bbox.%v field: %w", bboxCol.Xmax, err) + } + if err := json.Unmarshal(bbox[bboxCol.Ymax], &bboxValue.Ymax); err != nil { + return false, fmt.Errorf("trouble parsing bbox.%v field: %w", bboxCol.Ymax, err) + } + + // check whether the bbox passed to this function + // intersects with the bbox of the record + return inputBbox.Intersects(bboxValue), nil + }) + } else if inputBbox != nil && bboxCol.Index == -1 { + // bbox filter passed to function but there is no bbox col. + // this means we have to compute the bboxes of the rows ourselves + primaryColIdx := bboxCol.BaseColumn + col := (*record).Column(primaryColIdx) + defer col.Release() + + filteredRecord, filterErr = filterRecord(ctx, record, func(idx int64) (bool, error) { + value := col.GetOneForMarshal(int(idx)) + g, decodeErr := geo.DecodeGeometry(value, bboxCol.BaseColumnEncoding) + if decodeErr != nil { + return false, fmt.Errorf("trouble decoding geometry: %w", decodeErr) + } + bounds := g.Coordinates.Bound() + bboxValue := &geo.Bbox{ + Xmin: bounds.Min.X(), + Ymin: bounds.Min.Y(), + Xmax: bounds.Max.X(), + Ymax: bounds.Max.Y(), + } + // now that we've computed the bbox, same logic as above + return inputBbox.Intersects(bboxValue), nil + }) + } else { + filteredRecord = record + } + return filteredRecord, filterErr +} diff --git a/internal/geoparquet/filter_test.go b/internal/geoparquet/filter_test.go new file mode 100644 index 0000000..8dcd76c --- /dev/null +++ b/internal/geoparquet/filter_test.go @@ -0,0 +1,108 @@ +package geoparquet_test + +import ( + "os" + "testing" + + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/parquet/file" + "github.com/apache/arrow/go/v16/parquet/pqarrow" + "github.com/planetlabs/gpq/internal/geo" + "github.com/planetlabs/gpq/internal/geoparquet" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRowGroupIntersects(t *testing.T) { + fixturePath := "../testdata/cases/example-v1.1.0-partitioned.parquet" + input, openErr := os.Open(fixturePath) + require.NoError(t, openErr) + + fileReader, err := file.NewParquetReader(input) + require.NoError(t, err) + + require.Equal(t, 2, fileReader.NumRowGroups()) + + bbox := &geo.Bbox{Xmin: 34.0, Ymin: -7.0, Xmax: 36.0, Ymax: -6.0} // somewhere in tanzania + geoMetadata, err := geoparquet.GetMetadataFromFileReader(fileReader) + require.NoError(t, err) + + bboxCol := geoparquet.GetBboxColumn(fileReader.MetaData().Schema, geoMetadata) + + // the file has two row groups - the first one contains all data for the eastern hemisphere, + // the second for the western hemisphere + intersectsEasternHemisphere, err := geoparquet.RowGroupIntersects(fileReader.MetaData(), bboxCol, 0, bbox) + assert.NoError(t, err) + assert.Equal(t, intersectsEasternHemisphere, true) + + intersectsWesternHemisphere, err := geoparquet.RowGroupIntersects(fileReader.MetaData(), bboxCol, 1, bbox) + assert.NoError(t, err) + assert.Equal(t, intersectsWesternHemisphere, false) +} + +func TestGetColumnMinMax(t *testing.T) { + fixturePath := "../testdata/cases/example-v1.1.0-partitioned.parquet" + input, openErr := os.Open(fixturePath) + require.NoError(t, openErr) + + fileReader, err := file.NewParquetReader(input) + require.NoError(t, err) + + require.Equal(t, 2, fileReader.NumRowGroups()) + + xminMin, xminMax, err := geoparquet.GetColumnMinMax(fileReader.MetaData(), 0, "bbox.xmin") + assert.NoError(t, err) + assert.Equal(t, 29.339997592900346, xminMin) + assert.Equal(t, 29.339997592900346, xminMax) + + xmaxMin, xmaxMax, err := geoparquet.GetColumnMinMax(fileReader.MetaData(), 0, "bbox.xmax") + assert.NoError(t, err) + assert.Equal(t, 40.31659000000002, xmaxMin) + assert.Equal(t, 40.31659000000002, xmaxMax) + + xminMin, xminMax, err = geoparquet.GetColumnMinMax(fileReader.MetaData(), 1, "bbox.xmin") + assert.NoError(t, err) + assert.Equal(t, -180.0, xminMin) + assert.Equal(t, -17.06342322434257, xminMax) + + xmaxMin, xmaxMax, err = geoparquet.GetColumnMinMax(fileReader.MetaData(), 1, "bbox.xmax") + assert.NoError(t, err) + assert.Equal(t, -66.96465999999998, xmaxMin) + assert.Equal(t, 180.0, xmaxMax) +} + +func TestGetColumnIndices(t *testing.T) { + fixturePath := "../testdata/cases/example-v1.1.0.parquet" + input, openErr := os.Open(fixturePath) + require.NoError(t, openErr) + + fileReader, err := file.NewParquetReader(input) + require.NoError(t, err) + + arrowReader, err := pqarrow.NewFileReader(fileReader, pqarrow.ArrowReadProperties{BatchSize: 1024}, memory.DefaultAllocator) + require.NoError(t, err) + schema, err := arrowReader.Schema() + require.NoError(t, err) + + indices, err := geoparquet.GetColumnIndices([]string{"pop_est", "name", "iso_a3"}, schema) + assert.NoError(t, err) + assert.Equal(t, []int{0, 2, 3}, indices) +} + +func TestGetColumnIndicesByDifference(t *testing.T) { + fixturePath := "../testdata/cases/example-v1.1.0.parquet" + input, openErr := os.Open(fixturePath) + require.NoError(t, openErr) + + fileReader, err := file.NewParquetReader(input) + require.NoError(t, err) + + arrowReader, err := pqarrow.NewFileReader(fileReader, pqarrow.ArrowReadProperties{BatchSize: 1024}, memory.DefaultAllocator) + require.NoError(t, err) + schema, err := arrowReader.Schema() + require.NoError(t, err) + + indices, err := geoparquet.GetColumnIndicesByDifference([]string{"pop_est", "name", "iso_a3"}, schema) + assert.NoError(t, err) + assert.Equal(t, []int{1, 4, 5, 6}, indices) +} diff --git a/internal/geoparquet/geoparquet.go b/internal/geoparquet/geoparquet.go index 81a32be..a8ba7ec 100644 --- a/internal/geoparquet/geoparquet.go +++ b/internal/geoparquet/geoparquet.go @@ -1,7 +1,6 @@ package geoparquet import ( - "context" "encoding/json" "errors" "fmt" @@ -9,7 +8,6 @@ import ( "github.com/apache/arrow/go/v16/arrow" "github.com/apache/arrow/go/v16/arrow/array" - "github.com/apache/arrow/go/v16/arrow/compute" "github.com/apache/arrow/go/v16/arrow/memory" "github.com/apache/arrow/go/v16/parquet" "github.com/apache/arrow/go/v16/parquet/compress" @@ -192,125 +190,63 @@ func FromParquet(input parquet.ReaderAtSeeker, output io.Writer, convertOptions return pqutil.TransformByColumn(config) } -// Returns the index of the bbox column, -1 means not found. -// If there is no match for the standard name "bbox", the covering metadata is consulted. -func GetBboxColumnIndex(schema *schema.Schema, metadata *Metadata) int { - // try standard name first - bboxColIdx := schema.Root().FieldIndexByName("bbox") - // if no match, check covering metadata - if bboxColIdx == -1 && metadata.Columns[metadata.PrimaryColumn].Covering != nil && len(metadata.Columns[metadata.PrimaryColumn].Covering.Bbox.Xmin) == 2 { - bboxColName := metadata.Columns[metadata.PrimaryColumn].Covering.Bbox.Xmin[0] - bboxColIdx = schema.Root().FieldIndexByName(bboxColName) - } - return bboxColIdx +type BboxColumnFieldNames struct { + Xmin string + Ymin string + Xmax string + Ymax string } -func FilterRecordBatchByBbox(ctx context.Context, recordReader *RecordReader, record *arrow.Record, inputBbox *geo.Bbox) (*arrow.Record, error) { - - metadata := recordReader.Metadata() - schema := recordReader.Schema() +func getBboxColumnFieldNames(metadata *Metadata) *BboxColumnFieldNames { + // infer bbox struct field names + fieldNames := &BboxColumnFieldNames{} - bboxColIdx := -1 // -1 means no column found - if inputBbox != nil { - bboxColIdx = GetBboxColumnIndex(schema, metadata) + if metadata.Columns[metadata.PrimaryColumn].Covering != nil { + fieldNames.Xmin = metadata.Columns[metadata.PrimaryColumn].Covering.Bbox.Xmin[1] + fieldNames.Ymin = metadata.Columns[metadata.PrimaryColumn].Covering.Bbox.Ymin[1] + fieldNames.Xmax = metadata.Columns[metadata.PrimaryColumn].Covering.Bbox.Xmax[1] + fieldNames.Ymax = metadata.Columns[metadata.PrimaryColumn].Covering.Bbox.Ymax[1] + } else { + // fallback to standard names + fieldNames.Xmin = "xmin" + fieldNames.Ymin = "ymin" + fieldNames.Xmax = "xmax" + fieldNames.Ymax = "ymax" } - var filteredRecord *arrow.Record - - if inputBbox != nil && bboxColIdx != -1 { // bbox argument has been provided and there is a bbox column we can use for filtering - col := (*record).Column(bboxColIdx).(*array.Struct) - defer col.Release() + return fieldNames +} - // we build a boolean mask and pass it to compute.FilterRecordBatch later - maskBuilder := array.NewBooleanBuilder(memory.DefaultAllocator) - defer maskBuilder.Release() +type BboxColumn struct { + Index int + Name string + BaseColumn int // the primary geometry column the bbox column references + BaseColumnEncoding string + BboxColumnFieldNames +} - // infer bbox struct field names - var xminName string - var yminName string - var xmaxName string - var ymaxName string +// Returns a *BboxColumn struct that contains index, name and other data +// that describe the bounding box column contained in the schema. +// If there is no match for the standard name "bbox" in the schema, +// the covering metadata is consulted. +// An index field value of -1 (alongside an empty name field) means no bbox column found. +func GetBboxColumn(schema *schema.Schema, geoMetadata *Metadata) *BboxColumn { + bboxCol := &BboxColumn{} + // try standard name first + bboxCol.Name = "bbox" + bboxCol.Index = schema.Root().FieldIndexByName("bbox") - if metadata.Columns[metadata.PrimaryColumn].Covering != nil { - xminName = metadata.Columns[metadata.PrimaryColumn].Covering.Bbox.Xmin[1] - yminName = metadata.Columns[metadata.PrimaryColumn].Covering.Bbox.Ymin[1] - xmaxName = metadata.Columns[metadata.PrimaryColumn].Covering.Bbox.Xmax[1] - ymaxName = metadata.Columns[metadata.PrimaryColumn].Covering.Bbox.Ymax[1] + // if no match, check covering metadata + if bboxCol.Index == -1 { + if geoMetadata.Columns[geoMetadata.PrimaryColumn].Covering != nil && len(geoMetadata.Columns[geoMetadata.PrimaryColumn].Covering.Bbox.Xmin) == 2 { + bboxCol.Name = geoMetadata.Columns[geoMetadata.PrimaryColumn].Covering.Bbox.Xmin[0] + bboxCol.Index = schema.Root().FieldIndexByName(bboxCol.Name) } else { - // fallback to standard names - xminName = "xmin" - yminName = "ymin" - xmaxName = "xmax" - ymaxName = "ymax" + bboxCol.Name = "" } - - // loop over individual bbox values per record - for idx := 0; idx < col.Len(); idx++ { - var bbox map[string]json.RawMessage - if err := json.Unmarshal([]byte(col.ValueStr(idx)), &bbox); err != nil { - return nil, fmt.Errorf("trouble unmarshalling bbox struct: %w", err) - } - - bboxValue := &geo.Bbox{} // create empty struct to hold bbox values of this row - - if err := json.Unmarshal(bbox[xminName], &bboxValue.Xmin); err != nil { - return nil, fmt.Errorf("trouble parsing bbox.%v field: %w", xminName, err) - } - if err := json.Unmarshal(bbox[yminName], &bboxValue.Ymin); err != nil { - return nil, fmt.Errorf("trouble parsing bbox.%v field: %w", yminName, err) - } - if err := json.Unmarshal(bbox[xmaxName], &bboxValue.Xmax); err != nil { - return nil, fmt.Errorf("trouble parsing bbox.%v field: %w", xmaxName, err) - } - if err := json.Unmarshal(bbox[ymaxName], &bboxValue.Ymax); err != nil { - return nil, fmt.Errorf("trouble parsing bbox.%v field: %w", ymaxName, err) - } - - // check whether the bbox passed to this function - // intersects with the bbox of the record - maskBuilder.Append(inputBbox.Intersects(bboxValue)) - } - - r, filterErr := compute.FilterRecordBatch(ctx, *record, maskBuilder.NewBooleanArray(), &compute.FilterOptions{NullSelection: 0}) // TODO check what this is doing - if filterErr != nil { - return nil, fmt.Errorf("trouble filtering record batch: %w", filterErr) - } - filteredRecord = &r - } else if inputBbox != nil && bboxColIdx == -1 { - // bbox filter passed to function but there is no bbox col. - // this means we have to compute the bbox of the records ourselves - primaryColIdx := schema.ColumnIndexByName(metadata.PrimaryColumn) - col := (*record).Column(primaryColIdx) - defer col.Release() - - maskBuilder := array.NewBooleanBuilder(memory.DefaultAllocator) - defer maskBuilder.Release() - - for idx := 0; idx < col.Len(); idx++ { - value := col.GetOneForMarshal(idx) - g, decodeErr := geo.DecodeGeometry(value, metadata.Columns[metadata.PrimaryColumn].Encoding) - if decodeErr != nil { - return nil, fmt.Errorf("trouble decoding geometry: %w", decodeErr) - } - bounds := g.Coordinates.Bound() - bboxValue := &geo.Bbox{ - Xmin: bounds.Min.X(), - Ymin: bounds.Min.Y(), - Xmax: bounds.Max.X(), - Ymax: bounds.Max.Y(), - } - - // now that we've computed the bbox, same logic as above - maskBuilder.Append(inputBbox.Intersects(bboxValue)) - } - filter := maskBuilder.NewBooleanArray() - r, filterErr := compute.FilterRecordBatch(ctx, *record, filter, &compute.FilterOptions{NullSelection: 0}) // TODO check what this is doing - if filterErr != nil { - return nil, fmt.Errorf("trouble filtering record batch with computed bbox: %w (%v vs. %v)", filterErr, (*record).NumRows(), filter.Len()) - } - filteredRecord = &r - } else { - filteredRecord = record } - return filteredRecord, nil + + bboxCol.BaseColumn = schema.ColumnIndexByName(geoMetadata.PrimaryColumn) + bboxCol.BboxColumnFieldNames = *getBboxColumnFieldNames(geoMetadata) + return bboxCol } diff --git a/internal/geoparquet/geoparquet_test.go b/internal/geoparquet/geoparquet_test.go index 024a6fd..15e3e00 100644 --- a/internal/geoparquet/geoparquet_test.go +++ b/internal/geoparquet/geoparquet_test.go @@ -108,7 +108,7 @@ func TestRecordReaderV040(t *testing.T) { input, openErr := os.Open(fixturePath) require.NoError(t, openErr) - reader, err := geoparquet.NewRecordReader(&geoparquet.ReaderConfig{ + reader, err := geoparquet.NewRecordReaderFromConfig(&geoparquet.ReaderConfig{ Reader: input, }) require.NoError(t, err) @@ -131,7 +131,7 @@ func TestRowReaderV100Beta1(t *testing.T) { input, openErr := os.Open(fixturePath) require.NoError(t, openErr) - reader, err := geoparquet.NewRecordReader(&geoparquet.ReaderConfig{ + reader, err := geoparquet.NewRecordReaderFromConfig(&geoparquet.ReaderConfig{ Reader: input, }) require.NoError(t, err) @@ -152,14 +152,14 @@ func TestRowReaderV100Beta1(t *testing.T) { assert.Equal(t, 6, numCols) } -func TestRecordReaderV100ExcludeCols(t *testing.T) { +func TestRecordReaderV100Columns(t *testing.T) { fixturePath := "../testdata/cases/example-v1.0.0.parquet" input, openErr := os.Open(fixturePath) require.NoError(t, openErr) - reader, err := geoparquet.NewRecordReader(&geoparquet.ReaderConfig{ - Reader: input, - ExcludeColNames: []string{"continent", "gdp_md_est"}, + reader, err := geoparquet.NewRecordReaderFromConfig(&geoparquet.ReaderConfig{ + Reader: input, + Columns: []int{0, 1, 4, 5}, }) require.NoError(t, err) @@ -176,14 +176,14 @@ func TestRecordReaderV100ExcludeCols(t *testing.T) { assert.ElementsMatch(t, colNames, []string{"geometry", "pop_est", "iso_a3", "name"}) } -func TestRecordReaderV110IncludeCols(t *testing.T) { +func TestRecordReaderV110Columns(t *testing.T) { fixturePath := "../testdata/cases/example-v1.0.0.parquet" input, openErr := os.Open(fixturePath) require.NoError(t, openErr) - reader, err := geoparquet.NewRecordReader(&geoparquet.ReaderConfig{ - Reader: input, - IncludeColNames: []string{"geometry", "continent", "gdp_md_est"}, + reader, err := geoparquet.NewRecordReaderFromConfig(&geoparquet.ReaderConfig{ + Reader: input, + Columns: []int{0, 2, 3}, }) require.NoError(t, err) @@ -205,9 +205,9 @@ func TestRecordReaderV110NoGeomColError(t *testing.T) { input, openErr := os.Open(fixturePath) require.NoError(t, openErr) - reader, err := geoparquet.NewRecordReader(&geoparquet.ReaderConfig{ - Reader: input, - IncludeColNames: []string{"continent", "gdp_md_est"}, + reader, err := geoparquet.NewRecordReaderFromConfig(&geoparquet.ReaderConfig{ + Reader: input, + Columns: []int{2, 3}, }) require.ErrorContains(t, err, "geometry column") require.Nil(t, reader) @@ -459,7 +459,7 @@ func TestRecordReading(t *testing.T) { assert.Equal(t, reader.NumRows(), int64(numRows)) } -func TestGetBboxColumnIdxV100(t *testing.T) { +func TestGetBboxColumnV100(t *testing.T) { f, fileErr := os.Open("../testdata/cases/example-v1.0.0.parquet") require.NoError(t, fileErr) reader, readerErr := file.NewParquetReader(f) @@ -470,11 +470,18 @@ func TestGetBboxColumnIdxV100(t *testing.T) { require.NoError(t, err) // no bbox col in the file, we expect -1 - colIdx := geoparquet.GetBboxColumnIndex(reader.MetaData().Schema, metadata) - require.Equal(t, -1, colIdx) + bboxCol := geoparquet.GetBboxColumn(reader.MetaData().Schema, metadata) + assert.Equal(t, -1, bboxCol.Index) + assert.Equal(t, "", bboxCol.Name) + assert.Equal(t, 0, bboxCol.BaseColumn) + assert.Equal(t, "", bboxCol.BaseColumnEncoding) + assert.Equal(t, "xmin", bboxCol.BboxColumnFieldNames.Xmin) + assert.Equal(t, "ymin", bboxCol.BboxColumnFieldNames.Ymin) + assert.Equal(t, "xmax", bboxCol.BboxColumnFieldNames.Xmax) + assert.Equal(t, "ymax", bboxCol.BboxColumnFieldNames.Ymax) } -func TestGetBboxColumnIdxV110(t *testing.T) { +func TestGetBboxColumnV110(t *testing.T) { f, fileErr := os.Open("../testdata/cases/example-v1.1.0.parquet") require.NoError(t, fileErr) reader, readerErr := file.NewParquetReader(f) @@ -485,8 +492,15 @@ func TestGetBboxColumnIdxV110(t *testing.T) { require.NoError(t, err) // there is a bbox col in the file, we expect index 6 - colIdx := geoparquet.GetBboxColumnIndex(reader.MetaData().Schema, metadata) - require.Equal(t, 6, colIdx) + bboxCol := geoparquet.GetBboxColumn(reader.MetaData().Schema, metadata) + assert.Equal(t, 6, bboxCol.Index) + assert.Equal(t, "bbox", bboxCol.Name) + assert.Equal(t, 5, bboxCol.BaseColumn) + assert.Equal(t, "", bboxCol.BaseColumnEncoding) + assert.Equal(t, "xmin", bboxCol.BboxColumnFieldNames.Xmin) + assert.Equal(t, "ymin", bboxCol.BboxColumnFieldNames.Ymin) + assert.Equal(t, "xmax", bboxCol.BboxColumnFieldNames.Xmax) + assert.Equal(t, "ymax", bboxCol.BboxColumnFieldNames.Ymax) } func TestGetBboxColumnIdxV110NonStandardBboxCol(t *testing.T) { @@ -501,17 +515,24 @@ func TestGetBboxColumnIdxV110NonStandardBboxCol(t *testing.T) { // there is a bbox col in the file with the non-standard name "geometry_bbox", // we expect index 6 - colIdx := geoparquet.GetBboxColumnIndex(reader.MetaData().Schema, metadata) - assert.Equal(t, 6, colIdx) + bboxCol := geoparquet.GetBboxColumn(reader.MetaData().Schema, metadata) + assert.Equal(t, 6, bboxCol.Index) + assert.Equal(t, "geometry_bbox", bboxCol.Name) + assert.Equal(t, 5, bboxCol.BaseColumn) + assert.Equal(t, "", bboxCol.BaseColumnEncoding) + assert.Equal(t, "xmin", bboxCol.BboxColumnFieldNames.Xmin) + assert.Equal(t, "ymin", bboxCol.BboxColumnFieldNames.Ymin) + assert.Equal(t, "xmax", bboxCol.BboxColumnFieldNames.Xmax) + assert.Equal(t, "ymax", bboxCol.BboxColumnFieldNames.Ymax) assert.Equal(t, 6, reader.MetaData().Schema.Root().FieldIndexByName("geometry_bbox")) } -func TestFilterByBboxV100(t *testing.T) { +func TestFilterRecordBatchByBboxV100(t *testing.T) { fileReader, fileErr := os.Open("../testdata/cases/example-v1.0.0.parquet") require.NoError(t, fileErr) defer fileReader.Close() - recordReader, err := geoparquet.NewRecordReader(&geoparquet.ReaderConfig{Reader: fileReader}) + recordReader, err := geoparquet.NewRecordReaderFromConfig(&geoparquet.ReaderConfig{Reader: fileReader}) require.NoError(t, err) defer recordReader.Close() @@ -522,7 +543,16 @@ func TestFilterByBboxV100(t *testing.T) { inputBbox := &geo.Bbox{Xmin: 34.0, Ymin: -7.0, Xmax: 36.0, Ymax: -6.0} - filteredRecord, err := geoparquet.FilterRecordBatchByBbox(context.Background(), recordReader, &record, inputBbox) + filteredRecord, err := geoparquet.FilterRecordBatchByBbox(context.Background(), &record, inputBbox, &geoparquet.BboxColumn{ + Index: -1, + BaseColumn: 0, + BboxColumnFieldNames: geoparquet.BboxColumnFieldNames{ + Xmin: "xmin", + Ymin: "ymin", + Xmax: "xmax", + Ymax: "ymax", + }, + }) require.NoError(t, err) // we expect only one row, namely Tanzania @@ -533,12 +563,12 @@ func TestFilterByBboxV100(t *testing.T) { assert.Equal(t, "Tanzania", country) } -func TestFilterByBboxV110(t *testing.T) { +func TestFilterRecordBatchByBboxV110(t *testing.T) { fileReader, fileErr := os.Open("../testdata/cases/example-v1.1.0.parquet") require.NoError(t, fileErr) defer fileReader.Close() - recordReader, err := geoparquet.NewRecordReader(&geoparquet.ReaderConfig{Reader: fileReader}) + recordReader, err := geoparquet.NewRecordReaderFromConfig(&geoparquet.ReaderConfig{Reader: fileReader}) require.NoError(t, err) defer recordReader.Close() @@ -549,7 +579,17 @@ func TestFilterByBboxV110(t *testing.T) { inputBbox := &geo.Bbox{Xmin: 34.0, Ymin: -7.0, Xmax: 36.0, Ymax: -6.0} - filteredRecord, err := geoparquet.FilterRecordBatchByBbox(context.Background(), recordReader, &record, inputBbox) + filteredRecord, err := geoparquet.FilterRecordBatchByBbox(context.Background(), &record, inputBbox, &geoparquet.BboxColumn{ + Index: 6, + BaseColumn: 5, + BaseColumnEncoding: "wkb", + BboxColumnFieldNames: geoparquet.BboxColumnFieldNames{ + Xmin: "xmin", + Ymin: "ymin", + Xmax: "xmax", + Ymax: "ymax", + }, + }) require.NoError(t, err) // we expect only one row, namely Tanzania @@ -560,12 +600,12 @@ func TestFilterByBboxV110(t *testing.T) { assert.Equal(t, "Tanzania", country) } -func TestFilterByBboxV110NonStandardBboxCol(t *testing.T) { +func TestFilterRecordBatchByBboxV110NonStandardBboxCol(t *testing.T) { fileReader, fileErr := os.Open("../testdata/cases/example-v1.1.0-covering.parquet") require.NoError(t, fileErr) defer fileReader.Close() - recordReader, err := geoparquet.NewRecordReader(&geoparquet.ReaderConfig{Reader: fileReader}) + recordReader, err := geoparquet.NewRecordReaderFromConfig(&geoparquet.ReaderConfig{Reader: fileReader}) require.NoError(t, err) defer recordReader.Close() @@ -576,7 +616,17 @@ func TestFilterByBboxV110NonStandardBboxCol(t *testing.T) { inputBbox := &geo.Bbox{Xmin: 34.0, Ymin: -7.0, Xmax: 36.0, Ymax: -6.0} - filteredRecord, err := geoparquet.FilterRecordBatchByBbox(context.Background(), recordReader, &record, inputBbox) + filteredRecord, err := geoparquet.FilterRecordBatchByBbox(context.Background(), &record, inputBbox, &geoparquet.BboxColumn{ + Index: 6, + BaseColumn: 5, + BaseColumnEncoding: "wkb", + BboxColumnFieldNames: geoparquet.BboxColumnFieldNames{ + Xmin: "xmin", + Ymin: "ymin", + Xmax: "xmax", + Ymax: "ymax", + }, + }) require.NoError(t, err) // we expect only one row, namely Tanzania @@ -586,3 +636,144 @@ func TestFilterByBboxV110NonStandardBboxCol(t *testing.T) { country := (*filteredRecord).Column(recordReader.Schema().ColumnIndexByName("name")).ValueStr(0) assert.Equal(t, "Tanzania", country) } + +// func TestFilterByBboxV100(t *testing.T) { +// fileReader, fileErr := os.Open("../testdata/cases/example-v1.0.0.parquet") +// require.NoError(t, fileErr) +// defer fileReader.Close() + +// recordReader, err := geoparquet.NewRecordReaderFromConfig(&geoparquet.ReaderConfig{Reader: fileReader}) +// require.NoError(t, err) +// defer recordReader.Close() + +// record, readErr := recordReader.Read() +// require.NoError(t, readErr) +// assert.Equal(t, int64(6), record.NumCols()) +// assert.Equal(t, int64(5), record.NumRows()) + +// inputBbox := &geo.Bbox{Xmin: 34.0, Ymin: -7.0, Xmax: 36.0, Ymax: -6.0} + +// output := &bytes.Buffer{} +// recordWriter, rwErr := geoparquet.NewRecordWriter(&geoparquet.WriterConfig{ +// Writer: output, +// Metadata: recordReader.Metadata(), +// ArrowSchema: recordReader.ArrowSchema(), +// }) +// require.NoError(t, rwErr) +// defer recordWriter.Close() + +// filterErr := geoparquet.FilterByBbox(context.Background(), recordReader, recordWriter, inputBbox) +// require.NoError(t, filterErr) + +// NewRecordReaderFromConfig, err := geoparquet.NewRecordReaderFromConfig(&geoparquet.ReaderConfig{ +// Reader: bytes.NewReader(output.Bytes()), +// }) +// require.NoError(t, err) +// defer NewRecordReaderFromConfig.Close() + +// assert.Equal(t, int64(1), NewRecordReaderFromConfig.NumRows()) + +// filteredRecord, readErr := NewRecordReaderFromConfig.Read() +// require.NoError(t, readErr) + +// // we expect only one row, namely Tanzania +// assert.Equal(t, int64(6), filteredRecord.NumCols()) +// assert.Equal(t, int64(1), filteredRecord.NumRows()) + +// country := filteredRecord.Column(recordReader.Schema().ColumnIndexByName("name")).ValueStr(0) +// assert.Equal(t, "Tanzania", country) +// } + +// func TestFilterByBboxV110(t *testing.T) { +// fileReader, fileErr := os.Open("../testdata/cases/example-v1.1.0.parquet") +// require.NoError(t, fileErr) +// defer fileReader.Close() + +// recordReader, err := geoparquet.NewRecordReaderFromConfig(&geoparquet.ReaderConfig{Reader: fileReader}) +// require.NoError(t, err) +// defer recordReader.Close() + +// record, readErr := recordReader.Read() +// require.NoError(t, readErr) +// assert.Equal(t, int64(7), record.NumCols()) +// assert.Equal(t, int64(5), record.NumRows()) + +// inputBbox := &geo.Bbox{Xmin: 34.0, Ymin: -7.0, Xmax: 36.0, Ymax: -6.0} + +// output := &bytes.Buffer{} +// recordWriter, rwErr := geoparquet.NewRecordWriter(&geoparquet.WriterConfig{ +// Writer: output, +// Metadata: recordReader.Metadata(), +// ArrowSchema: recordReader.ArrowSchema(), +// }) +// require.NoError(t, rwErr) +// defer recordWriter.Close() + +// filterErr := geoparquet.FilterByBbox(context.Background(), recordReader, recordWriter, inputBbox) +// require.NoError(t, filterErr) + +// NewRecordReaderFromConfig, err := geoparquet.NewRecordReaderFromConfig(&geoparquet.ReaderConfig{ +// Reader: bytes.NewReader(output.Bytes()), +// }) +// require.NoError(t, err) +// defer NewRecordReaderFromConfig.Close() + +// assert.Equal(t, int64(1), NewRecordReaderFromConfig.NumRows()) + +// filteredRecord, readErr := NewRecordReaderFromConfig.Read() +// require.NoError(t, readErr) + +// // we expect only one row, namely Tanzania +// assert.Equal(t, int64(7), filteredRecord.NumCols()) +// assert.Equal(t, int64(1), filteredRecord.NumRows()) + +// country := filteredRecord.Column(recordReader.Schema().ColumnIndexByName("name")).ValueStr(0) +// assert.Equal(t, "Tanzania", country) +// } + +// func TestFilterByBboxV110NonStandardBboxCol(t *testing.T) { +// fileReader, fileErr := os.Open("../testdata/cases/example-v1.1.0-covering.parquet") +// require.NoError(t, fileErr) +// defer fileReader.Close() + +// recordReader, err := geoparquet.NewRecordReaderFromConfig(&geoparquet.ReaderConfig{Reader: fileReader}) +// require.NoError(t, err) +// defer recordReader.Close() + +// record, readErr := recordReader.Read() +// require.NoError(t, readErr) +// assert.Equal(t, int64(7), record.NumCols()) +// assert.Equal(t, int64(5), record.NumRows()) + +// inputBbox := &geo.Bbox{Xmin: 34.0, Ymin: -7.0, Xmax: 36.0, Ymax: -6.0} + +// output := &bytes.Buffer{} +// recordWriter, rwErr := geoparquet.NewRecordWriter(&geoparquet.WriterConfig{ +// Writer: output, +// Metadata: recordReader.Metadata(), +// ArrowSchema: recordReader.ArrowSchema(), +// }) +// require.NoError(t, rwErr) +// defer recordWriter.Close() + +// filterErr := geoparquet.FilterByBbox(context.Background(), recordReader, recordWriter, inputBbox) +// require.NoError(t, filterErr) + +// NewRecordReaderFromConfig, err := geoparquet.NewRecordReaderFromConfig(&geoparquet.ReaderConfig{ +// Reader: bytes.NewReader(output.Bytes()), +// }) +// require.NoError(t, err) +// defer NewRecordReaderFromConfig.Close() + +// assert.Equal(t, int64(1), NewRecordReaderFromConfig.NumRows()) + +// filteredRecord, readErr := NewRecordReaderFromConfig.Read() +// require.NoError(t, readErr) + +// // we expect only one row, namely Tanzania +// assert.Equal(t, int64(7), filteredRecord.NumCols()) +// assert.Equal(t, int64(1), filteredRecord.NumRows()) + +// country := filteredRecord.Column(recordReader.Schema().ColumnIndexByName("name")).ValueStr(0) +// assert.Equal(t, "Tanzania", country) +// } diff --git a/internal/geoparquet/metadata.go b/internal/geoparquet/metadata.go index 4d1ce27..9fd5e26 100644 --- a/internal/geoparquet/metadata.go +++ b/internal/geoparquet/metadata.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" + "github.com/apache/arrow/go/v16/parquet/file" "github.com/apache/arrow/go/v16/parquet/metadata" "github.com/planetlabs/gpq/internal/geo" ) @@ -192,3 +193,7 @@ func GetMetadataValue(keyValueMetadata metadata.KeyValueMetadata) (string, error } return *value, nil } + +func GetMetadataFromFileReader(fileReader *file.Reader) (*Metadata, error) { + return GetMetadata(fileReader.MetaData().GetKeyValueMetadata()) +} diff --git a/internal/geoparquet/recordreader.go b/internal/geoparquet/recordreader.go index 427de53..b2e6f5e 100644 --- a/internal/geoparquet/recordreader.go +++ b/internal/geoparquet/recordreader.go @@ -19,12 +19,12 @@ const ( ) type ReaderConfig struct { - BatchSize int - Reader parquet.ReaderAtSeeker - File *file.Reader - Context context.Context - ExcludeColNames []string - IncludeColNames []string + BatchSize int + Reader parquet.ReaderAtSeeker + File *file.Reader + Context context.Context + Columns []int + RowGroups []int } type RecordReader struct { @@ -33,72 +33,44 @@ type RecordReader struct { recordReader pqarrow.RecordReader } -// A Set type based on map, to hold arrow column indices. -// Implements common Set methods such as Difference() and Contains(). -// To instantiate, use the constructor newIndicesSet() followed by either -// Add() if you want to build the Set sequentially or the convenience function -// FromColNames(). -type indicesSet map[int]struct{} - -func newIndicesSet(size int) *indicesSet { - var s indicesSet = make(map[int]struct{}, size) - return &s -} - -func (s *indicesSet) Add(col int) *indicesSet { - (*s)[col] = struct{}{} - return s -} - -func (s *indicesSet) FromColNames(cols []string, schema *arrow.Schema) *indicesSet { - for _, col := range cols { - if indicesForColumn := schema.FieldIndices(col); indicesForColumn != nil { - for _, colIdx := range indicesForColumn { - s.Add(colIdx) - } +func NewParquetFileReader(config *ReaderConfig) (*file.Reader, error) { + fileReader := config.File + if fileReader == nil { + if config.Reader == nil { + return nil, errors.New("config must include a File or Reader value") } + fr, frErr := file.NewParquetReader(config.Reader) + if frErr != nil { + return nil, frErr + } + fileReader = fr } - return s + return fileReader, nil } -func (s *indicesSet) Contains(col int) bool { - _, ok := (*s)[col] - return ok -} - -func (s *indicesSet) Difference(other *indicesSet) *indicesSet { - sSize := s.Size() - otherSize := s.Size() - var newSet *indicesSet - if sSize < otherSize { - newSet = newIndicesSet(otherSize - sSize) - } else { - newSet = newIndicesSet(sSize - otherSize) - } - for key := range *s { - if !other.Contains(key) { - newSet.Add(key) - } +func NewArrowFileReader(config *ReaderConfig, parquetReader *file.Reader) (*pqarrow.FileReader, error) { + batchSize := config.BatchSize + if batchSize == 0 { + batchSize = defaultReadBatchSize } - return newSet -} -func (s *indicesSet) Size() int { - return len(*s) + return pqarrow.NewFileReader(parquetReader, pqarrow.ArrowReadProperties{BatchSize: int64(batchSize)}, memory.DefaultAllocator) } -func (s *indicesSet) List() []int { - keys := make([]int, 0, len(*s)) - for k := range *s { - keys = append(keys, k) +func NewRecordReaderFromConfig(config *ReaderConfig) (*RecordReader, error) { + parquetFileReader, err := NewParquetFileReader(config) + if err != nil { + return nil, fmt.Errorf("could not get ParquetFileReader: %w", err) } - return keys -} -func NewRecordReader(config *ReaderConfig) (*RecordReader, error) { - batchSize := config.BatchSize - if batchSize == 0 { - batchSize = defaultReadBatchSize + arrowFileReader, err := NewArrowFileReader(config, parquetFileReader) + if err != nil { + return nil, fmt.Errorf("could not get ArrowFileReader: %w", err) + } + + geoMetadata, err := GetMetadataFromFileReader(parquetFileReader) + if err != nil { + return nil, fmt.Errorf("could not get geo metadata from file reader: %w", err) } ctx := config.Context @@ -106,74 +78,61 @@ func NewRecordReader(config *ReaderConfig) (*RecordReader, error) { ctx = context.Background() } - fileReader := config.File - if fileReader == nil { - if config.Reader == nil { - return nil, errors.New("config must include a File or Reader value") - } - fr, frErr := file.NewParquetReader(config.Reader) - if frErr != nil { - return nil, frErr + if config.Columns != nil { + primaryGeomColIdx := parquetFileReader.MetaData().Schema.ColumnIndexByName(geoMetadata.PrimaryColumn) + + if !slices.Contains(config.Columns, primaryGeomColIdx) { + return nil, fmt.Errorf("columns must include primary geometry column '%v' (index %v)", geoMetadata.PrimaryColumn, primaryGeomColIdx) } - fileReader = fr } - geoMetadata, geoMetadataErr := GetMetadata(fileReader.MetaData().GetKeyValueMetadata()) - if geoMetadataErr != nil { - return nil, geoMetadataErr + if config.Columns != nil && len(config.Columns) == 0 { + config.Columns = nil } - arrowReader, arrowErr := pqarrow.NewFileReader(fileReader, pqarrow.ArrowReadProperties{BatchSize: int64(batchSize)}, memory.DefaultAllocator) - if arrowErr != nil { - return nil, arrowErr + if config.RowGroups != nil && len(config.RowGroups) == 0 { + config.RowGroups = nil } - var recordReader pqarrow.RecordReader - var recordErr error + recordReader, recordErr := arrowFileReader.GetRecordReader(ctx, config.Columns, config.RowGroups) - excludeColNamesProvided := len(config.ExcludeColNames) > 0 - includeColNamesProvided := len(config.IncludeColNames) > 0 - if excludeColNamesProvided || includeColNamesProvided { - if excludeColNamesProvided == includeColNamesProvided { - return nil, errors.New("config must only contain one of ExcludeColNames/IncludeColNames") - } + if recordErr != nil { + return nil, recordErr + } - schema, schemaErr := arrowReader.Schema() - if schemaErr != nil { - return nil, schemaErr - } + reader := &RecordReader{ + fileReader: arrowFileReader.ParquetReader(), + metadata: geoMetadata, + recordReader: recordReader, + } + return reader, nil +} + +func NewRecordReader(ctx context.Context, arrowFileReader *pqarrow.FileReader, geoMetadata *Metadata, columns []int, rowGroups []int) (*RecordReader, error) { + if columns != nil || len(columns) != 0 { + primaryGeomColIdx := arrowFileReader.ParquetReader().MetaData().Schema.ColumnIndexByName(geoMetadata.PrimaryColumn) - if excludeColNamesProvided { - if slices.Contains(config.ExcludeColNames, geoMetadata.PrimaryColumn) { - return nil, fmt.Errorf("can't exclude primary geometry column '%v'", geoMetadata.PrimaryColumn) - } - - // generate indices from col names and compute the indices to include - indicesToExclude := newIndicesSet(schema.NumFields()-len(config.ExcludeColNames)).FromColNames(config.ExcludeColNames, schema) - allIndices := newIndicesSet(schema.NumFields()) - for i := 0; i < schema.NumFields(); i++ { - allIndices.Add(i) - } - indices := allIndices.Difference(indicesToExclude) - recordReader, recordErr = arrowReader.GetRecordReader(ctx, indices.List(), nil) - } else { - if !slices.Contains(config.IncludeColNames, geoMetadata.PrimaryColumn) { - return nil, fmt.Errorf("column names must include primary geometry column '%v'", geoMetadata.PrimaryColumn) - } - - // generate indices from col names - indices := newIndicesSet(len(config.IncludeColNames)).FromColNames(config.IncludeColNames, schema) - recordReader, recordErr = arrowReader.GetRecordReader(ctx, indices.List(), nil) + if !slices.Contains(columns, primaryGeomColIdx) { + return nil, fmt.Errorf("columns (%v) must include primary geometry column '%v' (index %v)", columns, geoMetadata.PrimaryColumn, primaryGeomColIdx) } - } else { - recordReader, recordErr = arrowReader.GetRecordReader(ctx, nil, nil) } + + if columns != nil && len(columns) == 0 { + columns = nil + } + + if rowGroups != nil && len(rowGroups) == 0 { + rowGroups = nil + } + + recordReader, recordErr := arrowFileReader.GetRecordReader(ctx, columns, rowGroups) + if recordErr != nil { return nil, recordErr } reader := &RecordReader{ - fileReader: fileReader, + fileReader: arrowFileReader.ParquetReader(), metadata: geoMetadata, recordReader: recordReader, } diff --git a/internal/testdata/cases/example-v1.1.0-partitioned.parquet b/internal/testdata/cases/example-v1.1.0-partitioned.parquet new file mode 100644 index 0000000000000000000000000000000000000000..93a5444d2f286caa9f2831b4447dd3ede9e267cb GIT binary patch literal 30566 zcmeFZXH-<%vIg2MjW*b{h=72A9FP!HAUqQO#Q zvADV{7VFTl%6Z-zEDbdcO>4TYs?k-WtF9LJPsv}uG+7q^!&SOZ$)F4U@4{wxVNw0+ z8d`tqS2pQN1>Wf+vQm@(-hwmBqU-Mxt$%(0qb4m4-9I()teK{z$-~Ac56k+W>f-(> z)B5+Xf7HjLmDA_%1JmOEDf#OcGj;#G zN)P|{)U8+1*sAiUi47`Z_M|-!$%wJXZda=(4_Ej?)5*Y7OFIlO zbLnk^C8vDBKxfv%LRTYHHL;jn^u-sF?(~_MY;S^#?dzV+wD5xu9pBc+tT(|%{Y=GK zL_^=yw+vmj>Xbp%Nna><*m1|8W(dP(^=W(X8k=L=&g-TwsXkCNbFg!UtvTL1IX&lsw-4N&(yu{3&m3 zW^Y&;w(R`PCVGCuszW;hy`gCFZBTL&@$B+ z1=_l%yB+`(s6^~qdffb z=>aTr$tV6!4Xk(NYbZx~Ky&itP2Ya?L^v8}`y4%>K)bb{*QK7=`k>bIjGiapy(>cm zajMvKxvuBXsh&^}(6DV}5f6pOKA&HVM6++dzW-1en^yX~a5_eGhL_kcfQ!P+bJgmz ziLRYEjyAPc+qE$!AVbOUHkK1P-qv- zO0gaIw$T&39yQEDn{USpIOU^)htmBAtcAtqACBXcY4bEr6Fu$mnUX=R#~bTxL`Uy? z0c&zE<(|2%$N$AN~DY)8hUq`Rf;>g#PpDU-ftSt-t?Q{eSd>f7bt}M2q|9 z7gPVA-+x{G-|Np{4%z?VT-M_LDf#OcQ~y7&{_pkw(d+)%|34*K+&{mV`v3g?>*_!1 z*VNGYqoQ}W=tjkZlShskJYv}2n{Z}Xbo*D4)}J4LRrW7q2{a&7gWGA08R$|e>!d|L zZ#I|vD$|I~waqPTnd(g!H_R#C%UCL8T3OTG|6!tamUNnEEPBM78N#3Y|GG+7luZ~D zjmz$$@RYI9LN$b;Walw9pxIJp3=ye=8N=!?b4`UVh7TVxD$~F_XYjx5xBqIM8Q#C1 z`VRx{uXZY!cE)Io|1-Qn%^a#SAZtjL5`9P67)X*clSg#6=$kw&IW1X>F3W~xjT)S0 zAQ_cBYVb&d;pzWN<4XUoLaXcV3h4R2n(rK@i-y~u=5;#_zJTB$b~o^s2pFTv*E^i~ zl1ctCwKO&bx~!!@xBRnK&~X1#s{uD(gQdvk%u`V82CSEpb?9yuYi{>IHuL?@75h`7 z#hu4AO&?~({&|%O{x+1I=euZ>?d|fX;T4Xgnn^`fYDz>7cDt&QTDr}2_bQp2N ze${}Uwy3DqDkpO5OW*BA3vBUT*XjJtL(|~`D{rpp1zSuXyL*>IR65LGdD>D#!46k8 zL728A9h5v)$<}SK#n+c-WcN-@hsf^jE3}8(qW$&nV+ZA=L+Q9ldzF&4lX~WA%x|5= zEa-%xN!vJnRI)qMZO~LZG^zeRyDcOg)_VK?;zZeDAGLP**T{6}bIa<)<1#y(cKXrz zU7~b2tMf^9zL`BbG}K4m@<<2gCBqL?2ifB=vMU{})4^Qbs7ckz9=Duux=RHV+dj=a;9GAEVOg$AI!p@uSl_BT`*i~H?Qpgo7sR(iH+v$B6I7d>(Xfz!L~QLW|} zGKJ&R`qmXPU5cLEeQ1ubd0^(V#DVbMuv^E<clTC$gfj>g1?#-(`{ByVOWL1R4FpxA3Y*C+OBAj)i!wXy z0<2Gg>yG!ZK-S#5x1M)%13}rsg2Thj@%*jNxKYOq&~HuK_j)r-N`96qf7as3x<{NV-`JI6YGPclIjn_Umn@3}$cvW7R4^Ns&aCr$3NY1~yKWCebW-=sMR zw0dYMeOqpb<5aEMp6L$)RYmW?hu8nLKD1u;Rcy`xhn&GIeVf4wwraF1IxONAp4^{+ zs^02tS?mG%q2wP*9v#yE1V_65hL4vbO40C``+bKc2KL3BPFf=Fe~pgYzGo-fpi?LIXqh zJ1)b*fOm64uE&EUEV*!IfBE__SjNBc%k^Cn?xtzD{xl3qUYxie-6I(bnRFsH`FJJ_vKGCW=#`wDHN7=}6%NLCzf2xdkc<;@$KDub z91bB%#Eoy(C8OEP(uRTN;qaMR9-^JSuT5eV2qA6I!-)Ar8&-|TT$Ce%+|nmbY*n7X zt|pnS+9qI+5S|RZo0^QZtv&5b6GYHHzWvMuyJXy3a%*Comk5U4zaY_5O~xPV$jjg= zf{LftUAwj=q5aZcZ>*#uSpQMyh~I@I?40FH5!|Eh)vTe}jaoJ-DoF%8S5Dk}AtwoU z3{bl>VxkCY#%=W-=9+}x$z?XUNCb1E2h18_oP^%CfeRnzi6HWQ!1G1jlQ8V=@jX`x zMBw&v(vzW_By8x*b8lZOf{Sv;HEZrAV)U>qCzBEpJe9DDGIA5q?97@gC#e65z55NZ z?URUaU!$w>T&jP`H@&%P#9iy-XD*Evfyrvkn&#RB9M@IlN3S#yxbCYTb8K4zUfA`V zbz+PNk}j~Pxh5oFL-ehy{E;GvQ;0Q{e~-s|Yxdu@k&8fKl2OFs^mvrk9lvx4L?Csd ziRQ=S2tz?zKSd`IjF(qBUAPp76=&Y%e5DBs$&=k4J1!1CueiTtwzdc?4#`E+ug2lw zlO$!o2%-CxBLAz2aX4c@=qBYBAqeNM7^b#27WY_2<*)cC1o_Be&tdP^t@8PmzGeuxJx=SfnTaMSw!rzWP zFM{UV+rP$Lh(b%REuUY06+s*E_ow_QoG_ghs8s|StS8!zE>XW3_?ie(vJLCKA4Q^g zdX|i3zNA8#&#O?#J|5UEGv2~W4t&)R7|rwtpBbwh?P zeS2R!cwPw6YGs?2FOlI|lX!UIYst-fhcBmSJK-!K6|7 zBIp)K-o>smEH5Yb9x=E*qa19OBB=^*4TgvyV&Asn4iQNm{^K>pofTj5znP7J$rf&?gY#tv0%;|FUTO_VHed_s> zArYX)Fidb1KKDDKY^e|d1CS6i$0V?%#Q# zu<-i{99bQS8(y}gvwb45YP+MQVooTypUgTjuS+D}>lxlX!95fhd3~J5 z-$V8Md5+(fgAGL(8?=By3p-Bwo?lv0;G(Ij0xeja?K1w-d)i=m$uU znJ?SdArRmbdvT+Y1k?OgHYNLo!f+D+3Y-VtC({Ewr>IOUsL z?nRdfxE@PKhMV{|L#RZ+M!T3m*=rH1j(6VXSsM<`PGjc0x+y|MlIIH+g+o(ydv3#E z5#Fa}@68T}#mteJA;M}VJfs^AHimuMvc)32>~-Sw-0@-1)W5i_qqhi|;mo1&y<6D6 zb=gZH_8#n9Z_+0WEZOyKx4RSxG2_Dp=Yc6S0Y9eQuZj?2w%Nn%k?+Ew-Pk`(ZYsn% zC2B!!R^gC%IEdHfMFd_ieXh6pP#Dx@>dC?&0yCS}HKk~TLGYRavy*$m@uT;p^y?Es z;atSlNp8=>Q6T)THL_nQd|(cqLE(5m`p6rfCn4~7@sseE^hEvI`M^A#``m#$O+|@%-?s@KG)0JT8RP35U z@T6J~i&g7`p+4c=&-|MqIM-yn_Gz_X7&_#3Orm2bPGstR90apZ(FzxbVy~k`Db@Rf zV6O>;^-m2$)}qPZ+zNwWArlwjgyRa=c+FmYgTU^9>eI4&;W$i`K0Q=92h8R0)e7Avt-30OlQ_>a^UY1 z*^+4HV09LAr=!KC?yJ@YAGGL zfkJE_-L$%=k^oeVX9V_f5Mn}`Muqm2KzQ;nBhE!%i000HZ_d&Wgi1z}ULJuI?Qtr3 z5CF9UK7HJDG8`|g=$+K}eE_IAyB`(%grgB1U^(>oq)^O$n+OpuO9^};tPX^8 z$NEkCHARG-nwLBfT151~5)m@dTzwk9GGhR272yO%vp*aFEXL9~EkZcBPe*NY0En6T zZ;OyoTsD*ifN^>M7vd%nK7P4u4{vG!$Vegbi4@~x@tdpf3j$zNCRqRn#dyf9@Nf^J zHlwimCX4b?; zf2jX9WSZ-62|C?8{&WV>r*^lfYVVX_<5i=#U5FlhjlcK%SqWC}ThnFt5r4>I`iUeE z$!=P>-XDICOM=XBXHM`3ogYJ{>DQ1%&+L17?l6B4E;?&-qlXm5Ey^1n+W3R6$LN)H zq*0U}?-_qW#~&IW2n_UAOVPIL$D3I<{NUEWLD^Q@|Dsu$mouABNb#28UEa!9e(>({ zwV9d?QZ#dD(R=dU4+PA*oRMPZJS+SE-Y?ZsWTsttpg(l>i%v98=7vfBP*V59@DiP9 zwjrg1wHy2)DcSAinq^W%M!>oRzzomP(LNibSokg{d&2erkiP2c_x_m_g+WOj3!4Mr zR{Iitl?PIEDX@Jk`V{~MlcrP+dm+X1$=k-X^8;bP$jZow9=1$fxyzzR(@zl^{E#xC=vxi@rUT$JzePeyGC!FMg~G&|1%=YlHpT+q@nHV zK#-sN;k{^$47-l0Idiox5bS1sT4;Jt_8V`|5eU0)UQ=|?B8};0Gpml|%lD5p}<|PVJDxHgJJ#;r|1pga;%=;wN;bo)l9pV(a8Ad zSVSA-=ad5i^d|E7ycuxKd6Yv%Saq1viR zyZGfDuc69sS0yj28LOwKYEPqXy;=t|zFnR-Ke*(V6el+BjsD~s0kU3x_q#Pp@$q@G zCr3xX6}PRW2KiFF6y7zkVPpi1WcKnnDX#ccJo#Hp1env2oO()%K*mn8dIYrZt4&<; zUWzUE`@WNra>4Ul-^WEUmy7xjnd=1u6b+oqSo+@T{PP;Ed8JFq3{jV&EQ$_k)AD?Ggc{OPzCB_VZb-c?kpNrwz=ALfQVDfJ4W` zkhuKgS>?G=xa@m4`{hP4c#a#kuI+ggej3kuzGEv*)IweJxsGx?vT$ergd<{@*iSMr zUrmk;;)+O{TVgof&F=ch15tSUnSw(0OEEO{_DNqdnYcASb=Q|GVn`T3Eo;p8mbNBq zZsx2$IXViT>{z}&Fj)e19&E4Ii=)u>&O(b3i4xeW_i*)HWjQ{&@!|cfR0*7WwJocq zj~w+Zw-pZ=EdjOC;;G$NMPcgZxKHCpN#ITPia~vcMWMpZ<&R4;C7`*vbFuk1G+U=` zay`>iU`uA|EVo_?yCty1XRXt@Cs7z%>$mlEsRR<*rp-706oq=hg#kS3s<2h@=?68>-au5 z63vxs@BSdtqfyTL~zn zWACZ~avYqZWhV`mfM0}~YCYLp;|%jQ_y$Oz%h+LkkEBt3+3&9Hw39&3$NasAo=0N! zVP%i2L?;&NW{lqziC68WIo47AI&03p#qh87a;kE=;QgXC`S8U(8(JD;@lBZ#_v{=A zB(a`H?iv$|m(+jM*G`eZ&&?mtex4VL1D1z8U6n0?%kGkc0Sa+A;e4NqwlOrl?4UBc zlsKF^wl4qPFbPWFiNl8U{TY75dxtL)-`Y=NwI(aBuU@nLXe^GajQccUuLLrc zkABS^5{p@!mV>i4NkBC8<;aXfu^84;_C=Yv`1LPA*>0ma+_k^4M<1d&GacS9R3_tO4TAFrMd!2rLA7t1PxpwI2+ z8Wuf6G5g%jckiNtAotLzkq-xlqW#WyhrhfK07v@GSELq-e8;=IfxUu&%W1kBT@!+Z z$64p=+XbNUJSNxgWe66GujyufP5>rT7vY|`j~1Z9dv6ow^z7_;HZ6;mELCr zz*m^G_*YFZUVO8>{>eH4SXBtMS3eEL2^V_aH}xUKyD28O>%3s}T;csuNiKjwm$2{q z#s{O#H-)$Oz(M{I2d)lZN6P}Di9t_KiB_GVKDBSYMb&xLW=ckJN4rygK=p1 z4;zz+K(J*u#lFxC!TLK!Z^JSI!RoHTbp3C^=<#ZKeb>f7m>54?zg|BCySnbF3T_I7 zP0PIsL-&T@J_n8Qfq8+j^T4T*JATpl#HE9)M+d^(L&;kMHilqd;SZbpy943GdQEE` zgHW8dsKULS>a(19Shsu&8CK(>kBJ({aOtJfz*`@N2d+04xLpW@nt-tHyZ48o`G);r zh960B9=2uP=fPokN&oCOI|BhsyBG9u=`!Gye)U9i;Vw$g^ zTjsso5sIIEA~ITIgJ8s+=Nd%`p=hK0xbAF45VVwQT2Fizf^T!SjcINSf_n`kj`ZFa zf>k@kGeXY=!QapX zK7g88Qk+o#;k|*L5FYPqDIS_CMSg_mjnp3z5VI^}V}zp=^K7DT?NAUxLbBtBwRAcS zDf`^kaxVhP%+e?ady)N(HurC6VbS3Q^(jbldbl))aY6MtmYR~;#C&ga`9qsjWRXZ2|dsR)5*H)lmpPw&i~=&X-~D zAEu^);X>H=Wyu3^unb@9-uzK@h!E@^Yv3^x)lThu)l z zPedTNJx%4pc@dI#XX2y7BCuVVe<0jmgtzx9|5#Zrf?F2_O$)w~xlQzi*& zCPa_o<~z+HB6xf7hv94o5uRv}9qDZ&f>-KRtIn<#;-%RA8KE4C07x%e*bb!oyC$i5 zeip*zC5s^KKm?wc->j2;LVQ{X|Gwuu%qN`}a_$ih` z_<2rHHc2YPnPCrVCu<2o=gPGCH?4$d)_YiX$esxBtlwt&7=-x2J7Z5sWCTPv+saB8 z2{E#VrAIi zx=sG8HM^6xm5@0d)4Tfy^Y(Cnlh;n0xlsR8=SY35$e&eZ_H)V!5pKGbsbKs7aJtB9xb0U`mrVGy!R3@-oLc4GLQHt#e#jjgZi)XKtW*&1wi~%a7SHj19FsU78eRDywr{j7G* zrHEMVkK`E-B{<3F!k%+uiJ!B(-zbtuakpc%ym+_>)*Nb49pf&;>a=&#`;$a4?wap;!carcJRe?-z;ww;PrPm5N~a-0J&xlOplilQP%SRU$ZGIp5@+9w|FB zM&aiRA}A>7pMNty5@%)=g?L^RLDQ8|x#9ds^eT@V-JSODs1d^+w#G)H<)RRiAOR^O z(Z=NlI+3{BYsu;C6fzn}xp+8AhK9dhAIuvm29?1pj(pu9!-EsotnL*o2A|QDp$o}@ zZ=5F9A7CT~*>N+^OlKJ?4i@X5k%^&3Y+P=?REpQz)R*?i6@xjAH^WSdJq`GG?)DeM zjgUX`tM^$6Q`a&_}wbzV) zq!Wc7s~Rt*&l1C>Y{wC0Ws&&BX4<}MW5iHrCTRy&6yCi2;r$H{F-YvK`V@yo;od23 zl1w`>oV+uzhs&ZU%=5PWzL+frx9#O>gYHCO#**ecpQtmJlZ_u)bdlq#wZ|v-cNfFr zwrOGg*mAs2o&OmthMc95>PILVr6;PmVK5!NW1l_)o$HgxYG}WvI4Lb2l{UwX zeo9t_qhNl;eqKDjKK&>uG)n@ry(R6FG~#iPVV<#piv;o{X@v~|@fh}Pw{nZ21kmZB z?(%8z`0(PP8RzsR&`UdJ=z=ftxNDcg?FJRJPGSJjOp4}F$BbI=C6K@rQmMW{;;?nArRwqDQel&B*Z&AdvPZR!;P=W z2M-TU!h)YU*;>7W;X&HqrISV^Vd2l5mMEf?m2UK`o0){gzj9h0iGrc#@ZmWt7bRiR zgOK10MZv(8zfU|~o`jWW*L2yoFBo!1-j;6LOrdj|4ExrWU|6nnW5vmhNoXOjJ#j)K zgqdCYqpa3-7c*yVothYf)mGbtVOJ$!SL9#&@KH22d{tLpc2)vjwR=)8IYwh*{DWFI zQbV;jpS8K85RD4eId%0*C6JynYisK(InJRe*BCE>it^$UBPG$;{aHG1;8aq0(`RjM zsENjQT7vdv68NY+^rU}C3{L+%Ha@CE0t;$2c8pbt!F6W*J2BfNP|R!3xDykDBVQ^g z)E$;E{nR+hPc~}(WP5MV{S8eC2uXQL0&nt7YbUe0?^u}TdqGT5gSKL(@|a_fW^t|ap?yV@sRyp-pyJ8Ty)u(_4-62s=ixZFKQOR zj~iYZ&)G@H&F_0T=9d66lt*&i__P8;w~guIPrmD^ulnXwINf%^g_4StAh3VYa`yOD zs(;tJ!-oTcz^X0j+4$|$A7q^uIs`$0#`JDt3a1+!u(YTp-*@Tq%P9+E60yp!WzEVj z0yuVm&;2(9?xE zlX%bkj*#Udq|}+N-4d|X#XoSwLNUlSd#zfS8jl?ncQ!wnCkB^#6XOh13a=0P?i4*v z466F%%xQ}I9qmpKLouNpOzq;ajmd!s7ef_cC)6*-V-w@6G8BV5ArU)HPF_*c zE`smB=#xma@$U3}YGSxXQL3#DiC9rIWnG>(^_;Y(xh5Kr{1{`Cd$t zfD((?`nGD#nj{PiZqAyU*A{~z-CqCFB?EKvqQ>PTkHlbCiWOgBNnq>yD`6@5F~30o zKgE!uF+5E_Cq^^Mq*YG#<-2mNkHf|f?uQ1Ey+i|G&OP5R1MeUuo zz%&lWX1qDjT}c8T-chh)Uo7%}8n?iPH(zgYZINYN%5%GdN&eXg#H#qDRw zi{3{97H?FJ*$$4ygYlU=uiBHvNIldwjzu5xt7Y|*09WDC(b+1om`^^!W@_Hxrn8Xk zl#|<^X7v_9Q%01>V52fG`o5nSwohFC;zV=|PGqlX_3bByg63&om)phQp-1FdZ=sWz z3Hj^Apd(Fs(Qy&178Vxx?uo`xOiXp12xc$=CC_MVvKTjd!Bi1&*w3x~qp^gEpbsVv z;$PEl-WiQ8hC27X3EI+la_yLonbElYJ27vd2uhgv#Hwgq`kp4iRs_Wi(5OcYsxppg zH{!Tu+T2Mo=pO^5#kS}w?74X+2Ai0$c3@a3&np(C{O{WJcGQn~wUY-tjzx}t#S^gw zo!9{i`Q4>)Sf5=S6ktRR-!k2La7-Ll^;`FB9AQwh+_-TlXM>-J;0_b4CpCACM^(nN z7A(><(QI45mc^rAr!!mxLPAm4^ovIW%K7q{LE6D*iU!ul;kE0LEdv&cARlfo?KdnA zZ|(Fs8$ob}%=V|rV^LV%^KBH%JOi!9o}d2<&$y&{6_ zYrc+$Z^z@UvHq>c$y%`1XIvm&>Ans$D(dnhWT_V&zybSY$(LycM9$nT2OfraV#$u>#xkVseDOmb&ImH zUK5Y4?|RjiBZk5?mZK8137%7+bnmb~2_!a6%%d2rD7zb?6G38vc-98H$nLlL_8=Di z(oQHc-Dq529d6v8hSM2iNs0a)%ZQ@g*?eu;CCzC3a>pU03+Xej_7E=XtQ>t`P!u^+ z4E`LRT#k8s*FKzF@s{36pE`+3!)W9Tn=VJ|X(S%JRFZf`a;)I1>GF)lu#0WM zwUMK@HP7P}F=mi;=n6S*W1O>>Mesi4N6nUNa;)ROxA40mf|t~V53Fb`%4lr#dMDza z=}r1gS+r7twE?-hnCQUTXq?5@>BDs*vDd`X_`#R$kNDRspz~?=CI28 z>$D!Qkw6BCk6y#&xPU-^Lscd4`Qs)djT||SWbj?@#L%7DY;)yEb4#9bS}%f`o%|@r ztF&ChtHr>efabeIV+8r*x7CWFFZrR}21a8769gysFN5MSDvQQ$i6x;Lm~UeeElg$Lb0unx0JxzZX^mTr_ z3b%e^RXB%9;f?A1%WkCXO4OTsjT1_t;RoM;YJDVbTNQr?I z{JnS|WvJnwX(ENk%<*_ahNe!OR}xLRaqCu6=o#SA!s`+kGOh9(>%0t~?=*cn{iFoO zjo)bP>l2BT?fE5nvE(`Nu|Wp2~l{gf{+hqBv8cU(FR4~5C++EO9G>pXmKKsMPWfHK|yXx;L!N+@eeuj z-!SAi5(s7Du>EO?7e~F&QkH@zu$|&Vy=nUVXUS?7J&VM)dO`z^lftu2+*vgImxVK> zu#4o?)fM6Fg4qm<-osMj1X^MAPBpmn6fRE3I!I+aiTd^pL77sGYM( zs(HOH_nZ{HpYz5qrTgCTN-6d@mBTqG4Z5isAjL=H4F|hRA%db(>#j<0^<1e|Y(4EK zRkHEeYd!&*5jY z2q3UMs%?Q-_q0a)f3zn_1E#b&!unUz^a0+-` zIpjk)hB4x61u+bP@RBry7{+{0kOC^r7iN0t}&ND!ev!-?j0wj2@ieXRrzABmG0*h7RE=T^~i z+ANu4-Wr=PrgMOQX7*hPbi%FN{sjxVCV}qnM)Z96NQ?^@Skie3G|eDf8(GcS%Lvwb zLIV1mn*}zv#ppJk6p(TW%o}LG@H@q(#_JQ(avu4h`E$6`fCRr@*EJ{^CxMx!`aePb~C zWLAg-Hjd?}Mp`y;X#}RtX)k5SUh+X-YqA6g#2hn`d~)w^rp+IpBGFu>$5|%Rl=aP~ z!843s{6mIx6xWmuI z1<+Lrx``9ggT_el-SaK{YP}p=PFIA4KQjgu@=Hk87nx7+WJA$9Z> zZHrpxutrW%WJ8pI#8Nh8E6lqYiHGmZ{V_O|_AiUGOz(Fl!=A&}M_#!gM#3bo7M6&v0U6OFNuSHuA6YOx zLypUH7tgurOZuMmjPmuXNI+os%83#LfqhSN35j1-j!tjy?dxhN#Cjg#j^7F3CPpf0b|;@CN!}S>1@O)zZp6Ku2<*hp-w{Bm0%v5~ zt8h#yzv1|al$5G-5!mo5@>0-V0eI$8 zz|-(Wf$h}Tu^(AN>|<5;Wz~#8VB|LcdIWX?qyh+-P|oA#N1)nNr)H->I6XGy#yob! zZ)7yV7{XOY_bnh8Sv8xWt(-vE(^lU7)$j<^6ce!JTL7$L;B-ntV#@9U`#=aH*r0*7 z5OuP))abhAP8)HjkW%TEYJYePRxPTbYu6w zgAvqJlsT1AbB5&-P~q=DhQELgnhGY*oOuie;e}P*i`;^7|6e1B1WmEp3_a! z{thbn3v{AcYpU7_NwIojn*DdI%70>6_zq_3zp*abss{fC!RVx#mHizd^LGTzzY#Y4 zN!p zWkArS-_Rzr=@y(*yPY>!2>%X+>V!&JJ+aXAR8s#9pyDf74$f8ojjmENw&W7{%HG$K zHHtsRRO?@uEPkq`=X6UR1LLyUVCjE|OV}>{ZWRt;yPjC`zw>Mqthj`Au{XBj5_HVo z*UB^1N^Q0($5muB-O6)=73Y(^bFG!<6Dv=Jeo(@QuA@Sx=GNc_HN@MwurFoiqYGmc?6w%!?iZw zN6<3{!b#99e!pI-=ZVhe59rRb@&`YD@9vy+B2Am_ycd2j!+#N1gq7?}&J#b)Z9LvQ z5jOH;dK9YpgA2K1Hsli~j^u6M&wgMr#8#!2FpXLai1o4`s7+GlRnqP3EN%@^dFr8r zNn)LFH4Z-R2aTkXE4Pc#mOq)#tMr43>_VQJ7?GVMSnmg|7w_csb|FoI3{ub4eqdUj zog>CuXV=HsjP--i(e)Q{Df*DfNcx3-z~lR>@V<+%i7n?*@pk^=9-)Ml%V#iZLOz}*k5d%s^(La~I_wA`gN z&eZWm+}5e{#Ar*1&4xR^P{?Q{)nb%0ki2|f=pcmU@ncjyBfIETFRkl$8C*Co#zH3P zW1TP5Te>YiM_P{y_2S}j-`}}C1l22Gpn&7))h>xQTT4#S=w@6g*G}_=7CJ)9YpMH; z)$8C3CX|CB2Y8IWOd1yKA=A^s7u05}g=$LBmO(!)@dfK)8`w1jqZXV{ZHDc_ z$p=!Y>`P7sfk7M{y-nJ8d583UA-ApLXjm@^Ci$Lrnexd8s^-#Mkjk5xbNc6=2|i#= zC}g{zVzgin8FG7s4^+IUXrE6|*5!PW!aFO2>SkYtpdKb~SjE`)f5w*aAIvN?}y3Bve> z(t3^ql;~1+$^{9r5S=Yc0iNoSI@wE#9kX)g*?R+Ix2Ro@9Vn$pKAAEc0LPz`>SoHM zzr7;4UXV{JOll%Q{1}Y$VNb|ptV*dAs~B*)k|*dgR;{NL8yQ>L#sijKB>a|<6q`m9 zmO9@Z1Pt`eP>KeOwW;h5l#LL%RzZraPHdzbY~Qd|X}u!VPl(waj&4xMSgTyB&vvDi z)@4^{vfq3zMT^Rh-F{x@=nBn@6R{uFx6}K!g)8_Gu$B`-#q2{o2UjReuHsVXXKL{_ zxPs$UEp86!sljTh4_u+>q~SQKw=6l6bJ$JOySw!cddj&ArMeA*7Vl>30|YH&#`t&; zlsxL-ZINPYGIERDL9u{$gvQI(3f1<2w5QfBS#PL}F_zyE&39Jgb(PV*1&zHup@K3c zK9hQh(PRX#_kdI6_ZljeVWVN*!i~E;Ad^|GJOU&#O24@$EM3Wt=E|^uEwE1ag!A-P zoA;0CZvGMbR^I?mkdktHlmbkwLF7_N@q|!v>CO+Kj0al+VhcUNVCzIFn@-7M>DTyA z7M_sLB>2V9{aJPEUq1C9SU_W|F9ob}XI!1udlJ1*=NhH%ghiq%nLaauJ)q{&piP6OAN~k+UO3sWQ;P-l9+1tz#>bHPo40vX)d3GE z@5Z&*9%Wg^8}9**e7>qRfzjZW%3aN)?!e+tSGP7JmC`z?%fWDWFksv91adT(+QGRt zXbW>!ykU?-5hU3sv6SL2%8R+uSW0 z3m7N1t~07Ha?UBxh@*cc$NB4>&~bm45wW z(Dr@79?QO-n!9?N_{1QPcz96Z8B}bY8H2^N$46^;(ZI&^w5E(SRi(LoQ`5aj#_#N5 zQAXwcDK+zi7ig~2%8bRPhNj-0L~$t1Z93swgkuu{(G^RLt+=RY+vp7~+x> zv1lMV>Qp83g8B*9&(4s9-AYS^ls4Gt6aqI+WgDo5f;|U7bti`C(8A7Ji5%we({aoHZ~)p zN4v&7s;}pW=cU%(aIc%e{yDu9uz=Fq++4lEWQuL;J@W)C{4u!wX_z-Ke#H#~h-MNR zmwH2+^`+@YawsChh#qx%TUW~J<4PM6uz4+Iiq-i*W;kU!l_g+3BRZe@{6@bYr3e*+ z)a3g@Rn~*Nxo;Ej(5R}Pm&W-*&4VBB6WS?mVne6VV_$H5wBqYHZHh=S=v#Vw7`dkM zMs!ca+^RkY`Y)#AyrI@-hYCfgC{3>ImmhE$2dxoBl$gA#6o25{Pq?C`l!z=QA@7bq zXvK0z2-%5Pm{Ps*dUya>+}_=@@Jj+(I1mG!3;@Pqp{1CJmCW18h6Tb5Is~E!@$B72`zkVaQKRiG(S${ z@+%62&V1%&38*$(hqE(KvsIZ(lcR=QPH%zc6dkL*-bQQq{78M>LG$-XA7rPZnjBiU zsyo6Q%SCk>`J}1Gtdpa-tv=`E@;7eRCTfU;^`6 zw+X4}v_E^tfUgcPo$Ak`3N{$i>jbPFVLNkxQ+YRft6YYkBiNJa-FI9nK4lUtyd0T; zmXjVoO;xMY*qoB8DOV4zbp*SKexY-y-B;xEx?AB0`Fy9|ocXDm-?dxCiyfiNo{rwd zskoX}Z$!N#?7dQylJ_NhwPH{hff%KhTT2l2&K5ePK^0C=b4%pSd*^ zx6aNS)lJ(8a+zGO%2XsNJlNUAiCl^-jd=FgL!CKWn2LhA^;L5ZI>G4?A=?a(rQ*lD zoJy_DPGC5J-tD(96<1Pj;yAIBW|!Wf2U4++D&6ht1YsIQ!LJUbVi*4GL4E;5XM65c zIhBgC&+$FKM>tu|&a6+xdE_nR&UN~o2iri?J(^O6mr@rx2cl~;u|MtCy{}Wzx|EcR zfzIG#rtR9DXr=ACF=^juXan?H-EX`lPTEZt^kXM@NNF@{zR;A8m^rpu)fonzDQNlB zo{AeO(J$H98M@Rcs;PfWMWc>~!DHN=p}*U|(3!luHs&1{(=b(}G2YrP92C5ly;-_H z4GnJmR;T>$G1@?-I6ftesyWM5ydkWHHCr3-cL* zM;N&7aw=c+HVwaPR^ImS6Arw)o#;%%6|GaF9AZ&qo{FFAEG@cy4+UF=)3s`9sdzOt zsU!JgC|F;$37o-6#ieCKE^R(XIc1AyS34M_qK9GitE&q_;k~utUQ0;DuTuzQln@FX zGkYb@5~kv&%+kSXQ$k^a*^ATpvQ)(J+s1H@gu~Izq0H#OVoIjWv}Me z$Dcn%naAYit9{c^rL_E#7tXQ%x_^@S^(Xj0lUuFw&-{D#|9`w4K<%!-#ecj!K<%49 z*?jbF0joX%o`VAT>$owUy)f#n&g|1;J)|6pypcU&EzY|nEUOx z$tC4*nCN0}zUyZ)0@2TYg~8rfspVRa6fC^&R=w9P9Qvm&_*j{mg8AN@Ecd){N|Ugk z(SLRd9y%$WVGyI#4J#blnzr`t7JYwK#g->B%Z9Zdl@IiY1 z)|xJoJ7KWU>(b#dS5we7$!wnCt}vK+X;D?n=M=Q4D|-KaNf_9kZyQ(ejoMf1z9IHu z7_jdv4%K}__ZLmmc@amPTg5ZgIhTTs6&FfEXM}@(m(#{gRG(mCo>C?y&?-829N7Au z`ZxC474`UVxahiU$5dixmvy!&PrHS~uI79EHN>us%RjeW-yR0tA1H=6SQ2|?mk!=V z?}X4V|F$GIFcs_1)+N-ag~7LHUL7~GQt?(pU4q5^P*5FS-*kTtJ)e@4uXi96TF)FV zI=3hlOWKZ`{R^qs)OA=kP!h1I*mra|ZTY4} zOZv^J4*_Xk)f9iFG-Rn%-cGtgNx|AT=nb7|Smjc_Zr9EbK(*CpRt2VEvsv*1t?41) z8m83jl0uST&;GC=c?g^}R828nN%tQqdVjy~*Xj_>ES6xf(EUgGhAV3uBOajOwPZR>E&?GZJG$auMNqlV4wu`TJ ztHoW5uhzA!uj0EU+g6W4vl4d!c-#2|NQf9vlJR)`IBu|zd2!rZ6Tqy=vf*9eu14aF=Q971jn1We+`eHR|-2Rwic8rR_OM zT4Lt0t2g#tw=#x*363Otw8TqI$D2#t+qO0pW?Vr>gdn6w_}?^3vyk1Ed7}e`6%Y#9aQi zcEHD4qWZ*w%WoJcMxynqH{8(@tr3=!@l(NHV>fTazk%?g_A}B&6jSh1^u^R@9kFf4 znkA9zCWo zXyS`+_SO^QZW+$omVv$J-Hl?So~UnI&`+=s^e^67w{xMM_~!6-&F{QWf5rN9*it=l zZ%uw4(27=KsNA_2V8 zIiK`B1L4iirnQ49#$J4BoC@}M;wpJ)y_8~BDc(pw5M?0py`tC7Li*`S>2$B$Kul!L zuAUo9F{#%sB=3R^+OOBQTVB*r@D)(<{n4il1XfqmI=X;jmPRDMw=U8^+%PxyojwKX zBkXqOL5N@dq;l|XXrB{$H|%a0Xdtc}9CUrjQOIw%IJAzehC|-#RJ>w7qn5#&^`@!8(%RHt9E4^m;M>*6T;_K8HmS~<6rpF z-5Gvwah)>N2lmW*?8nP58;GQLejd9~K{1&>=w>?V3`F{=!yl~EQ_LyY6a2&m199AR z`a&-c#kk6!6;&ND5G7lkO^05kn8db&k(K9w#9+)nN6c7H0ZByWft?~CS$IRMsrd}r zPtK~gA2DE637L0&KHQ0?U+Q1~Qi_pSBY!I4Amo2Ry7Q^Uu(d>2SCdK8fDd~}8IzIt z_{S~s0~0Bx|7hv-_8KG13z|1Xj-nXN(9AJ;i-7c@kLBd~L4e=<_a7RU8j0dji@qbE zKlIy?wzeN^jvBi)V&5k)-t<3`H?noBk+{3Oc+g*97B_!w!kb^imVh0%=jIQySQ&f# z?}t?%G!jJ(drCucU}m&p!nFnKVVB9#tDhf9h5pz3g@xBzjKuov#^RPRD|2ekAOAXd zn~|7z5j*uaG4#Ke-L<8SMk2LlMQGEf7AEnv9O{)NMq-69j1!BcPG&me9PH?z`wHEE~REg+!OBHcX&Q9;_-*39PcI z*eoHwHWa@Vf^>DheW7~|p@D)0)mf15njXRLgo5|@FI&LCOukgmJqYK$j&{I8?25q- z#o)91(RB}Te9x;+dR}$%>%HpaKDV~O#3P2C)v&Xfpxb9l zJU%-fZl8Rbcy#;ZOCJ)$<_N<0Ns6~h0fSE!k1+Vshr&SfuON&TIX+ts7<{^Tgu$0S zB*wkvNWfbY0E15%k1+Vs`(SWWjiAXQTabl9tU-wHQsA={XtKdE$ahl7^JgyDF8K8a z7h?@c@c6=grm%^fN~EV7*7pZ%>YQkbu|^@bUWji^#lIMdbahg^cdw%RnI`ML zb|HRd6h3zpxQ|cXj;fz8{e|ieo@t7)jh~3H#vLNO<%$Tu`aBrhvl{Z7rzyX8HKh9{ z*X`iUwz$he{HL2Dyy2$E-(|b3a-cQ`9l4A2K5ecm3OlTEDBs=^;d5^x6#kk9Lg7mv z3MDW=vnW@D_{lpW{N`$4w;QzggQ247A2!$`b4@&ogv>5Xb z()F`K@GoYv-(I=Zi_T>Xenn#|zZPL)7<{p|BclEyfRu#5dV{1{I4!z{uB@^-9nmAA zWpT1NX>@F~tI}3suJn~yo%ZSq6u~V`K%8Gz6+L2FrxdPfht)~h9A%K$l%s-lT$U2I z&qYJ>@zDmS-B$)UvC-OUyW7SEC&osj#STa@T$Un@lg7d+PM(siRL1(lDG71%gamn_ zG;qi!&X_SnAc-JzbM>YX{?!@D`rh&ANGMAqNzw@a=*S_-L{e2oW{_kVsV-w>D%Cyl z9wbcfh}W};APfhqg^kZ5Ne>D6;Edwa*mww&q{a##s^lz3k=^Rhxh(QTo8Fe?)l5pS z)~d6rvK%U(*(58sI_#x|a%B}|l2vC-a@vaI`85_(zSQcNY}2T-Ct2iPnZ=}-nl;Il zoK->DEpnNi%FI^y!}-;ace2@}aG<>TQnxzG<_XeaFQqc=o}vP+O>eKV<&9BR(I%?K zVgvliGMOba*VQS@rU1`YSfNY=-4@Nb^hB*X(W6mIy?S+`PoHL0=*N1z`ZN;k_ath4 zi9YRAtxuOmdbHeszD!wKAVn@A$uVH72l)kDflm4|aEm$x`ED|DYX))~I1R!&l*sSY z^!S?icnGLexvUU5kxj^k*m@&L^O$_Mxj^TEv#$u{Hzdc$PfZ&ZAD;rr(seATzfwEv zBL`KQ3XCI3R<#O+ks%~mSdm>rqYF}1h49TtA1Z@U%O)Db#<7>|6-u-xLbs)W()OOe(XhcEnjm^v2pJ*}JtJz>qa-h6f zWb#AVTkb9_V0Du64tvQ$)HRTasPV(m)!(cugW4pUXsiJ;Nyotl>u)vdM0KY>*+irh zPAW%HrmECp$}KG_&>`Q0J`brT$xMHLVN?2RA}fv4m%-|T&It#5$)+NaD^)J5Tv=K; zP75tzDx@Fggpc6%(yhNO6&Y^SPN0A0q4ZK@KU=?ftduRJzx}aLY*EJOYAlXim!%@V znqs9!RFAAZ@R5h?2NR(j>^Q(ivdPGFfBiur;Rv!HxmM+|B&axFp!@(`h|d0_j2tD9 w3~`w&r&QDK;Z~PxxVf^@=@}9jhlj|Nand+h)J|AVe(;}QC2%Mh;0?R~1H# Date: Fri, 21 Mar 2025 14:48:15 +0100 Subject: [PATCH 07/11] use goroutines to process row groups, fix tests and logic for row group intersection --- internal/geo/geo.go | 19 +-- internal/geo/geo_test.go | 30 +++- internal/geoparquet/filter.go | 31 +++- internal/geoparquet/filter_test.go | 77 +++++++++- internal/geoparquet/geoparquet_test.go | 141 ------------------ .../cases/example-v1.1.0-partitioned.parquet | Bin 30566 -> 31781 bytes 6 files changed, 132 insertions(+), 166 deletions(-) diff --git a/internal/geo/geo.go b/internal/geo/geo.go index 9794fa7..74b367a 100644 --- a/internal/geo/geo.go +++ b/internal/geo/geo.go @@ -352,18 +352,15 @@ func (box1 *Bbox) Intersects(box2 *Bbox) bool { return false } - // shift all negative x coordinates to accomodate antimeridian crossings - if box1.Xmin < 0 { - box1.Xmin += 360 + // if box1 crosses the antimeridian and uses the coordinate range -180/180, + // represent e.g. xmin 170 as -190 + if box1.Xmin > 0 && box1.Xmax < 0 { + box1.Xmin = -180 - (180 - box1.Xmin) } - if box1.Xmax < 0 { - box1.Xmax += 360 - } - if box2.Xmin < 0 { - box2.Xmin += 360 - } - if box2.Xmax < 0 { - box2.Xmax += 360 + + // see above + if box2.Xmin > 0 && box2.Xmax < 0 { + box2.Xmin = -180 - (180 - box2.Xmin) } // check longitude overlap diff --git a/internal/geo/geo_test.go b/internal/geo/geo_test.go index f912b5a..eb341f1 100644 --- a/internal/geo/geo_test.go +++ b/internal/geo/geo_test.go @@ -22,7 +22,7 @@ func TestBboxIntersectsTrue(t *testing.T) { Ymax: 55, } - require.Equal(t, box1.Intersects(box2), true) + require.Equal(t, true, box1.Intersects(box2)) } func TestBboxIntersectsFalse(t *testing.T) { @@ -40,7 +40,7 @@ func TestBboxIntersectsFalse(t *testing.T) { Ymax: 70, } - require.Equal(t, box1.Intersects(box2), false) + require.Equal(t, false, box1.Intersects(box2)) } func TestBboxIntersectsTouches(t *testing.T) { @@ -58,7 +58,25 @@ func TestBboxIntersectsTouches(t *testing.T) { Ymax: 40, } - require.Equal(t, box1.Intersects(box2), true) + require.Equal(t, true, box1.Intersects(box2)) +} + +func TestBboxIntersectsWholeGlobe(t *testing.T) { + box1 := &Bbox{ + Xmin: -180, + Ymin: -90, + Xmax: 180, + Ymax: 90, + } + + box2 := &Bbox{ + Xmin: 10, + Ymin: 10, + Xmax: 30, + Ymax: 30, + } + + require.Equal(t, true, box1.Intersects(box2)) } func TestBboxIntersectsContains(t *testing.T) { @@ -76,7 +94,7 @@ func TestBboxIntersectsContains(t *testing.T) { Ymax: 40, } - require.Equal(t, box1.Intersects(box2), true) + require.Equal(t, true, box1.Intersects(box2)) } func TestBboxIntersectsTrueAntimeridian(t *testing.T) { @@ -94,7 +112,7 @@ func TestBboxIntersectsTrueAntimeridian(t *testing.T) { Ymax: 15, } - require.Equal(t, box1.Intersects(box2), true) + require.Equal(t, true, box1.Intersects(box2)) } func TestBboxIntersectsFalseAntimeridian(t *testing.T) { @@ -112,7 +130,7 @@ func TestBboxIntersectsFalseAntimeridian(t *testing.T) { Ymax: 15, } - require.Equal(t, box1.Intersects(box2), false) + require.Equal(t, false, box1.Intersects(box2)) } func TestNewBboxFromString(t *testing.T) { diff --git a/internal/geoparquet/filter.go b/internal/geoparquet/filter.go index 621c82f..9065c0e 100644 --- a/internal/geoparquet/filter.go +++ b/internal/geoparquet/filter.go @@ -104,20 +104,39 @@ func GetColumnIndicesByDifference(excludeColumns []string, arrowSchema *arrow.Sc // PREDICATE PUSHDOWN - ROW FILTERING UTILS +type rowGroupIntersectionResult struct { + Index int + Intersects bool + Error error +} + // Get row group indices that intersect with the input bbox. Uses the bbox column row group // stats to calculate intersection. func GetRowGroupsByBbox(fileReader *file.Reader, bboxCol *BboxColumn, inputBbox *geo.Bbox) ([]int, error) { numRowGroups := fileReader.NumRowGroups() intersectingRowGroups := make([]int, 0, numRowGroups) + + // process row groups concurrently + queue := make(chan *rowGroupIntersectionResult) for i := 0; i < numRowGroups; i += 1 { - intersects, err := RowGroupIntersects(fileReader.MetaData(), bboxCol, i, inputBbox) - if err != nil { - return nil, err + go func(i int) { + result := &rowGroupIntersectionResult{Index: i} + result.Intersects, result.Error = RowGroupIntersects(fileReader.MetaData(), bboxCol, i, inputBbox) + queue <- result + }(i) + } + + // read goroutine results + for i := 0; i < numRowGroups; i += 1 { + res := <-queue + if res.Error != nil { + return intersectingRowGroups, res.Error } - if intersects { - intersectingRowGroups = append(intersectingRowGroups, i) + if res.Intersects { + intersectingRowGroups = append(intersectingRowGroups, res.Index) } } + slices.Sort(intersectingRowGroups) return intersectingRowGroups, nil } @@ -162,7 +181,7 @@ func GetColumnMinMax(fileMetadata *metadata.FileMetaData, rowGroup int, columnPa // Check whether the bbox features in a row group intersect with the input bbox, based on the row group min/max stats. func RowGroupIntersects(fileMetadata *metadata.FileMetaData, bboxCol *BboxColumn, rowGroup int, inputBbox *geo.Bbox) (bool, error) { if bboxCol.Name == "" { - return false, errors.New("bboxCol.Name is empty") + return false, errors.New("name field of bbox column struct is empty") } xminPath := fmt.Sprintf("%v.%v", bboxCol.Name, bboxCol.Xmin) xmin, _, err := GetColumnMinMax(fileMetadata, rowGroup, xminPath) diff --git a/internal/geoparquet/filter_test.go b/internal/geoparquet/filter_test.go index 8dcd76c..c070154 100644 --- a/internal/geoparquet/filter_test.go +++ b/internal/geoparquet/filter_test.go @@ -40,6 +40,79 @@ func TestRowGroupIntersects(t *testing.T) { assert.Equal(t, intersectsWesternHemisphere, false) } +func TestGetRowGroupsByBbox(t *testing.T) { + fixturePath := "../testdata/cases/example-v1.1.0-partitioned.parquet" + input, openErr := os.Open(fixturePath) + require.NoError(t, openErr) + + fileReader, err := file.NewParquetReader(input) + require.NoError(t, err) + + require.Equal(t, 2, fileReader.NumRowGroups()) + + bbox := &geo.Bbox{Xmin: 34.0, Ymin: -7.0, Xmax: 36.0, Ymax: -6.0} // somewhere in tanzania + geoMetadata, err := geoparquet.GetMetadataFromFileReader(fileReader) + require.NoError(t, err) + + bboxCol := geoparquet.GetBboxColumn(fileReader.MetaData().Schema, geoMetadata) + + // the file has two row groups - the first one contains all data for the eastern hemisphere, + // the second for the western hemisphere + rowGroups, err := geoparquet.GetRowGroupsByBbox(fileReader, bboxCol, bbox) + require.NoError(t, err) + + // only the eastern hemisphere row group matches + require.Len(t, rowGroups, 1) + assert.Equal(t, 0, rowGroups[0]) +} + +func TestGetRowGroupsByBbox2(t *testing.T) { + fixturePath := "../testdata/cases/example-v1.1.0-partitioned.parquet" + input, openErr := os.Open(fixturePath) + require.NoError(t, openErr) + + fileReader, err := file.NewParquetReader(input) + require.NoError(t, err) + + require.Equal(t, 2, fileReader.NumRowGroups()) + + bbox := &geo.Bbox{Xmin: -92.0, Ymin: 32.0, Xmax: -88.0, Ymax: 35.0} // somewhere in louisiana + geoMetadata, err := geoparquet.GetMetadataFromFileReader(fileReader) + require.NoError(t, err) + + bboxCol := geoparquet.GetBboxColumn(fileReader.MetaData().Schema, geoMetadata) + + // the file has two row groups - the first one contains all data for the eastern hemisphere, + // the second for the western hemisphere + rowGroups, err := geoparquet.GetRowGroupsByBbox(fileReader, bboxCol, bbox) + require.NoError(t, err) + + // only the western hemisphere row group matches + require.Len(t, rowGroups, 1) + assert.Equal(t, 1, rowGroups[0]) +} + +func TestGetRowGroupsByBboxErrorNoBboxCol(t *testing.T) { + fixturePath := "../testdata/cases/example-v1.1.0-partitioned.parquet" + input, openErr := os.Open(fixturePath) + require.NoError(t, openErr) + + fileReader, err := file.NewParquetReader(input) + require.NoError(t, err) + + require.Equal(t, 2, fileReader.NumRowGroups()) + + bbox := &geo.Bbox{Xmin: -92.0, Ymin: 32.0, Xmax: -88.0, Ymax: 35.0} // somewhere in louisiana + + bboxCol := &geoparquet.BboxColumn{} // empty bbox col, will raise error + + // the file has two row groups - the first one contains all data for the eastern hemisphere, + // the second for the western hemisphere + rowGroups, err := geoparquet.GetRowGroupsByBbox(fileReader, bboxCol, bbox) + require.ErrorContains(t, err, "bbox column") + assert.Empty(t, rowGroups) +} + func TestGetColumnMinMax(t *testing.T) { fixturePath := "../testdata/cases/example-v1.1.0-partitioned.parquet" input, openErr := os.Open(fixturePath) @@ -62,13 +135,13 @@ func TestGetColumnMinMax(t *testing.T) { xminMin, xminMax, err = geoparquet.GetColumnMinMax(fileReader.MetaData(), 1, "bbox.xmin") assert.NoError(t, err) - assert.Equal(t, -180.0, xminMin) + assert.Equal(t, -171.79111060289122, xminMin) assert.Equal(t, -17.06342322434257, xminMax) xmaxMin, xmaxMax, err = geoparquet.GetColumnMinMax(fileReader.MetaData(), 1, "bbox.xmax") assert.NoError(t, err) assert.Equal(t, -66.96465999999998, xmaxMin) - assert.Equal(t, 180.0, xmaxMax) + assert.Equal(t, -8.665124477564191, xmaxMax) } func TestGetColumnIndices(t *testing.T) { diff --git a/internal/geoparquet/geoparquet_test.go b/internal/geoparquet/geoparquet_test.go index 15e3e00..5f3726e 100644 --- a/internal/geoparquet/geoparquet_test.go +++ b/internal/geoparquet/geoparquet_test.go @@ -636,144 +636,3 @@ func TestFilterRecordBatchByBboxV110NonStandardBboxCol(t *testing.T) { country := (*filteredRecord).Column(recordReader.Schema().ColumnIndexByName("name")).ValueStr(0) assert.Equal(t, "Tanzania", country) } - -// func TestFilterByBboxV100(t *testing.T) { -// fileReader, fileErr := os.Open("../testdata/cases/example-v1.0.0.parquet") -// require.NoError(t, fileErr) -// defer fileReader.Close() - -// recordReader, err := geoparquet.NewRecordReaderFromConfig(&geoparquet.ReaderConfig{Reader: fileReader}) -// require.NoError(t, err) -// defer recordReader.Close() - -// record, readErr := recordReader.Read() -// require.NoError(t, readErr) -// assert.Equal(t, int64(6), record.NumCols()) -// assert.Equal(t, int64(5), record.NumRows()) - -// inputBbox := &geo.Bbox{Xmin: 34.0, Ymin: -7.0, Xmax: 36.0, Ymax: -6.0} - -// output := &bytes.Buffer{} -// recordWriter, rwErr := geoparquet.NewRecordWriter(&geoparquet.WriterConfig{ -// Writer: output, -// Metadata: recordReader.Metadata(), -// ArrowSchema: recordReader.ArrowSchema(), -// }) -// require.NoError(t, rwErr) -// defer recordWriter.Close() - -// filterErr := geoparquet.FilterByBbox(context.Background(), recordReader, recordWriter, inputBbox) -// require.NoError(t, filterErr) - -// NewRecordReaderFromConfig, err := geoparquet.NewRecordReaderFromConfig(&geoparquet.ReaderConfig{ -// Reader: bytes.NewReader(output.Bytes()), -// }) -// require.NoError(t, err) -// defer NewRecordReaderFromConfig.Close() - -// assert.Equal(t, int64(1), NewRecordReaderFromConfig.NumRows()) - -// filteredRecord, readErr := NewRecordReaderFromConfig.Read() -// require.NoError(t, readErr) - -// // we expect only one row, namely Tanzania -// assert.Equal(t, int64(6), filteredRecord.NumCols()) -// assert.Equal(t, int64(1), filteredRecord.NumRows()) - -// country := filteredRecord.Column(recordReader.Schema().ColumnIndexByName("name")).ValueStr(0) -// assert.Equal(t, "Tanzania", country) -// } - -// func TestFilterByBboxV110(t *testing.T) { -// fileReader, fileErr := os.Open("../testdata/cases/example-v1.1.0.parquet") -// require.NoError(t, fileErr) -// defer fileReader.Close() - -// recordReader, err := geoparquet.NewRecordReaderFromConfig(&geoparquet.ReaderConfig{Reader: fileReader}) -// require.NoError(t, err) -// defer recordReader.Close() - -// record, readErr := recordReader.Read() -// require.NoError(t, readErr) -// assert.Equal(t, int64(7), record.NumCols()) -// assert.Equal(t, int64(5), record.NumRows()) - -// inputBbox := &geo.Bbox{Xmin: 34.0, Ymin: -7.0, Xmax: 36.0, Ymax: -6.0} - -// output := &bytes.Buffer{} -// recordWriter, rwErr := geoparquet.NewRecordWriter(&geoparquet.WriterConfig{ -// Writer: output, -// Metadata: recordReader.Metadata(), -// ArrowSchema: recordReader.ArrowSchema(), -// }) -// require.NoError(t, rwErr) -// defer recordWriter.Close() - -// filterErr := geoparquet.FilterByBbox(context.Background(), recordReader, recordWriter, inputBbox) -// require.NoError(t, filterErr) - -// NewRecordReaderFromConfig, err := geoparquet.NewRecordReaderFromConfig(&geoparquet.ReaderConfig{ -// Reader: bytes.NewReader(output.Bytes()), -// }) -// require.NoError(t, err) -// defer NewRecordReaderFromConfig.Close() - -// assert.Equal(t, int64(1), NewRecordReaderFromConfig.NumRows()) - -// filteredRecord, readErr := NewRecordReaderFromConfig.Read() -// require.NoError(t, readErr) - -// // we expect only one row, namely Tanzania -// assert.Equal(t, int64(7), filteredRecord.NumCols()) -// assert.Equal(t, int64(1), filteredRecord.NumRows()) - -// country := filteredRecord.Column(recordReader.Schema().ColumnIndexByName("name")).ValueStr(0) -// assert.Equal(t, "Tanzania", country) -// } - -// func TestFilterByBboxV110NonStandardBboxCol(t *testing.T) { -// fileReader, fileErr := os.Open("../testdata/cases/example-v1.1.0-covering.parquet") -// require.NoError(t, fileErr) -// defer fileReader.Close() - -// recordReader, err := geoparquet.NewRecordReaderFromConfig(&geoparquet.ReaderConfig{Reader: fileReader}) -// require.NoError(t, err) -// defer recordReader.Close() - -// record, readErr := recordReader.Read() -// require.NoError(t, readErr) -// assert.Equal(t, int64(7), record.NumCols()) -// assert.Equal(t, int64(5), record.NumRows()) - -// inputBbox := &geo.Bbox{Xmin: 34.0, Ymin: -7.0, Xmax: 36.0, Ymax: -6.0} - -// output := &bytes.Buffer{} -// recordWriter, rwErr := geoparquet.NewRecordWriter(&geoparquet.WriterConfig{ -// Writer: output, -// Metadata: recordReader.Metadata(), -// ArrowSchema: recordReader.ArrowSchema(), -// }) -// require.NoError(t, rwErr) -// defer recordWriter.Close() - -// filterErr := geoparquet.FilterByBbox(context.Background(), recordReader, recordWriter, inputBbox) -// require.NoError(t, filterErr) - -// NewRecordReaderFromConfig, err := geoparquet.NewRecordReaderFromConfig(&geoparquet.ReaderConfig{ -// Reader: bytes.NewReader(output.Bytes()), -// }) -// require.NoError(t, err) -// defer NewRecordReaderFromConfig.Close() - -// assert.Equal(t, int64(1), NewRecordReaderFromConfig.NumRows()) - -// filteredRecord, readErr := NewRecordReaderFromConfig.Read() -// require.NoError(t, readErr) - -// // we expect only one row, namely Tanzania -// assert.Equal(t, int64(7), filteredRecord.NumCols()) -// assert.Equal(t, int64(1), filteredRecord.NumRows()) - -// country := filteredRecord.Column(recordReader.Schema().ColumnIndexByName("name")).ValueStr(0) -// assert.Equal(t, "Tanzania", country) -// } diff --git a/internal/testdata/cases/example-v1.1.0-partitioned.parquet b/internal/testdata/cases/example-v1.1.0-partitioned.parquet index 93a5444d2f286caa9f2831b4447dd3ede9e267cb..42084f95603c6b893129d4356c0b2468f4af7379 100644 GIT binary patch delta 6836 zcmaht30RXy(+OY%1Vl?>5YQ+ZFQ5q!?qh?2AP~s$2Ew5RP((xps-k!V&my9Nulga? zilWw5YoVbEO6z#kqbl)VBEh@4n$uzvutBrW4IhHt3QjyPhsKlFmtFBe{zc27L#w)%2K10%i=n4`EFbeeabe#z+s!2 z4`XpHxIWx~82ll;PMI7`EX~N9nx1Nlf5pzso0Tb$PM=IBFnfP453_Ua@gK+8hc(jn zJpw+*#EBWL1ZMa@hs`$UV9^=etWNE`PFZ0@f`@=!Yvek2{PxH+&38wf!o`@^zl*hMRWzOow4tlU?Af{un ziu=gv?7p^a4qebUU}%?RR-7>7w*G9^$+fhK{c=D19Gk^5)}$EOBmFBws@cOn^4rQr3BhYV4 zzdZNNVkN{Jd)iNITsb((BFrSdFcQ^k0(9@B8;Xj)DbE#>7|taoxB+LDt!ga;>nBJUSTH_YjqrS7 zIgf$S)*J?g!OJv+9@S4(qu0bnTn${$w4bytg2A&A2iP+Ng?GF=7$_6N5g!H?^xMS1 z0w+IaK^Sz?gfIM*!N0Fy=gutglJwJIc}`1aeCj}fHX-E93+o=cWAItS^PE;s@|}_Z z+8?Gz)cu6zck=fp{J#xPbF$ccban#V`F7)Wd8(!xPePhj370sX>=!B0LG+4%Pt0q2 z;GHqv*djc@=7DeU>orcBK9s#>WSZ0WRc%el@ThXA0laY{{PJ#P@#9p8!ao zYZWQ&5v2Vv&XRVNEb>4>*3DHpE)$^jUD4x`%PKVT$nBhaKTiObIBafhy$VI%Dsb5M z`vhnwaYi4OXdEr0gDSJrCV<}NH`$@hJ@@G36Uc!&&i<%sI@u?2a57BEz~dJ+YH!&U z`p+lhVSU!tH7Ad%QLmrZiu397KCZ(e!jc&G;l8J+_9Qy%T_W2HgQpLlsV z*Cqq4ZhAy>ed7nLD9(ME1kt_!c<21O=bE{`E*6A`Ugs-wAl%`JP=52;%PEubUaXXw zGxP)BKzBrTxiAS{s_F(Uk5-{JXY=oyPbNXDv@K?vT!q@-;Srh)6h{9+NBMaTua@JD zH>QA&%&1hTXujLR`<(G$@7wy~Sd_|?0E}Pytz~obwk1K6wN2s;Sx?2h17uo;SEqyA zV6wj2WK6jZ_7UL|)M!W|SjrO*lh-oTh|yY^1f#I$Uu80xfyELe!F-2( zaq?S9YE(~5qf0WaCQ>6pH?koKLWsWO)#yWkUGYt-A~`p#$_`u58G#45U64nQFs5CD zyg2Bnx6R;wbW+e5N#j{OQsQC8zT89Q0tzvvdZ7g_V^a#<#^EYIqJ=(;$u#K7#5r+; z=$fEC0i!l1C)u)fu;Ge;E)i1QXGw|`316XYDEloQUY;6k6Eu%POysVJ2PSZ6L$E8Z zta#|P(7eg*%1>WY=htAc{d*mc$iD3dWF zC?MxaeuP4C5A<)hUeLhiVv3)9ib9NhoVU%%I71;H>BWVt$7(R*qUCs;qHRKkSP+}h z;t*#p*_?HA98_$=qn<+Nh*jwEAt@HjG5s;baYUy^D)1uX1-5C6nN@L|8j7oLJ=kzx zV{#w;8Z(jLr4vF^f=Ix+QqWR8U6s*CgZ}((ulFdtr_+g-FW9+g(3j<}(nfx+1?tSD zzlVOJF!d)>LcIng>?VK{_oI=ypra(eqK1!DML;g`6kO&lop{f+zRB zwySsPuq;OlB-RHF$M^-Qo?5KcKl3zY)+#+ja}wg=AvX9ipAKV{#GAraNug|f zs8%({!@;Y?Zzrcy=)+hrg)J##w5#;^v6kIo^mv9dChWxqY?&A{kp+ryc~%ZC+ELl zDZy#D z#jrOCw57@K!^e`kV8L|f8lHI)@T1D7{$q$ln@RMJN;LI%n1A(KNn%y34(eBPp7Jn! z;g43a$1e$8{C8vVnK_*RbAQ@8(C}$ZB#O+p57g8qfI#{z{Vax=Otm|~6iw5xz!$#T z>^CGpaNoy`SGAGo_~KGGeP#k=rp5)fDwMWTjaos%gI1%ZrcGM)Q5v31K7>uXwd~ z_;oEXh2V`G>CU{BmTl1jXHLV(3n%0#*ZrglrfT7?UHlNyW;rT-o>)+^aw`--)1ct)S!Ww%)$RIo)KPvHcef)Yon?t2q>oj@Pu=JMPm!(}kgr&6>i|seiJT_|McBK+=5p z?lL^FYqlY0zPAS6oW0LlfMKRi2{dqYarX5Tvk1f@gRg-#u9t3WY$DKCP4iZdzo^Cy zIBVwO%m_3qv+U7=18S%f79H$c7=g4qj$RInSHmLpqYKY4?7GeUcXg`@Za>)Yz0KkX z#KhVag9=DSud2igEWJ@Z{bv=R1GNMDe-*)0px|#*rX+hX!c=Hzl_}l5ia=+68nfgb z=1=}()w2CMIoh_|qQoOk1sl@jca-&V#Pl@&Do~Qj^DjAiiO&SPuL>fmnYP*A%F!#& z;@>vvo-5&tTbu=pisYyt`*Y<^3~$@D@!MBdB9J?2qfRNo!Z~b1MH!Bfr_pa4epG_! z^lza%rbM77m*b5NW-8EA>JgQrB2XpS$ymGqX#?n45v~eSHb&IqEo~_j9>q>0)#@yU z=aRZHIvmM}YFkyNTKT&SEn~c?g55;Y9hs@fuTnu1>5u3e5w2QN=3ec_GelAYjx*A( z;7<2o@ZJ}Y0sGi4$6SoG5Uvx9E!YVgYv z*&d_ykzOuLp5bXgL=q}J7P-x+VmA!cKqSdy^mmadHe?$SVW_&;*K@7WVVs0*AbR>E z+2|5?i)adz|GV(SaxZ^WL@cSLfbmI{Kk_C%nN9)I7`FJM65O$+ti^a@pMwFYRGjZV z!dCCzb*2TY159D6)qy>!JCcBI0iG&?h1>aM?obzVdx?of{&c~wIw_Sd2;`bpd#ixY>>*U?**vZ(h^4&L2e7wMm# z1gGR(H>AVgzd8%bk9i~U0$I$;CF$_6;+pH`q99y0a54N_Zg}Y0T-J5n%*rs%_k>YnBxxqiW>XUBdI>W86Ohvg=@40bs=D}qq}u@G>BIf}(nH2S(!s5wF_z=&gTm+- zbzckScC1$iJAZ)f(w2EO*{M!*@)+@^aT27fxCL(F*ly9xZtOKn49TDxkq)m({#q#Dr^$9?H+1{9)U61)~4bI2r7gc`Hz_RmV0> z{FffCUoZc%`f?1yfu6c+jvg)_3T@dFf#L6)4}~H<@Dp!5^nVqN$n7^JRByV9?ubUM z+eS8xQ0c)OXVN1X(P(h`>vIHVjjP-Z7`9JZ!{F!oZre)W+VAf4I-I75x=F^VRG=lB zW89f?o6YNbNPX{ln%*%%pz3@?=35pu*?O*`h5+BPvpVxwBmcuJwpqu6u35(e@%vJp zC-#(Dn>=)Jki8g7{p*IuN#(rBK%{;=``@H)cu(rq-uRuSkwSb+md(=@eRc;TBK70h z|0dN=FF@TE=xiYQmi)FA(^oz8WTbvP`@U3qhJIMr1G+w!M}Mccw7@U!9Qu&{w7I3D z8~u2iqp>n|inSGoca_b%%@#hV?X#1eIJ^cn?>Jj{)laz0@BLSNnwREf!{Oay^B%K> zjRC?_^xn*5 zx@G3V+7RQOl#6V7^~^D%&h;ObmRwy!2kAA9dV`jmb) zQ%En&=^fB1VPC1auw^Zg@Uf@>1SBO*R_3lZ`0ENz(PZpY!R zWAiH5!ppR|a63&;%pJ{tdc%yz5pMV4RkWBnn6Vt)_~uh4&*WdFx8<(!^%Z;g`isS4 zk&npJ&tL56>Em_mX7I}DyeI)*(j^Rm<%@bO+T761}kCe{K z3o)1GhtbA7SDTqqJi^Go*qQY0JjnD?XuQ4RsPMT8O6sYQjq}F8=P9P<&r(p~vttX! z<;PJ9Z-q=cPeCPm#ug~VF)~$tOqOD9Y?fAn=kq)M=EcaS6vSkWD^L`S6ZM#%tJ8`7 dGgMOJ)z5;=dy$2u^niT1?6MzWlMWhH5Vwl7qL(egJlo}_=`xPNLskApfXiy_&`--McwFnb= ziNp#K$sTi0sUvZY%lDXw5a~s^dXejE1lfimNI0joykuHsi7R?l+h$glmbjz0anonc zs*tFw%bkorkVFEc>FS?^$%p)|C3gKy2dVHwIcRj+`M_ysj%fa%r>T8}(@<9BluC9M zor;re2OTE)!=q$T%Almb*u^P0IvXDX*vGUF66JaLhx{Q5L}`Pf5^}Bx2IfDxCY{wIta16tpGj*E5xm*GIM`G@{yxs<8UX0r^%u@-u zLWdQRAKuH{6S7tbuemOvcTBZ`^Vp1)Vds_5RH$R!qT>IGwcOg6wv z-(qQLSHij<{`FRscc~5JxjlQwXSx!`eKSy#T5SWH=hb%dN_>hv?V>e~Wjthg!PMg2 zga&=tSCNw15DP;O4J_{;V*~H2GhY$;vCvQW%=#kF2K+8Z9=+Kb3m2cbaqeS-4V>}) zNw7v12jfq77hF!afh2bNfkWAGaGLk#9}?;mHW2O=(kqCJgS(CAL=ME&W1{yOxyT#Gn^$@#&9lLn6TvI3TqP_|=X;N@wZSQR z_iMB2m9Z%pj7^CijX}@*Du7|OX)xcVX`vJ>lKvV2DwUWu0K=4qOt~ffK{x ztIM&#m~t1T5H0(2dC}X)KA8vwYOnrEymOuB6r8AB;>cq0p`$2VKliD1U=5n8VUhfFEfSLz8O0sTxsV3-W^R0%voh75I-vmb@n z>XM2vQ7z`i+o1$Io|qu=6;6sb0yRj;qBGQ7L~-h-&+XayoCy zy*OM;Kz{<>`6@CIcTZ8`iDR*Bx`o%@M06Sy>j*b`S`b{LgMajzo9NlfET$)UvM<7Rg0|qTW}Hk5*qc0Qu3I_B$sgLM(RV z-2I0PK)6{%-JJ*r=#Wr{k=>nW;;Aqk!vmGVKGr)amCNLZI;^ZUsld-2=+79z!1e7J zzYLuKH%w_+d+L}GaOubXjciYnR4RPsPX?NR@Uf3BtO(Xjet5u*X97{UO|w~ZRRnaUUs z&GdjzH(Q<@A6%J27C|cxGzJ4`T%f4!bDaPM!+U=`KH9*9GW~1u&Lhch<3nN~$MSDv zwIoGTrzI$>70#eX_z`yFF7M(|R=9W6jrm>^jUc!H64RDgpVMR{D2nv zQ&J=N7?+bVR;X4`>G#pXW-IjXSmExEF26D7Uk31G%*(D_7p(9k{Y0+0XaM&vZ2ad_ ztN^Z_+|!fs*g&81MmwOjfLE`O)ER+f=?{j3?-#&jw67nTXauTWO!SgAM>jkoZQBc1 zJR@yriCdVr|*`k z0G>EO?>s-X%Pd{%Zs)=N?jR~{o*CTUPAt1oP9}2%4a)*oP{A!P3>_r5xz{G7tm8^AV2mY+yza zL4VCFW*24nNMe+CxwP*`c?A3kk? z33N<=yu$?6-s6o)vs$3=BZVWI%~Ow-Rs!H^2L5j>uw~CN!^d}a(T4)6m4vH>36m0G zX6A&VnEgua6Ee1kSe)m{}B9o}pW%10PJ4eOGlh2cEv4_s{H39Z=a(E)Bk0&$?@Z`E3+A zHwOwE68qQ`O=_KRo0~3f6v$q>Z>hv57y4H2>I#@J3aIKaVu8PJd_2aduabf9(!@P1 zD?a|jqR7k;RcVM<6FP}RE-7z@5TJ-&5sYCp8-gXYO^!Ss!W9t53q_JNNwxnz#-$L7 zM>kjgQl|sk4iFIqA)WEmnwNcE;Ghqnnf=EHQ4M_*mB=)ji^O^V`*d{|M!xP;zLqQdiG z(+QP3koV<5<3aQ@^V9*ljZX!Bk_UxT+=vhz^BqOm@}T3!LbLZ--s8Ks-ldcs>eP&%RWzd(Q%0=X9O=`ec?g%-Zy{ zV4r8Q1!7l29wojx3SeR%{&%sKkkT;KlCUy+UhUokG7FUN8rq{t9tEgS`J&Y#Bz=}S z9wwN1amGS%<^im0P&-E_%Wp!wv>;D0unp>oXTa-Y*$K;LsCmu@jLOWlZC z?P(Qp$jlt0h~ZjhJw=={Gm|KauLX}&O2Y}PB56rDb$NJdRZM%hSI>2wB;Z}y@ojyP-z$&{aXL_kwECfTY5 zen6EA)7{aZa+uzhx&eL>_cIiuovnc%U8s%x}M zM7iBdY=Ud07lqOYa!1d+xXcJ_7Ck3S{HYNhlr$Ga-s6{?Tim5XMrBtKTN?fz+$s8yfq>VwBo0!Oas*6hZ@4fBeZwTV#UQ8(A<9NadndsN{(*UrD=e| z`^kIFwMOVDvv251LLROoQMq+)^kU(zRt263`lH`GtQauDOtp^y2@PVh8l~vbLc#SBVD+)!d88S(mQ)ajQy8_I;v=M$#CdXfQ7U+# z?o%8vLr~Gz(t9%%G~akm?J`402_#pi0d76@nHhHAV!cl~DEnUVpzAKTFmBF?|KPMmd&e$=l<3KuS8$!zWagox`HIER-T>gNCDe= zD2o*eXda(POz{EqX&!}^Q%|qp5hd0Lh+Q%$Nd&4+7I>t9YxC(OPedwlB{4CDfqdd? z>n=NX$z4L-aDv(1&b=+p+{esg1;#fM<#3xP_Fn~uqPm54e0|l^2 z{am3<2Z~&uSE>M1JgO@+>(jx(Y>E>VT?my=>i-cDq+&ky4jowt!Sij4u79A)ZcT6Mc<=|5%QxMfN9}~E7 zOZ#Osvi-SIm*6k#te+vkl@9jrzlnFI8E)ax!sWii-cgK$Pqt2kLP!$cI;PGyvQr)M z9!=*3`lqhhZ@ykt3WZjHs-hCpZZ3V&0|i@P*2JSyJ28MR6Nn$D7tOp@SIStu^v4rG z>_Uxx#o|(UfAi_#S3gJqbLm8HVofPC;Dbu^_su3LIyceH>zh*WJLs3o$zMGA z^+idr3YB^Gl<^w+3zAsVB3qylsuQpl*%l2phCU*R{7kcS305z=su5SNd$=&8q=ab} zQrI{kO!3pwb$n`mObHbJ;wyM_ENH%)G+yh!k-qq9#N%QZxIlZ`8o5PV!RA=~#t5%! z<=j=rH@)#rF}!xIS@??<X&6{v^7 z29!#yzpMC*Ni<~)Lu@fQnDQA+{x@7C^?Ngi^Lx_>i2ec&*|nvTg8v8KKR%=(O%MzJ zFaw?7f93uixaJSf7Y=qG*jN97-ZBJT(Y`Vx*9Qfh=zrz@7xckS32QHtpmx!^vZ0DW zmJ?6u`s1a_iyCBQko%u_Y)@GzQv0balps;P&IXj9cH#Rd*j&3$C&xCK?@luJapn;& z`!z|nHHqzZSFl^{QDTzm#+e>mHjpf9O8!&#j9p+;QY7;;&isnYeom3~vf(p;FUf4j znLW5{f12!is#Dsz+a)tO%Gb^~aM|-09NFdj3Li{yOe;++31ejM{IZec2UOR$N&eglf9=vj*{53j%8#A_ddJIi6 z>v3i?F59r!MRvV`JvBR0y!AE$MPXqsp5u6NFdv23Wpep+@uw_r&RVZzWsFi09h)2z zmppvL2t}-uX1|&vY;_GxP8ge+P@~UO z&dte8n3tm|P0TUX%+1lLXOGRSNfZjj{g+RuP$+ From 96412453a611ee88b12b3535a491006e411c500e Mon Sep 17 00:00:00 2001 From: Felix Schott Date: Fri, 21 Mar 2025 15:10:43 +0100 Subject: [PATCH 08/11] update readme --- readme.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/readme.md b/readme.md index 542d7c0..049b2ad 100644 --- a/readme.md +++ b/readme.md @@ -82,6 +82,18 @@ The `describe` command prints schema information and metadata about a GeoParquet gpq describe example.parquet ``` +### extract + +The `extract` command can be use to extract columns and/or rows from a local or remote GeoParquet file. + +``` +gpq extract input.parquet output.parquet --bbox=xmin,ymin,xmax,ymax --drop-cols=col1,col2 +``` + +Instead of negatively selecting columns by specifying which ones to drop (`--drop-cols`), you can alternatively use the `--keep-only-cols` argument to explicitely select those columns that you wish to keep in the data set. + +The `--bbox` argument allows you to extract features whose bounding box intersects with the provided bbox. Note that this doesn't support exact geometry filtering and will only operate on bounding boxes of full feature geometries. It is thus recommended to use the `--bbox` argument for preliminary filtering only. The algorithm will attempt to use an existing bounding box column in the file. If bounding box information is not available, the bounding boxes will be computed on the fly. If the GeoParquet file is spatially partitioned using row groups, the algorithm will use row group statistics to speed up the filtering process. + ## Limitations * Non-geographic CRS information is not preserved when converting GeoParquet to GeoJSON. From 90f3e1429325f62bdc0d1deb0895b1ba4ebe3f2d Mon Sep 17 00:00:00 2001 From: Felix Schott Date: Thu, 27 Mar 2025 14:33:50 +0100 Subject: [PATCH 09/11] add write support for bbox data --- cmd/gpq/command/convert.go | 2 + internal/geo/geo.go | 23 +- internal/geojson/featurereader.go | 18 + internal/geojson/featurereader_test.go | 43 + internal/geojson/geojson.go | 39 +- internal/geojson/geojson_test.go | 155 +- internal/geojson/recordwriter.go | 30 + internal/geojson/testdata/bad-bbox.geojson | 135 + internal/geojson/testdata/bbox.geojson | 5658 ++++++++++++++++++++ internal/geoparquet/featurewriter.go | 19 +- internal/geoparquet/geoparquet.go | 18 +- internal/geoparquet/geoparquet_test.go | 2 +- internal/geoparquet/metadata.go | 16 +- internal/geoparquet/recordwriter.go | 14 +- internal/geoparquet/writer.go | 11 +- internal/pqutil/arrow.go | 11 + 16 files changed, 6137 insertions(+), 57 deletions(-) create mode 100644 internal/geojson/testdata/bad-bbox.geojson create mode 100644 internal/geojson/testdata/bbox.geojson diff --git a/cmd/gpq/command/convert.go b/cmd/gpq/command/convert.go index 4e6b1f8..dd12574 100644 --- a/cmd/gpq/command/convert.go +++ b/cmd/gpq/command/convert.go @@ -35,6 +35,7 @@ type ConvertCmd struct { InputPrimaryColumn string `help:"Primary geometry column name when reading Parquet without metadata." default:"geometry"` Compression string `help:"Parquet compression to use. Possible values: ${enum}." enum:"uncompressed, snappy, gzip, brotli, zstd" default:"zstd"` RowGroupLength int `help:"Maximum number of rows per group when writing Parquet."` + AddBbox bool `help:"Compute the bounding box of features where not yet available and write to Parquet output."` } type FormatType string @@ -157,6 +158,7 @@ func (c *ConvertCmd) Run() error { MaxFeatures: c.Max, Compression: c.Compression, RowGroupLength: c.RowGroupLength, + AddBbox: c.AddBbox, } if err := geojson.ToParquet(input, output, convertOptions); err != nil { return NewCommandError("%w", err) diff --git a/internal/geo/geo.go b/internal/geo/geo.go index 74b367a..f6a263e 100644 --- a/internal/geo/geo.go +++ b/internal/geo/geo.go @@ -36,6 +36,7 @@ type Feature struct { Id any `json:"id,omitempty"` Type string `json:"type"` Geometry orb.Geometry `json:"geometry"` + Bbox *orb.Bound `json:"bbox,omitempty"` Properties map[string]any `json:"properties"` } @@ -53,12 +54,16 @@ func (f *Feature) MarshalJSON() ([]byte, error) { if f.Id != nil { m["id"] = f.Id } + if f.Bbox != nil { + m["bbox"] = f.Bbox + } return json.Marshal(m) } type jsonFeature struct { Id any `json:"id,omitempty"` Type string `json:"type"` + Bbox *orbjson.BBox `json:"bbox,omitempty"` Geometry json.RawMessage `json:"geometry"` Properties map[string]any `json:"properties"` } @@ -96,6 +101,14 @@ func (f *Feature) UnmarshalJSON(data []byte) error { } f.Geometry = geometry.Geometry() + + if jf.Bbox != nil { + if !jf.Bbox.Valid() { + return errors.New("invalid bbox, make sure it is an array of at least 4 floats") + } + bound := jf.Bbox.Bound() + f.Bbox = &bound + } return nil } @@ -338,11 +351,13 @@ func (i *DatasetStats) Types(name string) []string { return collection.Types() } +// BBOX type + type Bbox struct { - Xmin float64 - Ymin float64 - Xmax float64 - Ymax float64 + Xmin float64 `parquet:"name=xmin" json:"xmin"` + Ymin float64 `parquet:"name=ymin" json:"ymin"` + Xmax float64 `parquet:"name=xmax" json:"xmax"` + Ymax float64 `parquet:"name=ymax" json:"ymax"` } // Checks whether the bbox overlaps with another axis-aligned bbox. diff --git a/internal/geojson/featurereader.go b/internal/geojson/featurereader.go index af64571..a04832a 100644 --- a/internal/geojson/featurereader.go +++ b/internal/geojson/featurereader.go @@ -95,6 +95,24 @@ func (r *FeatureReader) Read() (*geo.Feature, error) { continue } + if key == "bbox" { + if feature == nil { + feature = &geo.Feature{} + } else if feature.Bbox != nil { + return nil, errors.New("found duplicate bbox") + } + bbox := &orbjson.BBox{} + if err := r.decoder.Decode(bbox); err != nil { + return nil, fmt.Errorf("trouble parsing bbox: %w", err) + } + if !bbox.Valid() { + return nil, errors.New("invalid bbox, make sure it is an array of at least 4 floats") + } + bound := bbox.Bound() + feature.Bbox = &bound + continue + } + if key == "properties" { if feature == nil { feature = &geo.Feature{} diff --git a/internal/geojson/featurereader_test.go b/internal/geojson/featurereader_test.go index 31f4ed4..1c0cc4d 100644 --- a/internal/geojson/featurereader_test.go +++ b/internal/geojson/featurereader_test.go @@ -120,6 +120,49 @@ func TestFeatureReaderNewLineDelimited(t *testing.T) { assert.Equal(t, float64(326625791), usa.Properties["pop_est"]) } +func TestFeatureReaderBbox(t *testing.T) { + file, openErr := os.Open("testdata/bbox.geojson") + require.NoError(t, openErr) + + reader := geojson.NewFeatureReader(file) + + features := []*geo.Feature{} + for { + feature, err := reader.Read() + if err == io.EOF { + break + } + require.NoError(t, err) + features = append(features, feature) + } + require.Len(t, features, 5) + + feature := features[0] + require.NotNil(t, feature.Geometry) + assert.Equal(t, "MultiPolygon", feature.Geometry.GeoJSONType()) + assert.Equal(t, map[string]any{ + "continent": "Oceania", + "gdp_md_est": 8374.0, + "iso_a3": "FJI", + "name": "Fiji", + "pop_est": 920938.0}, feature.Properties) + assert.Equal(t, -180.0, feature.Bbox.Min.X()) + assert.Equal(t, -18.28799, feature.Bbox.Min.Y()) + assert.Equal(t, 180.0, feature.Bbox.Max.X()) + assert.Equal(t, -16.020882256741224, feature.Bbox.Max.Y()) +} + +func TestFeatureReaderBboxInvalid(t *testing.T) { + file, openErr := os.Open("testdata/bad-bbox.geojson") + require.NoError(t, openErr) + + reader := geojson.NewFeatureReader(file) + + feature, err := reader.Read() + require.ErrorContains(t, err, "invalid bbox") + require.Nil(t, feature) +} + func TestFeatureReaderBadNewLineDelimited(t *testing.T) { file, openErr := os.Open("testdata/bad-new-line-delimited.ndgeojson") require.NoError(t, openErr) diff --git a/internal/geojson/geojson.go b/internal/geojson/geojson.go index be16c0d..8a7d7db 100644 --- a/internal/geojson/geojson.go +++ b/internal/geojson/geojson.go @@ -10,21 +10,6 @@ import ( "github.com/planetlabs/gpq/internal/pqutil" ) -const primaryColumn = "geometry" - -func GetDefaultMetadata() *geoparquet.Metadata { - return &geoparquet.Metadata{ - Version: geoparquet.Version, - PrimaryColumn: primaryColumn, - Columns: map[string]*geoparquet.GeometryColumn{ - primaryColumn: { - Encoding: "WKB", - GeometryTypes: []string{}, - }, - }, - } -} - func FromParquet(reader parquet.ReaderAtSeeker, writer io.Writer) error { recordReader, rrErr := geoparquet.NewRecordReaderFromConfig(&geoparquet.ReaderConfig{ Reader: reader, @@ -63,6 +48,7 @@ type ConvertOptions struct { Compression string RowGroupLength int Metadata string + AddBbox bool } var defaultOptions = &ConvertOptions{ @@ -96,6 +82,8 @@ func ToParquet(input io.Reader, output io.Writer, convertOptions *ConvertOptions pqWriterProps = parquet.NewWriterProperties(writerOptions...) } + writeCoveringMetadata := convertOptions.AddBbox + var featureWriter *geoparquet.FeatureWriter writeBuffered := func() error { if !builder.Ready() { @@ -109,9 +97,10 @@ func ToParquet(input io.Reader, output io.Writer, convertOptions *ConvertOptions return scErr } fw, fwErr := geoparquet.NewFeatureWriter(&geoparquet.WriterConfig{ - Writer: output, - ArrowSchema: sc, - ParquetWriterProps: pqWriterProps, + Writer: output, + ArrowSchema: sc, + ParquetWriterProps: pqWriterProps, + WriteCoveringMetadata: writeCoveringMetadata, }) if fwErr != nil { return fwErr @@ -135,11 +124,25 @@ func ToParquet(input io.Reader, output io.Writer, convertOptions *ConvertOptions return err } featuresRead += 1 + + if feature.Bbox == nil && convertOptions.AddBbox { + bound := feature.Geometry.Bound() + feature.Bbox = &bound + } + + if feature.Bbox != nil { + writeCoveringMetadata = true + } + if featureWriter == nil { if err := builder.Add(feature.Properties); err != nil { return err } + if feature.Bbox != nil { + builder.AddBbox(geoparquet.DefaultBboxColumn) + } + if !builder.Ready() { buffer = append(buffer, feature) if len(buffer) > convertOptions.MaxFeatures { diff --git a/internal/geojson/geojson_test.go b/internal/geojson/geojson_test.go index 852c4bf..86e547c 100644 --- a/internal/geojson/geojson_test.go +++ b/internal/geojson/geojson_test.go @@ -17,6 +17,7 @@ package geojson_test import ( "bytes" "encoding/json" + "fmt" "os" "strings" "testing" @@ -83,6 +84,12 @@ func TestToParquet(t *testing.T) { metadata, geoErr := geoparquet.GetMetadata(fileReader.MetaData().KeyValueMetadata()) require.NoError(t, geoErr) + assert.Equal(t, "geometry", metadata.PrimaryColumn) + // check if covering metadata has been written + metadata, err := geoparquet.GetMetadata(fileReader.MetaData().KeyValueMetadata()) + require.NoError(t, err) + + assert.Nil(t, metadata.Columns[metadata.PrimaryColumn].Covering) geometryTypes := metadata.Columns[metadata.PrimaryColumn].GetGeometryTypes() assert.Len(t, geometryTypes, 2) assert.Contains(t, geometryTypes, "MultiPolygon") @@ -273,6 +280,78 @@ func TestToParquetNumberId(t *testing.T) { assert.Equal(t, []string{"Point"}, geometryTypes) } +func TestToParquetExistingBbox(t *testing.T) { + geojsonFile, openErr := os.Open("testdata/bbox.geojson") + require.NoError(t, openErr) + + parquetBuffer := &bytes.Buffer{} + toParquetErr := geojson.ToParquet(geojsonFile, parquetBuffer, nil) + assert.NoError(t, toParquetErr) + + parquetInput := bytes.NewReader(parquetBuffer.Bytes()) + fileReader, fileErr := file.NewParquetReader(parquetInput) + require.NoError(t, fileErr) + defer fileReader.Close() + + // check if covering metadata has been written + metadata, err := geoparquet.GetMetadata(fileReader.MetaData().KeyValueMetadata()) + require.NoError(t, err) + + require.NotNil(t, metadata.Columns[metadata.PrimaryColumn].Covering) + assert.Equal(t, []string{"bbox", "xmin"}, metadata.Columns[metadata.PrimaryColumn].Covering.Bbox.Xmin) + assert.Equal(t, []string{"bbox", "ymin"}, metadata.Columns[metadata.PrimaryColumn].Covering.Bbox.Ymin) + assert.Equal(t, []string{"bbox", "xmax"}, metadata.Columns[metadata.PrimaryColumn].Covering.Bbox.Xmax) + assert.Equal(t, []string{"bbox", "ymax"}, metadata.Columns[metadata.PrimaryColumn].Covering.Bbox.Ymax) + + assert.NotEqual(t, -1, fileReader.MetaData().Schema.ColumnIndexByName("bbox.xmin")) + assert.Equal(t, int64(5), fileReader.NumRows()) + + geojsonBuffer := &bytes.Buffer{} + fromParquetErr := geojson.FromParquet(parquetInput, geojsonBuffer) + require.NoError(t, fromParquetErr) + + expected, err := os.ReadFile("testdata/bbox.geojson") + require.NoError(t, err) + + assert.JSONEq(t, string(expected), geojsonBuffer.String()) +} + +func TestToParquetAddBbox(t *testing.T) { + geojsonFile, openErr := os.Open("testdata/example.geojson") + require.NoError(t, openErr) + + parquetBuffer := &bytes.Buffer{} + toParquetErr := geojson.ToParquet(geojsonFile, parquetBuffer, &geojson.ConvertOptions{AddBbox: true}) + assert.NoError(t, toParquetErr) + + parquetInput := bytes.NewReader(parquetBuffer.Bytes()) + fileReader, fileErr := file.NewParquetReader(parquetInput) + require.NoError(t, fileErr) + defer fileReader.Close() + + assert.NotEqual(t, -1, fileReader.MetaData().Schema.ColumnIndexByName("bbox.xmin")) + assert.Equal(t, int64(5), fileReader.NumRows()) + + // check if covering metadata has been written + metadata, err := geoparquet.GetMetadata(fileReader.MetaData().KeyValueMetadata()) + require.NoError(t, err) + + require.NotNil(t, metadata.Columns[metadata.PrimaryColumn].Covering) + assert.Equal(t, []string{"bbox", "xmin"}, metadata.Columns[metadata.PrimaryColumn].Covering.Bbox.Xmin) + assert.Equal(t, []string{"bbox", "ymin"}, metadata.Columns[metadata.PrimaryColumn].Covering.Bbox.Ymin) + assert.Equal(t, []string{"bbox", "xmax"}, metadata.Columns[metadata.PrimaryColumn].Covering.Bbox.Xmax) + assert.Equal(t, []string{"bbox", "ymax"}, metadata.Columns[metadata.PrimaryColumn].Covering.Bbox.Ymax) + + geojsonBuffer := &bytes.Buffer{} + fromParquetErr := geojson.FromParquet(parquetInput, geojsonBuffer) + require.NoError(t, fromParquetErr) + + expected, err := os.ReadFile("testdata/bbox.geojson") + require.NoError(t, err) + + assert.JSONEq(t, string(expected), geojsonBuffer.String()) +} + func TestToParquetBooleanId(t *testing.T) { geojsonFile, openErr := os.Open("testdata/boolean-id.geojson") require.NoError(t, openErr) @@ -457,7 +536,7 @@ func TestRoundTripSparseProperties(t *testing.T) { assert.JSONEq(t, string(inputData), jsonBuffer.String()) } -func makeGeoParquetReader[T any](rows []T, metadata *geoparquet.Metadata) (*bytes.Reader, error) { +func makeGeoParquetReader[T any](rows []T, metadata *geoparquet.Metadata, writeCoveringMetadata bool) (*bytes.Reader, error) { data, err := json.Marshal(rows) if err != nil { return nil, err @@ -465,19 +544,20 @@ func makeGeoParquetReader[T any](rows []T, metadata *geoparquet.Metadata) (*byte parquetSchema, err := schema.NewSchemaFromStruct(rows[0]) if err != nil { - return nil, err + return nil, fmt.Errorf("cannot create parquet schema from struct: %w", err) } arrowSchema, err := pqarrow.FromParquet(parquetSchema, nil, nil) if err != nil { - return nil, err + return nil, fmt.Errorf("cannot create arrow schema from struct: %w", err) } output := &bytes.Buffer{} recordWriter, err := geoparquet.NewRecordWriter(&geoparquet.WriterConfig{ - Writer: output, - Metadata: metadata, - ArrowSchema: arrowSchema, + Writer: output, + Metadata: metadata, + ArrowSchema: arrowSchema, + WriteCoveringMetadata: writeCoveringMetadata, }) if err != nil { return nil, err @@ -485,7 +565,7 @@ func makeGeoParquetReader[T any](rows []T, metadata *geoparquet.Metadata) (*byte rec, _, err := array.RecordFromJSON(memory.DefaultAllocator, arrowSchema, strings.NewReader(string(data))) if err != nil { - return nil, err + return nil, fmt.Errorf("cannot create record from json: %w", err) } if err := recordWriter.Write(rec); err != nil { @@ -515,10 +595,10 @@ func TestWKT(t *testing.T) { }, } - metadata := geoparquet.DefaultMetadata() + metadata := geoparquet.DefaultMetadata(false) metadata.Columns[metadata.PrimaryColumn].Encoding = geo.EncodingWKT - reader, readerErr := makeGeoParquetReader(rows, metadata) + reader, readerErr := makeGeoParquetReader(rows, metadata, false) require.NoError(t, readerErr) output := &bytes.Buffer{} @@ -567,10 +647,10 @@ func TestWKTNoEncoding(t *testing.T) { }, } - metadata := geoparquet.DefaultMetadata() + metadata := geoparquet.DefaultMetadata(false) metadata.Columns[metadata.PrimaryColumn].Encoding = "" - reader, readerErr := makeGeoParquetReader(rows, metadata) + reader, readerErr := makeGeoParquetReader(rows, metadata, false) require.NoError(t, readerErr) output := &bytes.Buffer{} @@ -612,9 +692,9 @@ func TestWKB(t *testing.T) { }, } - metadata := geoparquet.DefaultMetadata() + metadata := geoparquet.DefaultMetadata(false) - reader, readerErr := makeGeoParquetReader(rows, metadata) + reader, readerErr := makeGeoParquetReader(rows, metadata, false) require.NoError(t, readerErr) output := &bytes.Buffer{} @@ -656,10 +736,10 @@ func TestWKBNoEncoding(t *testing.T) { }, } - metadata := geoparquet.DefaultMetadata() + metadata := geoparquet.DefaultMetadata(false) metadata.Columns[metadata.PrimaryColumn].Encoding = "" - reader, readerErr := makeGeoParquetReader(rows, metadata) + reader, readerErr := makeGeoParquetReader(rows, metadata, false) require.NoError(t, readerErr) output := &bytes.Buffer{} @@ -774,3 +854,48 @@ func TestCodecInvalid(t *testing.T) { toParquetErr := geojson.ToParquet(geojsonFile, parquetBuffer, convertOptions) assert.EqualError(t, toParquetErr, "invalid compression codec invalid") } + +func TestCoveringMetadata(t *testing.T) { + type Row struct { + Name string `parquet:"name=name, logical=String" json:"name"` + Geometry string `parquet:"name=geometry, logical=String" json:"geometry"` + Bbox geo.Bbox `parquet:"name=bbox" json:"bbox"` + } + + rows := []*Row{ + { + Name: "test-point", + Geometry: "POINT (1 2)", + Bbox: geo.Bbox{Xmin: 1, Ymin: 2, Xmax: 1, Ymax: 2}, + }, + } + + metadata := geoparquet.DefaultMetadata(true) + metadata.Columns[metadata.PrimaryColumn].Encoding = geo.EncodingWKT + + reader, readerErr := makeGeoParquetReader(rows, metadata, true) + require.NoError(t, readerErr) + + output := &bytes.Buffer{} + convertErr := geojson.FromParquet(reader, output) + require.NoError(t, convertErr) + + expected := `{ + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "bbox": [1, 2, 1, 2], + "properties": { + "name": "test-point" + }, + "geometry": { + "type": "Point", + "coordinates": [1, 2] + } + } + ] + }` + + assert.JSONEq(t, expected, output.String()) +} diff --git a/internal/geojson/recordwriter.go b/internal/geojson/recordwriter.go index 69a67dc..52d6488 100644 --- a/internal/geojson/recordwriter.go +++ b/internal/geojson/recordwriter.go @@ -2,6 +2,8 @@ package geojson import ( "encoding/json" + "errors" + "fmt" "io" "github.com/apache/arrow/go/v16/arrow" @@ -51,6 +53,7 @@ func (w *RecordWriter) Write(record arrow.Record) error { } var geometry *orbjson.Geometry + var bbox *orbjson.BBox properties := map[string]any{} for fieldNum := 0; fieldNum < arr.NumField(); fieldNum += 1 { value := arr.Field(fieldNum).GetOneForMarshal(rowNum) @@ -67,6 +70,29 @@ func (w *RecordWriter) Write(record arrow.Record) error { properties[name] = g continue } + + bboxCol := geoparquet.GetBboxColumnNameFromMetadata(w.geoMetadata) + if value != nil && (name == bboxCol || (bboxCol == "" && name == geoparquet.DefaultBboxColumn)) { + bboxMap, ok := value.(map[string]any) + if !ok { + return errors.New("value is not of type map[string]any") + } + fieldNames := geoparquet.GetBboxColumnFieldNames(w.geoMetadata) + xmin, xminOk := bboxMap[fieldNames.Xmin] + ymin, yminOk := bboxMap[fieldNames.Ymin] + xmax, xmaxOk := bboxMap[fieldNames.Xmax] + ymax, ymaxOk := bboxMap[fieldNames.Ymax] + if !(xminOk && yminOk && xmaxOk && ymaxOk) { + return fmt.Errorf("bbox struct must have fields %v/%v/%v/%v", fieldNames.Xmin, fieldNames.Ymin, fieldNames.Xmax, fieldNames.Ymax) + } + if xmin == nil || ymin == nil || xmax == nil || ymax == nil { + return errors.New("bbox struct must have non-null values") + } + orbBbox := orbjson.BBox([]float64{xmin.(float64), ymin.(float64), xmax.(float64), ymax.(float64)}) + bbox = &orbBbox + continue + } + properties[name] = value } @@ -76,6 +102,10 @@ func (w *RecordWriter) Write(record arrow.Record) error { "geometry": geometry, } + if bbox != nil { + feature["bbox"] = bbox + } + featureData, jsonErr := json.Marshal(feature) if jsonErr != nil { return jsonErr diff --git a/internal/geojson/testdata/bad-bbox.geojson b/internal/geojson/testdata/bad-bbox.geojson new file mode 100644 index 0000000..3670651 --- /dev/null +++ b/internal/geojson/testdata/bad-bbox.geojson @@ -0,0 +1,135 @@ +{ + "type": "FeatureCollection", + "features": [ + { + "bbox": [ + -17.06342322434257, + 20.999752102130827, + -8.665124477564191 + ], + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -8.665589565454809, + 27.656425889592356 + ], + [ + -8.665124477564191, + 27.589479071558227 + ], + [ + -8.684399786809053, + 27.395744126896005 + ], + [ + -8.6872936670174, + 25.881056219988906 + ], + [ + -11.96941891117116, + 25.933352769468268 + ], + [ + -11.937224493853321, + 23.374594224536168 + ], + [ + -12.874221564169575, + 23.284832261645178 + ], + [ + -13.118754441774712, + 22.771220201096256 + ], + [ + -12.929101935263532, + 21.327070624267563 + ], + [ + -16.845193650773993, + 21.33332347257488 + ], + [ + -17.06342322434257, + 20.999752102130827 + ], + [ + -17.02042843267577, + 21.422310288981578 + ], + [ + -17.00296179856109, + 21.420734157796577 + ], + [ + -14.750954555713534, + 21.500600083903663 + ], + [ + -14.630832688851072, + 21.860939846274903 + ], + [ + -14.221167771857253, + 22.31016307218816 + ], + [ + -13.891110398809047, + 23.691009019459305 + ], + [ + -12.50096269372537, + 24.7701162785782 + ], + [ + -12.03075883630163, + 26.030866197203068 + ], + [ + -11.718219773800357, + 26.104091701760623 + ], + [ + -11.392554897497007, + 26.883423977154393 + ], + [ + -10.551262579785273, + 26.990807603456886 + ], + [ + -10.189424200877582, + 26.860944729107405 + ], + [ + -9.735343390328879, + 26.860944729107405 + ], + [ + -9.41303748212448, + 27.088476060488574 + ], + [ + -8.794883999049077, + 27.120696316022507 + ], + [ + -8.817828334986672, + 27.656425889592356 + ], + [ + -8.665589565454809, + 27.656425889592356 + ] + ] + ] + }, + "properties": { + "continent": "Africa" + }, + "type": "Feature" + } + ] +} \ No newline at end of file diff --git a/internal/geojson/testdata/bbox.geojson b/internal/geojson/testdata/bbox.geojson new file mode 100644 index 0000000..aa2797f --- /dev/null +++ b/internal/geojson/testdata/bbox.geojson @@ -0,0 +1,5658 @@ +{ + "type": "FeatureCollection", + "features": [ + { + "geometry": { + "type": "MultiPolygon", + "coordinates": [ + [ + [ + [ + 180, + -16.067132663642447 + ], + [ + 180, + -16.555216566639196 + ], + [ + 179.36414266196414, + -16.801354076946883 + ], + [ + 178.72505936299711, + -17.01204167436804 + ], + [ + 178.59683859511713, + -16.639150000000004 + ], + [ + 179.0966093629971, + -16.433984277547403 + ], + [ + 179.4135093629971, + -16.379054277547404 + ], + [ + 180, + -16.067132663642447 + ] + ] + ], + [ + [ + [ + 178.12557, + -17.50481 + ], + [ + 178.3736, + -17.33992 + ], + [ + 178.71806, + -17.62846 + ], + [ + 178.55271, + -18.15059 + ], + [ + 177.93266000000003, + -18.28799 + ], + [ + 177.38146, + -18.16432 + ], + [ + 177.28504, + -17.72465 + ], + [ + 177.67087, + -17.381140000000002 + ], + [ + 178.12557, + -17.50481 + ] + ] + ], + [ + [ + [ + -179.79332010904864, + -16.020882256741224 + ], + [ + -179.9173693847653, + -16.501783135649397 + ], + [ + -180, + -16.555216566639196 + ], + [ + -180, + -16.067132663642447 + ], + [ + -179.79332010904864, + -16.020882256741224 + ] + ] + ] + ] + }, + "properties": { + "continent": "Oceania", + "gdp_md_est": 8374, + "iso_a3": "FJI", + "name": "Fiji", + "pop_est": 920938 + }, + "type": "Feature", + "bbox": [ + -180.0, + -18.28799, + 180.0, + -16.020882256741224 + ] + }, + { + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 33.90371119710453, + -0.9500000000000001 + ], + [ + 34.07261999999997, + -1.0598199999999451 + ], + [ + 37.69868999999994, + -3.0969899999999484 + ], + [ + 37.7669, + -3.6771200000000004 + ], + [ + 39.20222, + -4.67677 + ], + [ + 38.74053999999995, + -5.9089499999999475 + ], + [ + 38.79977000000008, + -6.475660000000005 + ], + [ + 39.44, + -6.839999999999861 + ], + [ + 39.47000000000014, + -7.099999999999966 + ], + [ + 39.19468999999998, + -7.703899999999976 + ], + [ + 39.25203000000005, + -8.00780999999995 + ], + [ + 39.18652000000009, + -8.48550999999992 + ], + [ + 39.53574000000009, + -9.112369999999885 + ], + [ + 39.94960000000003, + -10.098400000000026 + ], + [ + 40.316586229110854, + -10.317097752817492 + ], + [ + 40.31659000000002, + -10.317099999999868 + ], + [ + 39.52099999999996, + -10.89688000000001 + ], + [ + 38.42755659358775, + -11.285202325081656 + ], + [ + 37.827639999999974, + -11.26878999999991 + ], + [ + 37.471289999999954, + -11.568759999999997 + ], + [ + 36.775150994622805, + -11.594537448780805 + ], + [ + 36.51408165868426, + -11.720938002166735 + ], + [ + 35.31239790216904, + -11.439146416879147 + ], + [ + 34.55998904799935, + -11.520020033415925 + ], + [ + 34.27999999999997, + -10.160000000000025 + ], + [ + 33.940837724096525, + -9.693673841980285 + ], + [ + 33.73972000000009, + -9.417149999999992 + ], + [ + 32.75937544122132, + -9.23059905358906 + ], + [ + 32.19186486179194, + -8.930358981973257 + ], + [ + 31.556348097466497, + -8.762048841998642 + ], + [ + 31.15775133695005, + -8.594578747317366 + ], + [ + 30.740009731422095, + -8.34000593035372 + ], + [ + 30.74001549655179, + -8.340007419470915 + ], + [ + 30.199996779101696, + -7.079980970898163 + ], + [ + 29.620032179490014, + -6.520015150583426 + ], + [ + 29.419992710088167, + -5.939998874539434 + ], + [ + 29.519986606572928, + -5.419978936386315 + ], + [ + 29.339997592900346, + -4.4999834122940925 + ], + [ + 29.753512404099865, + -4.452389418153302 + ], + [ + 30.11632000000003, + -4.090120000000013 + ], + [ + 30.505539999999996, + -3.5685799999999404 + ], + [ + 30.752240000000086, + -3.3593099999999936 + ], + [ + 30.743010000000027, + -3.034309999999948 + ], + [ + 30.527660000000026, + -2.807619999999986 + ], + [ + 30.469673645761223, + -2.41385475710134 + ], + [ + 30.469670000000008, + -2.4138299999999617 + ], + [ + 30.75830895358311, + -2.2872502579883687 + ], + [ + 30.816134881317712, + -1.6989140763453887 + ], + [ + 30.419104852019245, + -1.1346591121504161 + ], + [ + 30.769860000000108, + -1.0145499999999856 + ], + [ + 31.866170000000068, + -1.0273599999999306 + ], + [ + 33.90371119710453, + -0.9500000000000001 + ] + ] + ] + }, + "properties": { + "continent": "Africa", + "gdp_md_est": 150600, + "iso_a3": "TZA", + "name": "Tanzania", + "pop_est": 53950935 + }, + "type": "Feature", + "bbox": [ + 29.339997592900346, + -11.720938002166735, + 40.31659000000002, + -0.9500000000000001 + ] + }, + { + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -8.665589565454809, + 27.656425889592356 + ], + [ + -8.665124477564191, + 27.589479071558227 + ], + [ + -8.684399786809053, + 27.395744126896005 + ], + [ + -8.6872936670174, + 25.881056219988906 + ], + [ + -11.96941891117116, + 25.933352769468268 + ], + [ + -11.937224493853321, + 23.374594224536168 + ], + [ + -12.874221564169575, + 23.284832261645178 + ], + [ + -13.118754441774712, + 22.771220201096256 + ], + [ + -12.929101935263532, + 21.327070624267563 + ], + [ + -16.845193650773993, + 21.33332347257488 + ], + [ + -17.06342322434257, + 20.999752102130827 + ], + [ + -17.02042843267577, + 21.422310288981578 + ], + [ + -17.00296179856109, + 21.420734157796577 + ], + [ + -14.750954555713534, + 21.500600083903663 + ], + [ + -14.630832688851072, + 21.860939846274903 + ], + [ + -14.221167771857253, + 22.31016307218816 + ], + [ + -13.891110398809047, + 23.691009019459305 + ], + [ + -12.50096269372537, + 24.7701162785782 + ], + [ + -12.03075883630163, + 26.030866197203068 + ], + [ + -11.718219773800357, + 26.104091701760623 + ], + [ + -11.392554897497007, + 26.883423977154393 + ], + [ + -10.551262579785273, + 26.990807603456886 + ], + [ + -10.189424200877582, + 26.860944729107405 + ], + [ + -9.735343390328879, + 26.860944729107405 + ], + [ + -9.41303748212448, + 27.088476060488574 + ], + [ + -8.794883999049077, + 27.120696316022507 + ], + [ + -8.817828334986672, + 27.656425889592356 + ], + [ + -8.665589565454809, + 27.656425889592356 + ] + ] + ] + }, + "properties": { + "continent": "Africa", + "gdp_md_est": 906.5, + "iso_a3": "ESH", + "name": "W. Sahara", + "pop_est": 603253 + }, + "type": "Feature", + "bbox": [ + -17.06342322434257, + 20.999752102130827, + -8.665124477564191, + 27.656425889592356 + ] + }, + { + "geometry": { + "type": "MultiPolygon", + "coordinates": [ + [ + [ + [ + -122.84000000000003, + 49.000000000000114 + ], + [ + -122.97421000000001, + 49.00253777777778 + ], + [ + -124.91024, + 49.98456 + ], + [ + -125.62461, + 50.416560000000004 + ], + [ + -127.43561000000001, + 50.83061 + ], + [ + -127.99276, + 51.71583 + ], + [ + -127.85032, + 52.32961 + ], + [ + -129.12979, + 52.75538 + ], + [ + -129.30523, + 53.561589999999995 + ], + [ + -130.51497, + 54.28757 + ], + [ + -130.53610895273684, + 54.80275447679924 + ], + [ + -130.53611, + 54.802780000000006 + ], + [ + -129.98, + 55.285000000000004 + ], + [ + -130.00778000000003, + 55.915830000000085 + ], + [ + -131.70781, + 56.55212 + ], + [ + -132.73042, + 57.692890000000006 + ], + [ + -133.35556000000003, + 58.41028000000001 + ], + [ + -134.27111000000002, + 58.86111000000005 + ], + [ + -134.94500000000005, + 59.2705600000001 + ], + [ + -135.47583, + 59.787780000000005 + ], + [ + -136.47972000000004, + 59.46389000000005 + ], + [ + -137.4525, + 58.905 + ], + [ + -138.34089, + 59.562110000000004 + ], + [ + -139.03900000000002, + 60 + ], + [ + -140.013, + 60.27682000000001 + ], + [ + -140.99778, + 60.30639000000001 + ], + [ + -140.9925, + 66.00003000000001 + ], + [ + -140.986, + 69.712 + ], + [ + -140.98598761037601, + 69.71199839952635 + ], + [ + -139.12052, + 69.47102 + ], + [ + -137.54636000000002, + 68.99002 + ], + [ + -136.50358, + 68.89804 + ], + [ + -135.62576, + 69.31512000000001 + ], + [ + -134.41464000000002, + 69.62743 + ], + [ + -132.92925000000002, + 69.50534 + ], + [ + -131.43135999999998, + 69.94451 + ], + [ + -129.79471, + 70.19369 + ], + [ + -129.10773, + 69.77927000000001 + ], + [ + -128.36156, + 70.01286 + ], + [ + -128.13817, + 70.48384 + ], + [ + -127.44712000000001, + 70.37721 + ], + [ + -125.75632000000002, + 69.48058 + ], + [ + -124.42483, + 70.1584 + ], + [ + -124.28968, + 69.39968999999999 + ], + [ + -123.06108, + 69.56372 + ], + [ + -122.6835, + 69.85553 + ], + [ + -121.47226, + 69.79778 + ], + [ + -119.94288, + 69.37786 + ], + [ + -117.60268, + 69.01128 + ], + [ + -116.22643, + 68.84151 + ], + [ + -115.24690000000001, + 68.90591 + ], + [ + -113.89793999999999, + 68.3989 + ], + [ + -115.30489, + 67.90261000000001 + ], + [ + -113.49727, + 67.68815000000001 + ], + [ + -110.798, + 67.80611999999999 + ], + [ + -109.94619, + 67.98104000000001 + ], + [ + -108.8802, + 67.38144 + ], + [ + -107.79239, + 67.88736 + ], + [ + -108.81299, + 68.31164 + ], + [ + -108.16721000000001, + 68.65392 + ], + [ + -106.95, + 68.7 + ], + [ + -106.15, + 68.8 + ], + [ + -105.34282000000002, + 68.56122 + ], + [ + -104.33791000000001, + 68.018 + ], + [ + -103.22115000000001, + 68.09775 + ], + [ + -101.45433, + 67.64689 + ], + [ + -99.90195, + 67.80566 + ], + [ + -98.4432, + 67.78165 + ], + [ + -98.5586, + 68.40394 + ], + [ + -97.66948000000001, + 68.57864000000001 + ], + [ + -96.11991, + 68.23939 + ], + [ + -96.12588, + 67.29338 + ], + [ + -95.48943, + 68.0907 + ], + [ + -94.685, + 68.06383 + ], + [ + -94.23282000000002, + 69.06903000000001 + ], + [ + -95.30408, + 69.68571 + ], + [ + -96.47131, + 70.08976 + ], + [ + -96.39115, + 71.19482 + ], + [ + -95.2088, + 71.92053 + ], + [ + -93.88997, + 71.76015 + ], + [ + -92.87818, + 71.31869 + ], + [ + -91.51964000000001, + 70.19129000000001 + ], + [ + -92.40692000000001, + 69.69997000000001 + ], + [ + -90.5471, + 69.49766 + ], + [ + -90.55151000000001, + 68.47499 + ], + [ + -89.21515, + 69.25873 + ], + [ + -88.01966, + 68.61508 + ], + [ + -88.31748999999999, + 67.87338000000001 + ], + [ + -87.35017, + 67.19872 + ], + [ + -86.30606999999999, + 67.92146 + ], + [ + -85.57664, + 68.78456 + ], + [ + -85.52197, + 69.88211 + ], + [ + -84.10081000000001, + 69.80539 + ], + [ + -82.62258, + 69.65826 + ], + [ + -81.28043000000001, + 69.16202000000001 + ], + [ + -81.22019999999999, + 68.66567 + ], + [ + -81.96436000000001, + 68.13253 + ], + [ + -81.25928, + 67.59716 + ], + [ + -81.38653000000001, + 67.11078 + ], + [ + -83.34456, + 66.41154 + ], + [ + -84.73542, + 66.2573 + ], + [ + -85.76943, + 66.55833 + ], + [ + -86.06760000000001, + 66.05625 + ], + [ + -87.03143, + 65.21297 + ], + [ + -87.32324, + 64.77563 + ], + [ + -88.48296, + 64.09897000000001 + ], + [ + -89.91444, + 64.03273 + ], + [ + -90.70398, + 63.610170000000004 + ], + [ + -90.77004000000001, + 62.960210000000004 + ], + [ + -91.93342, + 62.83508 + ], + [ + -93.15698, + 62.02469000000001 + ], + [ + -94.24153, + 60.89865 + ], + [ + -94.62930999999999, + 60.11021 + ], + [ + -94.6846, + 58.94882 + ], + [ + -93.21502000000001, + 58.78212 + ], + [ + -92.76462000000001, + 57.84571 + ], + [ + -92.29702999999999, + 57.08709 + ], + [ + -90.89769, + 57.28468 + ], + [ + -89.03953, + 56.85172 + ], + [ + -88.03978000000001, + 56.47162 + ], + [ + -87.32421, + 55.999140000000004 + ], + [ + -86.07121, + 55.72383 + ], + [ + -85.01181000000001, + 55.302600000000005 + ], + [ + -83.36055, + 55.24489 + ], + [ + -82.27285, + 55.14832 + ], + [ + -82.43620000000001, + 54.282270000000004 + ], + [ + -82.12502, + 53.27703 + ], + [ + -81.40075, + 52.157880000000006 + ], + [ + -79.91289, + 51.208420000000004 + ], + [ + -79.14301, + 51.533930000000005 + ], + [ + -78.60191, + 52.56208 + ], + [ + -79.12421, + 54.14145 + ], + [ + -79.82958, + 54.66772 + ], + [ + -78.22874, + 55.136449999999996 + ], + [ + -77.0956, + 55.83741 + ], + [ + -76.54137, + 56.53423000000001 + ], + [ + -76.62319000000001, + 57.20263 + ], + [ + -77.30226, + 58.05209 + ], + [ + -78.51688, + 58.80458 + ], + [ + -77.33676, + 59.852610000000006 + ], + [ + -77.77272, + 60.75788000000001 + ], + [ + -78.10687, + 62.31964000000001 + ], + [ + -77.41067, + 62.55053 + ], + [ + -75.69621000000001, + 62.2784 + ], + [ + -74.6682, + 62.181110000000004 + ], + [ + -73.83988000000001, + 62.4438 + ], + [ + -72.90853, + 62.10507 + ], + [ + -71.67708, + 61.52535 + ], + [ + -71.37369000000001, + 61.137170000000005 + ], + [ + -69.59042, + 61.06141 + ], + [ + -69.62033, + 60.221250000000005 + ], + [ + -69.28790000000001, + 58.95736 + ], + [ + -68.37455, + 58.80106 + ], + [ + -67.64976, + 58.21206 + ], + [ + -66.20178, + 58.76731 + ], + [ + -65.24517, + 59.87071 + ], + [ + -64.58352000000001, + 60.33558 + ], + [ + -63.804750000000006, + 59.442600000000006 + ], + [ + -62.502359999999996, + 58.16708 + ], + [ + -61.396550000000005, + 56.96745000000001 + ], + [ + -61.798660000000005, + 56.33945 + ], + [ + -60.46853, + 55.775479999999995 + ], + [ + -59.56962, + 55.20407 + ], + [ + -57.97508, + 54.94549000000001 + ], + [ + -57.3332, + 54.6265 + ], + [ + -56.93689, + 53.780319999999996 + ], + [ + -56.15811, + 53.647490000000005 + ], + [ + -55.75632, + 53.27036 + ], + [ + -55.68338, + 52.146640000000005 + ], + [ + -56.40916000000001, + 51.770700000000005 + ], + [ + -57.12691, + 51.419720000000005 + ], + [ + -58.77482, + 51.0643 + ], + [ + -60.03309000000001, + 50.24277 + ], + [ + -61.72366, + 50.08046 + ], + [ + -63.86251, + 50.29099 + ], + [ + -65.36331, + 50.2982 + ], + [ + -66.39905, + 50.228970000000004 + ], + [ + -67.23631, + 49.511559999999996 + ], + [ + -68.51114, + 49.068360000000006 + ], + [ + -69.95362, + 47.74488 + ], + [ + -71.10458, + 46.82171 + ], + [ + -70.25522, + 46.986059999999995 + ], + [ + -68.65, + 48.3 + ], + [ + -66.55243, + 49.1331 + ], + [ + -65.05626, + 49.232780000000005 + ], + [ + -64.17099, + 48.74248 + ], + [ + -65.11545000000001, + 48.07085 + ], + [ + -64.79854, + 46.99297 + ], + [ + -64.47219, + 46.238490000000006 + ], + [ + -63.17329000000001, + 45.73902 + ], + [ + -61.520720000000004, + 45.883770000000005 + ], + [ + -60.518150000000006, + 47.00793 + ], + [ + -60.448600000000006, + 46.28264 + ], + [ + -59.80287, + 45.9204 + ], + [ + -61.03988, + 45.265249999999995 + ], + [ + -63.254709999999996, + 44.67014 + ], + [ + -64.24656, + 44.265530000000005 + ], + [ + -65.36406000000001, + 43.54523 + ], + [ + -66.1234, + 43.61867 + ], + [ + -66.16173, + 44.46512 + ], + [ + -64.42549, + 45.29204 + ], + [ + -66.02605000000001, + 45.25931 + ], + [ + -67.13741, + 45.13753 + ], + [ + -67.79134, + 45.70281000000001 + ], + [ + -67.79046000000001, + 47.066359999999996 + ], + [ + -68.23444, + 47.354859999999974 + ], + [ + -68.90500000000003, + 47.18500000000006 + ], + [ + -69.237216, + 47.447781 + ], + [ + -69.99997, + 46.69307 + ], + [ + -70.305, + 45.915 + ], + [ + -70.66, + 45.46 + ], + [ + -71.08482000000004, + 45.30524000000014 + ], + [ + -71.405, + 45.254999999999995 + ], + [ + -71.50506, + 45.0082 + ], + [ + -73.34783, + 45.00738 + ], + [ + -74.86700000000002, + 45.000480000000096 + ], + [ + -75.31821000000001, + 44.81645 + ], + [ + -76.375, + 44.09631 + ], + [ + -76.50000000000001, + 44.01845889375865 + ], + [ + -76.82003414580558, + 43.628784288093755 + ], + [ + -77.7378850979577, + 43.62905558936328 + ], + [ + -78.72027991404235, + 43.62508942318493 + ], + [ + -79.17167355011186, + 43.46633942318426 + ], + [ + -79.01, + 43.27 + ], + [ + -78.92, + 42.964999999999996 + ], + [ + -78.93936214874375, + 42.86361135514798 + ], + [ + -80.24744767934794, + 42.36619985612255 + ], + [ + -81.27774654816716, + 42.209025987306816 + ], + [ + -82.4392777167916, + 41.675105088867326 + ], + [ + -82.69008928092023, + 41.675105088867326 + ], + [ + -83.029810146807, + 41.83279572200598 + ], + [ + -83.14199968131264, + 41.975681057292874 + ], + [ + -83.12, + 42.08 + ], + [ + -82.9, + 42.43 + ], + [ + -82.42999999999999, + 42.980000000000004 + ], + [ + -82.13764238150395, + 43.57108755143997 + ], + [ + -82.33776312543114, + 44.440000000000055 + ], + [ + -82.55092464875821, + 45.34751658790543 + ], + [ + -83.59285071484311, + 45.81689362241252 + ], + [ + -83.46955074739469, + 45.994686387712534 + ], + [ + -83.61613094759059, + 46.116926988299014 + ], + [ + -83.89076534700574, + 46.116926988299014 + ], + [ + -84.0918512641615, + 46.27541860613826 + ], + [ + -84.1421195136734, + 46.51222585711571 + ], + [ + -84.33670000000001, + 46.408770000000004 + ], + [ + -84.60490000000004, + 46.439599999999984 + ], + [ + -84.54374874544584, + 46.538684190449146 + ], + [ + -84.77923824739992, + 46.63710195574902 + ], + [ + -84.8760798815149, + 46.90008331968238 + ], + [ + -85.65236324740341, + 47.22021881773051 + ], + [ + -86.46199083122826, + 47.553338019392 + ], + [ + -87.43979262330028, + 47.94 + ], + [ + -88.37811418328671, + 48.302917588893706 + ], + [ + -89.27291744663665, + 48.01980825458281 + ], + [ + -89.60000000000002, + 48.010000000000105 + ], + [ + -90.83, + 48.27 + ], + [ + -91.64, + 48.14 + ], + [ + -92.61000000000001, + 48.44999999999993 + ], + [ + -93.63087000000002, + 48.609260000000006 + ], + [ + -94.32914000000001, + 48.67074 + ], + [ + -94.64, + 48.84 + ], + [ + -94.81758000000002, + 49.38905 + ], + [ + -95.15609, + 49.38425000000001 + ], + [ + -95.15906950917206, + 49 + ], + [ + -97.2287200000048, + 49.0007 + ], + [ + -100.65000000000003, + 49.000000000000114 + ], + [ + -104.04826000000003, + 48.99986000000007 + ], + [ + -107.05000000000001, + 49 + ], + [ + -110.05000000000001, + 49 + ], + [ + -113, + 49 + ], + [ + -116.04818, + 49 + ], + [ + -117.03121, + 49 + ], + [ + -120, + 49.000000000000114 + ], + [ + -122.84000000000003, + 49.000000000000114 + ] + ] + ], + [ + [ + [ + -83.99367000000001, + 62.452799999999996 + ], + [ + -83.25048, + 62.91409 + ], + [ + -81.87699, + 62.90458 + ], + [ + -81.89825, + 62.7108 + ], + [ + -83.06857000000001, + 62.159220000000005 + ], + [ + -83.77462000000001, + 62.18231 + ], + [ + -83.99367000000001, + 62.452799999999996 + ] + ] + ], + [ + [ + [ + -79.77583312988281, + 72.8029022216797 + ], + [ + -80.87609863281251, + 73.33318328857422 + ], + [ + -80.83388519287111, + 73.69318389892578 + ], + [ + -80.35305786132812, + 73.75971984863281 + ], + [ + -78.06443786621094, + 73.65193176269531 + ], + [ + -76.34, + 73.10268498995305 + ], + [ + -76.25140380859375, + 72.82638549804688 + ], + [ + -77.31443786621094, + 72.85554504394531 + ], + [ + -78.39167022705078, + 72.87665557861328 + ], + [ + -79.4862518310547, + 72.74220275878906 + ], + [ + -79.77583312988281, + 72.8029022216797 + ] + ] + ], + [ + [ + [ + -80.315395, + 62.08556500000001 + ], + [ + -79.92939, + 62.3856 + ], + [ + -79.52002, + 62.363710000000005 + ], + [ + -79.26582, + 62.158674999999995 + ], + [ + -79.65752, + 61.63308 + ], + [ + -80.09956000000001, + 61.71810000000001 + ], + [ + -80.36215, + 62.016490000000005 + ], + [ + -80.315395, + 62.08556500000001 + ] + ] + ], + [ + [ + [ + -93.61275590694046, + 74.97999726022438 + ], + [ + -94.15690873897391, + 74.59234650338688 + ], + [ + -95.60868058956564, + 74.66686391875176 + ], + [ + -96.82093217648455, + 74.92762319609658 + ], + [ + -96.28858740922982, + 75.37782827422338 + ], + [ + -94.85081987178917, + 75.64721751576089 + ], + [ + -93.97774654821797, + 75.29648956979595 + ], + [ + -93.61275590694046, + 74.97999726022438 + ] + ] + ], + [ + [ + [ + -93.84000301794399, + 77.51999726023455 + ], + [ + -94.29560828324529, + 77.49134267852868 + ], + [ + -96.16965410031007, + 77.55511139597685 + ], + [ + -96.43630449093614, + 77.83462921824362 + ], + [ + -94.42257727738641, + 77.820004787905 + ], + [ + -93.7206562975659, + 77.63433136668031 + ], + [ + -93.84000301794399, + 77.51999726023455 + ] + ] + ], + [ + [ + [ + -96.75439876990876, + 78.76581268992702 + ], + [ + -95.5592779202946, + 78.41831452098033 + ], + [ + -95.83029496944934, + 78.05694122996324 + ], + [ + -97.30984290239799, + 77.85059723582181 + ], + [ + -98.12428931353404, + 78.08285696075761 + ], + [ + -98.55286780474668, + 78.45810537384507 + ], + [ + -98.63198442258553, + 78.87193024363837 + ], + [ + -97.33723141151266, + 78.83198436147676 + ], + [ + -96.75439876990876, + 78.76581268992702 + ] + ] + ], + [ + [ + [ + -88.15035030796028, + 74.39230703398503 + ], + [ + -89.7647220527584, + 74.51555532500116 + ], + [ + -92.42244096552946, + 74.83775788034099 + ], + [ + -92.76828548864282, + 75.38681997344214 + ], + [ + -92.88990597204175, + 75.88265534128267 + ], + [ + -93.89382402217599, + 76.31924367950056 + ], + [ + -95.9624574450358, + 76.4413809272224 + ], + [ + -97.1213789538295, + 76.7510777859476 + ], + [ + -96.74512285031237, + 77.16138865834507 + ], + [ + -94.68408586299944, + 77.09787832305837 + ], + [ + -93.57392106807313, + 76.77629588490605 + ], + [ + -91.6050231595366, + 76.7785179714946 + ], + [ + -90.7418458727493, + 76.44959747995681 + ], + [ + -90.96966142450802, + 76.07401317005947 + ], + [ + -89.82223792189926, + 75.84777374948565 + ], + [ + -89.18708289259985, + 75.61016551380762 + ], + [ + -87.83827633334965, + 75.56618886992725 + ], + [ + -86.37919226758864, + 75.4824213731821 + ], + [ + -84.78962521029058, + 75.69920400664653 + ], + [ + -82.75344458691006, + 75.78431509063124 + ], + [ + -81.12853084992436, + 75.71398346628199 + ], + [ + -80.05751095245915, + 75.33684886341591 + ], + [ + -79.83393286814837, + 74.92312734648716 + ], + [ + -80.45777075877587, + 74.65730377877777 + ], + [ + -81.94884253612557, + 74.44245901152432 + ], + [ + -83.22889360221143, + 74.56402781849094 + ], + [ + -86.09745235873332, + 74.41003205026117 + ], + [ + -88.15035030796028, + 74.39230703398503 + ] + ] + ], + [ + [ + [ + -111.26444332563088, + 78.15295604116154 + ], + [ + -109.85445187054711, + 77.99632477488488 + ], + [ + -110.18693803591302, + 77.69701487905034 + ], + [ + -112.0511911690585, + 77.4092288276169 + ], + [ + -113.53427893761912, + 77.73220652944111 + ], + [ + -112.7245867582539, + 78.05105011668196 + ], + [ + -111.26444332563088, + 78.15295604116154 + ] + ] + ], + [ + [ + [ + -110.96366065147602, + 78.8044408230652 + ], + [ + -109.6631457182026, + 78.60197256134565 + ], + [ + -110.88131425661892, + 78.40691986765997 + ], + [ + -112.54209143761516, + 78.4079017198735 + ], + [ + -112.52589087609164, + 78.55055451121522 + ], + [ + -111.5000103422334, + 78.8499935981305 + ], + [ + -110.96366065147602, + 78.8044408230652 + ] + ] + ], + [ + [ + [ + -55.600218268442056, + 51.31707469339794 + ], + [ + -56.13403581401709, + 50.68700979267928 + ], + [ + -56.795881720595276, + 49.81230866149089 + ], + [ + -56.14310502788433, + 50.15011749938286 + ], + [ + -55.471492275603, + 49.93581533466846 + ], + [ + -55.82240108908096, + 49.58712860777905 + ], + [ + -54.935142584845636, + 49.3130109726868 + ], + [ + -54.473775397343786, + 49.556691189159125 + ], + [ + -53.47654944519137, + 49.24913890237404 + ], + [ + -53.786013759971254, + 48.516780503933624 + ], + [ + -53.08613399922626, + 48.68780365660358 + ], + [ + -52.958648240762216, + 48.15716421161447 + ], + [ + -52.64809872090421, + 47.53554840757552 + ], + [ + -53.069158291218386, + 46.65549876564492 + ], + [ + -53.521456264853, + 46.61829173439477 + ], + [ + -54.17893551290251, + 46.80706574155698 + ], + [ + -53.9618686590605, + 47.62520701760193 + ], + [ + -54.24048214376214, + 47.752279364607645 + ], + [ + -55.40077307801157, + 46.884993801453135 + ], + [ + -55.99748084168583, + 46.919720363953275 + ], + [ + -55.29121904155279, + 47.38956248635099 + ], + [ + -56.250798712780586, + 47.632545070987376 + ], + [ + -57.32522925477708, + 47.57280711525797 + ], + [ + -59.26601518414682, + 47.60334788674247 + ], + [ + -59.419494188053676, + 47.899453843774886 + ], + [ + -58.79658647320744, + 48.25152537697942 + ], + [ + -59.23162451845657, + 48.52318838153781 + ], + [ + -58.3918049790652, + 49.12558055276418 + ], + [ + -57.35868974468606, + 50.71827403421587 + ], + [ + -56.738650071832026, + 51.28743825947855 + ], + [ + -55.87097693543532, + 51.63209422464921 + ], + [ + -55.40697424988659, + 51.5882726100657 + ], + [ + -55.600218268442056, + 51.31707469339794 + ] + ] + ], + [ + [ + [ + -83.88262630891977, + 65.10961782496354 + ], + [ + -82.78757687043883, + 64.76669302027467 + ], + [ + -81.6420137193926, + 64.45513580998697 + ], + [ + -81.55344031444432, + 63.97960928003714 + ], + [ + -80.81736121287886, + 64.057485663501 + ], + [ + -80.10345130076664, + 63.72598135034862 + ], + [ + -80.99101986359572, + 63.41124603947496 + ], + [ + -82.54717810741704, + 63.65172231714521 + ], + [ + -83.10879757356511, + 64.10187571883971 + ], + [ + -84.10041663281388, + 63.569711819098 + ], + [ + -85.52340471061905, + 63.052379055424055 + ], + [ + -85.8667687649824, + 63.63725291610349 + ], + [ + -87.22198320183678, + 63.54123810490519 + ], + [ + -86.35275977247133, + 64.0358332383707 + ], + [ + -86.2248864407651, + 64.82291697860823 + ], + [ + -85.88384782585486, + 65.7387783881171 + ], + [ + -85.1613079495499, + 65.6572846543928 + ], + [ + -84.97576371940592, + 65.21751821558898 + ], + [ + -84.4640120104195, + 65.37177236598022 + ], + [ + -83.88262630891977, + 65.10961782496354 + ] + ] + ], + [ + [ + [ + -78.77063859731078, + 72.35217316353418 + ], + [ + -77.8246239895596, + 72.74961660429098 + ], + [ + -75.60584469267573, + 72.2436784939374 + ], + [ + -74.228616095665, + 71.76714427355789 + ], + [ + -74.09914079455771, + 71.33084015571758 + ], + [ + -72.24222571479768, + 71.55692454699452 + ], + [ + -71.20001542833518, + 70.92001251899718 + ], + [ + -68.7860542466849, + 70.52502370877427 + ], + [ + -67.91497046575694, + 70.12194753689765 + ], + [ + -66.9690333726542, + 69.18608734809182 + ], + [ + -68.8051228502006, + 68.72019847276444 + ], + [ + -66.4498660956339, + 68.06716339789203 + ], + [ + -64.86231441919524, + 67.84753856065159 + ], + [ + -63.424934454996794, + 66.92847321234059 + ], + [ + -61.851981370680605, + 66.86212067327783 + ], + [ + -62.16317684594226, + 66.16025136988962 + ], + [ + -63.918444383384184, + 64.9986685248329 + ], + [ + -65.14886023625368, + 65.42603261988667 + ], + [ + -66.72121904159852, + 66.38804108343219 + ], + [ + -68.015016038674, + 66.26272573512439 + ], + [ + -68.1412874009792, + 65.68978913030439 + ], + [ + -67.08964616562342, + 65.10845510523696 + ], + [ + -65.73208045109976, + 64.64840566675856 + ], + [ + -65.32016760930125, + 64.38273712834605 + ], + [ + -64.66940629744968, + 63.392926744227495 + ], + [ + -65.01380388045888, + 62.67418508569598 + ], + [ + -66.27504472519048, + 62.94509878198612 + ], + [ + -68.7831862046927, + 63.74567007105183 + ], + [ + -67.36968075221309, + 62.88396556258484 + ], + [ + -66.32829728866726, + 62.28007477482201 + ], + [ + -66.16556820338015, + 61.93089712182582 + ], + [ + -68.87736650254465, + 62.330149237712824 + ], + [ + -71.02343705919385, + 62.91070811629588 + ], + [ + -72.23537858751902, + 63.39783600529522 + ], + [ + -71.88627844917127, + 63.67998932560887 + ], + [ + -73.37830624051838, + 64.19396312118384 + ], + [ + -74.83441891142263, + 64.6790756293238 + ], + [ + -74.81850257027673, + 64.38909332951793 + ], + [ + -77.70997982452008, + 64.22954234481678 + ], + [ + -78.5559488593542, + 64.57290639918013 + ], + [ + -77.89728105336198, + 65.30919220647475 + ], + [ + -76.01827429879717, + 65.32696889918314 + ], + [ + -73.95979529488268, + 65.45476471624094 + ], + [ + -74.29388342964964, + 65.81177134872938 + ], + [ + -73.94491248238262, + 66.31057811142666 + ], + [ + -72.65116716173942, + 67.28457550726391 + ], + [ + -72.92605994331605, + 67.72692576768235 + ], + [ + -73.31161780464572, + 68.06943716091287 + ], + [ + -74.84330725777684, + 68.55462718370127 + ], + [ + -76.86910091826672, + 68.89473562283025 + ], + [ + -76.22864905465738, + 69.14776927354741 + ], + [ + -77.28736996123715, + 69.76954010688321 + ], + [ + -78.1686339993266, + 69.82648753526887 + ], + [ + -78.95724219431673, + 70.16688019477543 + ], + [ + -79.49245500356366, + 69.87180776638884 + ], + [ + -81.30547095409176, + 69.74318512641436 + ], + [ + -84.94470618359851, + 69.96663401964442 + ], + [ + -87.06000342481789, + 70.26000112576538 + ], + [ + -88.68171322300148, + 70.4107412787608 + ], + [ + -89.51341956252303, + 70.76203766548095 + ], + [ + -88.46772111688082, + 71.21818553332132 + ], + [ + -89.88815121128755, + 71.22255219184997 + ], + [ + -90.20516028518205, + 72.23507436796079 + ], + [ + -89.436576707705, + 73.12946421985238 + ], + [ + -88.40824154331287, + 73.53788890247121 + ], + [ + -85.82615108920098, + 73.80381582304518 + ], + [ + -86.56217851433412, + 73.15744700793844 + ], + [ + -85.77437130404454, + 72.53412588163387 + ], + [ + -84.85011247428822, + 73.34027822538708 + ], + [ + -82.31559017610101, + 73.7509508328106 + ], + [ + -80.60008765330768, + 72.71654368762417 + ], + [ + -80.74894161652443, + 72.06190664335072 + ], + [ + -78.77063859731078, + 72.35217316353418 + ] + ] + ], + [ + [ + [ + -94.50365759965237, + 74.13490672473922 + ], + [ + -92.42001217321173, + 74.1000251329422 + ], + [ + -90.50979285354263, + 73.85673248971206 + ], + [ + -92.00396521682987, + 72.96624420845852 + ], + [ + -93.19629553910026, + 72.77199249947334 + ], + [ + -94.26904659704726, + 72.02459625923599 + ], + [ + -95.40985551632266, + 72.06188080513458 + ], + [ + -96.03374508338244, + 72.94027680123183 + ], + [ + -96.01826799191102, + 73.43742991809582 + ], + [ + -95.49579342322404, + 73.86241689726417 + ], + [ + -94.50365759965237, + 74.13490672473922 + ] + ] + ], + [ + [ + [ + -122.85492448615902, + 76.11654287383568 + ], + [ + -122.85492529360326, + 76.11654287383568 + ], + [ + -121.15753536032824, + 76.86450755482828 + ], + [ + -119.1039389718211, + 77.51221995717462 + ], + [ + -117.570130784966, + 77.4983189968881 + ], + [ + -116.19858659550738, + 77.6452867703262 + ], + [ + -116.33581336145845, + 76.87696157501061 + ], + [ + -117.10605058476882, + 76.53003184681911 + ], + [ + -118.04041215703819, + 76.48117178008714 + ], + [ + -119.89931758688572, + 76.053213406062 + ], + [ + -121.49999507712648, + 75.90001862253276 + ], + [ + -122.85492448615902, + 76.11654287383568 + ] + ] + ], + [ + [ + [ + -132.71000788443126, + 54.04000931542356 + ], + [ + -131.74998958400334, + 54.12000438090922 + ], + [ + -132.049480347351, + 52.98462148702447 + ], + [ + -131.1790425218266, + 52.180432847698285 + ], + [ + -131.57782954982298, + 52.18237071390928 + ], + [ + -132.18042842677852, + 52.639707139692405 + ], + [ + -132.54999243231384, + 53.100014960332146 + ], + [ + -133.05461117875552, + 53.411468817755406 + ], + [ + -133.2396644827927, + 53.851080227262344 + ], + [ + -133.1800040417117, + 54.169975490935315 + ], + [ + -132.71000788443126, + 54.04000931542356 + ] + ] + ], + [ + [ + [ + -105.4922891914932, + 79.30159393992916 + ], + [ + -103.52928239623795, + 79.16534902619163 + ], + [ + -100.8251580472688, + 78.80046173777872 + ], + [ + -100.0601918200522, + 78.32475434031589 + ], + [ + -99.67093909381364, + 77.90754466420744 + ], + [ + -101.30394019245301, + 78.01898489044486 + ], + [ + -102.94980872273302, + 78.34322866486023 + ], + [ + -105.17613277873151, + 78.3803323432458 + ], + [ + -104.21042945027713, + 78.67742015249176 + ], + [ + -105.41958045125853, + 78.91833567983649 + ], + [ + -105.4922891914932, + 79.30159393992916 + ] + ] + ], + [ + [ + [ + -123.51000158755119, + 48.51001089130341 + ], + [ + -124.01289078839955, + 48.37084625914139 + ], + [ + -125.65501277733838, + 48.8250045843385 + ], + [ + -125.95499446679275, + 49.17999583596759 + ], + [ + -126.85000443587185, + 49.53000031188043 + ], + [ + -127.02999344954443, + 49.81499583597008 + ], + [ + -128.0593363043662, + 49.9949590114266 + ], + [ + -128.44458410710214, + 50.539137681676095 + ], + [ + -128.35841365625546, + 50.77064809834371 + ], + [ + -127.30858109602994, + 50.552573554071955 + ], + [ + -126.69500097721235, + 50.400903225295394 + ], + [ + -125.7550066738232, + 50.29501821552935 + ], + [ + -125.4150015875588, + 49.95000051533259 + ], + [ + -124.92076818911934, + 49.475274970083376 + ], + [ + -123.92250870832106, + 49.06248362893581 + ], + [ + -123.51000158755119, + 48.51001089130341 + ] + ] + ], + [ + [ + [ + -121.53787999999997, + 74.44893000000002 + ], + [ + -120.10978, + 74.24135000000001 + ], + [ + -117.55563999999993, + 74.18576999999993 + ], + [ + -116.58442000000002, + 73.89607000000007 + ], + [ + -115.51080999999999, + 73.47519 + ], + [ + -116.76793999999995, + 73.22291999999999 + ], + [ + -119.22000000000003, + 72.51999999999998 + ], + [ + -120.45999999999998, + 71.82000000000005 + ], + [ + -120.45999999999998, + 71.38360179308756 + ], + [ + -123.09218999999996, + 70.90164000000004 + ], + [ + -123.62, + 71.34000000000009 + ], + [ + -125.92894873747338, + 71.86868846301138 + ], + [ + -125.49999999999994, + 72.29226081179502 + ], + [ + -124.80729000000002, + 73.02255999999994 + ], + [ + -123.93999999999994, + 73.68000000000012 + ], + [ + -124.91774999999996, + 74.29275000000013 + ], + [ + -121.53787999999997, + 74.44893000000002 + ] + ] + ], + [ + [ + [ + -107.81943000000001, + 75.84552000000001 + ], + [ + -106.92893000000001, + 76.01282 + ], + [ + -105.881, + 75.96940000000001 + ], + [ + -105.70498, + 75.47951 + ], + [ + -106.31347000000001, + 75.00527 + ], + [ + -109.70000000000002, + 74.85000000000001 + ], + [ + -112.22306999999999, + 74.41696 + ], + [ + -113.74381, + 74.39427 + ], + [ + -113.87135, + 74.72029 + ], + [ + -111.79420999999999, + 75.16250000000001 + ], + [ + -116.31221, + 75.04343 + ], + [ + -117.7104, + 75.2222 + ], + [ + -116.34602000000001, + 76.19903000000001 + ], + [ + -115.40487, + 76.47887 + ], + [ + -112.59056000000001, + 76.14134 + ], + [ + -110.81422, + 75.54919 + ], + [ + -109.06710000000001, + 75.47321000000001 + ], + [ + -110.49726000000001, + 76.42982 + ], + [ + -109.58109999999999, + 76.79417 + ], + [ + -108.54858999999999, + 76.67832000000001 + ], + [ + -108.21141, + 76.20168000000001 + ], + [ + -107.81943000000001, + 75.84552000000001 + ] + ] + ], + [ + [ + [ + -106.52258999999992, + 73.07601 + ], + [ + -105.40245999999996, + 72.67259000000007 + ], + [ + -104.77484000000004, + 71.6984000000001 + ], + [ + -104.4647599999999, + 70.99297000000007 + ], + [ + -102.78537, + 70.49776000000003 + ], + [ + -100.98077999999992, + 70.02431999999999 + ], + [ + -101.08928999999995, + 69.58447000000012 + ], + [ + -102.73115999999993, + 69.50402000000003 + ], + [ + -102.09329000000002, + 69.11962000000011 + ], + [ + -102.43024000000003, + 68.75281999999999 + ], + [ + -104.24000000000001, + 68.91000000000008 + ], + [ + -105.96000000000004, + 69.18000000000012 + ], + [ + -107.12254000000001, + 69.11922000000004 + ], + [ + -108.99999999999994, + 68.78000000000003 + ], + [ + -111.53414887520017, + 68.63005915681794 + ], + [ + -113.31320000000005, + 68.53553999999997 + ], + [ + -113.85495999999989, + 69.00744000000009 + ], + [ + -115.22000000000003, + 69.28000000000009 + ], + [ + -116.10793999999999, + 69.16821000000004 + ], + [ + -117.34000000000003, + 69.9600000000001 + ], + [ + -116.67472999999995, + 70.06655 + ], + [ + -115.13112000000001, + 70.23730000000006 + ], + [ + -113.72140999999999, + 70.1923700000001 + ], + [ + -112.41610000000003, + 70.36637999999999 + ], + [ + -114.35000000000002, + 70.60000000000002 + ], + [ + -116.48684000000003, + 70.52044999999998 + ], + [ + -117.90480000000002, + 70.54056000000014 + ], + [ + -118.43238000000002, + 70.90920000000006 + ], + [ + -116.11311, + 71.30917999999997 + ], + [ + -117.65567999999996, + 71.29520000000002 + ], + [ + -119.40199000000001, + 71.55858999999998 + ], + [ + -118.56266999999997, + 72.30785000000003 + ], + [ + -117.86641999999995, + 72.70594000000006 + ], + [ + -115.18909000000002, + 73.31459000000012 + ], + [ + -114.16716999999994, + 73.1214500000001 + ], + [ + -114.66633999999999, + 72.65277000000009 + ], + [ + -112.44101999999992, + 72.95540000000011 + ], + [ + -111.05039, + 72.45040000000006 + ], + [ + -109.92034999999993, + 72.96113000000008 + ], + [ + -109.00653999999997, + 72.63335000000001 + ], + [ + -108.18834999999996, + 71.65089 + ], + [ + -107.68599, + 72.0654800000001 + ], + [ + -108.39639, + 73.08953000000008 + ], + [ + -107.51645000000002, + 73.23597999999998 + ], + [ + -106.52258999999992, + 73.07601 + ] + ] + ], + [ + [ + [ + -100.43836, + 72.70588000000001 + ], + [ + -101.54, + 73.36 + ], + [ + -100.35642000000001, + 73.84389 + ], + [ + -99.16387, + 73.63339 + ], + [ + -97.38, + 73.76 + ], + [ + -97.12, + 73.47 + ], + [ + -98.05359, + 72.99052 + ], + [ + -96.54, + 72.56 + ], + [ + -96.72000000000001, + 71.66 + ], + [ + -98.35966, + 71.27284999999999 + ], + [ + -99.32286, + 71.35639 + ], + [ + -100.01482, + 71.73827 + ], + [ + -102.5, + 72.51 + ], + [ + -102.48000000000002, + 72.83000000000001 + ], + [ + -100.43836, + 72.70588000000001 + ] + ] + ], + [ + [ + [ + -106.6, + 73.60000000000001 + ], + [ + -105.26, + 73.64 + ], + [ + -104.5, + 73.42 + ], + [ + -105.38000000000001, + 72.76 + ], + [ + -106.94, + 73.46000000000001 + ], + [ + -106.6, + 73.60000000000001 + ] + ] + ], + [ + [ + [ + -98.50000000000001, + 76.72 + ], + [ + -97.735585, + 76.25656000000001 + ], + [ + -97.70441500000001, + 75.74344 + ], + [ + -98.16000000000001, + 75 + ], + [ + -99.80874, + 74.89744 + ], + [ + -100.88365999999999, + 75.05736 + ], + [ + -100.86292000000002, + 75.64075 + ], + [ + -102.50209, + 75.5638 + ], + [ + -102.56552, + 76.3366 + ], + [ + -101.48973, + 76.30537 + ], + [ + -99.98349, + 76.64634 + ], + [ + -98.57699, + 76.58859 + ], + [ + -98.50000000000001, + 76.72 + ] + ] + ], + [ + [ + [ + -96.01644, + 80.60233000000001 + ], + [ + -95.32345000000001, + 80.90729 + ], + [ + -94.29843, + 80.97727 + ], + [ + -94.73542, + 81.20646000000002 + ], + [ + -92.40983999999999, + 81.25739000000003 + ], + [ + -91.13288999999999, + 80.72345000000003 + ], + [ + -89.45000000000002, + 80.50932203389831 + ], + [ + -87.81, + 80.32000000000001 + ], + [ + -87.02000000000001, + 79.66000000000001 + ], + [ + -85.81435, + 79.3369 + ], + [ + -87.18755999999999, + 79.0393 + ], + [ + -89.03535000000001, + 78.28723 + ], + [ + -90.80436, + 78.21533000000001 + ], + [ + -92.87669000000001, + 78.34333000000001 + ], + [ + -93.95116000000002, + 78.75099 + ], + [ + -93.93574, + 79.11373 + ], + [ + -93.14524, + 79.3801 + ], + [ + -94.974, + 79.37248 + ], + [ + -96.07614000000001, + 79.70502 + ], + [ + -96.70972, + 80.15777 + ], + [ + -96.01644, + 80.60233000000001 + ] + ] + ], + [ + [ + [ + -91.58702000000001, + 81.89429000000001 + ], + [ + -90.10000000000001, + 82.08500000000004 + ], + [ + -88.93227, + 82.11751000000001 + ], + [ + -86.97024, + 82.27961 + ], + [ + -85.5, + 82.65227345805702 + ], + [ + -84.260005, + 82.60000000000001 + ], + [ + -83.18, + 82.32 + ], + [ + -82.42, + 82.86000000000001 + ], + [ + -81.1, + 83.02 + ], + [ + -79.30664, + 83.13056 + ], + [ + -76.25, + 83.17205882352941 + ], + [ + -75.71878000000001, + 83.06404000000002 + ], + [ + -72.83153, + 83.23324000000001 + ], + [ + -70.66576500000001, + 83.16978075838284 + ], + [ + -68.50000000000001, + 83.10632151676572 + ], + [ + -65.82735, + 83.02801000000001 + ], + [ + -63.68, + 82.9 + ], + [ + -61.85, + 82.62860000000002 + ], + [ + -61.89388, + 82.36165000000001 + ], + [ + -64.334, + 81.92775000000002 + ], + [ + -66.75342, + 81.72527000000001 + ], + [ + -67.65755, + 81.50141 + ], + [ + -65.48031, + 81.50657000000002 + ], + [ + -67.84, + 80.90000000000003 + ], + [ + -69.4697, + 80.61683000000001 + ], + [ + -71.18, + 79.8 + ], + [ + -73.2428, + 79.63415 + ], + [ + -73.88000000000001, + 79.43016220480206 + ], + [ + -76.90773, + 79.32309000000001 + ], + [ + -75.52924, + 79.19766000000001 + ], + [ + -76.22046, + 79.01907 + ], + [ + -75.39345, + 78.52581 + ], + [ + -76.34354, + 78.18296000000001 + ], + [ + -77.88851000000001, + 77.89991 + ], + [ + -78.36269, + 77.50859000000001 + ], + [ + -79.75951, + 77.20967999999999 + ], + [ + -79.61965000000001, + 76.98336 + ], + [ + -77.91089000000001, + 77.022045 + ], + [ + -77.88911, + 76.777955 + ], + [ + -80.56125, + 76.17812 + ], + [ + -83.17439, + 76.45403 + ], + [ + -86.11184, + 76.29901000000001 + ], + [ + -87.60000000000001, + 76.42 + ], + [ + -89.49068, + 76.47239 + ], + [ + -89.6161, + 76.95213000000001 + ], + [ + -87.76739, + 77.17833 + ], + [ + -88.26, + 77.9 + ], + [ + -87.65, + 77.97022222222223 + ], + [ + -84.97634, + 77.53873 + ], + [ + -86.34, + 78.18 + ], + [ + -87.96191999999999, + 78.37181 + ], + [ + -87.15198000000001, + 78.75867 + ], + [ + -85.37868, + 78.99690000000001 + ], + [ + -85.09495, + 79.34543000000001 + ], + [ + -86.50734, + 79.73624 + ], + [ + -86.93179, + 80.25145 + ], + [ + -84.19844, + 80.20836 + ], + [ + -83.40869565217389, + 80.10000000000001 + ], + [ + -81.84823, + 80.46442 + ], + [ + -84.1, + 80.58 + ], + [ + -87.59895, + 80.51627 + ], + [ + -89.36663, + 80.85569000000001 + ], + [ + -90.2, + 81.26 + ], + [ + -91.36786000000001, + 81.5531 + ], + [ + -91.58702000000001, + 81.89429000000001 + ] + ] + ], + [ + [ + [ + -75.21597, + 67.44425 + ], + [ + -75.86588, + 67.14886 + ], + [ + -76.98687, + 67.09873 + ], + [ + -77.2364, + 67.58809000000001 + ], + [ + -76.81166, + 68.14856 + ], + [ + -75.89521, + 68.28721 + ], + [ + -75.11449999999999, + 68.01035999999999 + ], + [ + -75.10333, + 67.58202 + ], + [ + -75.21597, + 67.44425 + ] + ] + ], + [ + [ + [ + -96.25740120380055, + 69.49003035832177 + ], + [ + -95.64768120380054, + 69.10769035832178 + ], + [ + -96.26952120380055, + 68.75704035832177 + ], + [ + -97.61740120380055, + 69.06003035832177 + ], + [ + -98.43180120380055, + 68.95070035832177 + ], + [ + -99.79740120380055, + 69.40003035832177 + ], + [ + -98.91740120380055, + 69.71003035832177 + ], + [ + -98.21826120380055, + 70.14354035832177 + ], + [ + -97.15740120380055, + 69.86003035832177 + ], + [ + -96.55740120380055, + 69.68003035832177 + ], + [ + -96.25740120380055, + 69.49003035832177 + ] + ] + ], + [ + [ + [ + -64.51912, + 49.87304 + ], + [ + -64.17322, + 49.95718 + ], + [ + -62.858290000000004, + 49.70641 + ], + [ + -61.835584999999995, + 49.28855 + ], + [ + -61.806304999999995, + 49.10506000000001 + ], + [ + -62.29318, + 49.08717 + ], + [ + -63.589259999999996, + 49.400690000000004 + ], + [ + -64.51912, + 49.87304 + ] + ] + ], + [ + [ + [ + -64.01486, + 47.03601 + ], + [ + -63.6645, + 46.55001 + ], + [ + -62.9393, + 46.41587 + ], + [ + -62.012080000000005, + 46.44314 + ], + [ + -62.503910000000005, + 46.033390000000004 + ], + [ + -62.87433, + 45.968180000000004 + ], + [ + -64.14280000000001, + 46.39265 + ], + [ + -64.39261, + 46.72747 + ], + [ + -64.01486, + 47.03601 + ] + ] + ] + ] + }, + "properties": { + "continent": "North America", + "gdp_md_est": 1674000, + "iso_a3": "CAN", + "name": "Canada", + "pop_est": 35623680 + }, + "type": "Feature", + "bbox": [ + -140.99778, + 41.675105088867326, + -52.64809872090421, + 83.23324000000001 + ] + }, + { + "geometry": { + "type": "MultiPolygon", + "coordinates": [ + [ + [ + [ + -122.84000000000003, + 49.000000000000114 + ], + [ + -120, + 49.000000000000114 + ], + [ + -117.03121, + 49 + ], + [ + -116.04818, + 49 + ], + [ + -113, + 49 + ], + [ + -110.05000000000001, + 49 + ], + [ + -107.05000000000001, + 49 + ], + [ + -104.04826000000003, + 48.99986000000007 + ], + [ + -100.65000000000003, + 49.000000000000114 + ], + [ + -97.2287200000048, + 49.0007 + ], + [ + -95.15906950917206, + 49 + ], + [ + -95.15609, + 49.38425000000001 + ], + [ + -94.81758000000002, + 49.38905 + ], + [ + -94.64, + 48.84 + ], + [ + -94.32914000000001, + 48.67074 + ], + [ + -93.63087000000002, + 48.609260000000006 + ], + [ + -92.61000000000001, + 48.44999999999993 + ], + [ + -91.64, + 48.14 + ], + [ + -90.83, + 48.27 + ], + [ + -89.60000000000002, + 48.010000000000105 + ], + [ + -89.27291744663665, + 48.01980825458281 + ], + [ + -88.37811418328671, + 48.302917588893706 + ], + [ + -87.43979262330028, + 47.94 + ], + [ + -86.46199083122826, + 47.553338019392 + ], + [ + -85.65236324740341, + 47.22021881773051 + ], + [ + -84.8760798815149, + 46.90008331968238 + ], + [ + -84.77923824739992, + 46.63710195574902 + ], + [ + -84.54374874544584, + 46.538684190449146 + ], + [ + -84.60490000000004, + 46.439599999999984 + ], + [ + -84.33670000000001, + 46.408770000000004 + ], + [ + -84.1421195136734, + 46.51222585711571 + ], + [ + -84.0918512641615, + 46.27541860613826 + ], + [ + -83.89076534700574, + 46.116926988299014 + ], + [ + -83.61613094759059, + 46.116926988299014 + ], + [ + -83.46955074739469, + 45.994686387712534 + ], + [ + -83.59285071484311, + 45.81689362241252 + ], + [ + -82.55092464875821, + 45.34751658790543 + ], + [ + -82.33776312543114, + 44.440000000000055 + ], + [ + -82.13764238150395, + 43.57108755143997 + ], + [ + -82.42999999999999, + 42.980000000000004 + ], + [ + -82.9, + 42.43 + ], + [ + -83.12, + 42.08 + ], + [ + -83.14199968131264, + 41.975681057292874 + ], + [ + -83.029810146807, + 41.83279572200598 + ], + [ + -82.69008928092023, + 41.675105088867326 + ], + [ + -82.4392777167916, + 41.675105088867326 + ], + [ + -81.27774654816716, + 42.209025987306816 + ], + [ + -80.24744767934794, + 42.36619985612255 + ], + [ + -78.93936214874375, + 42.86361135514798 + ], + [ + -78.92, + 42.964999999999996 + ], + [ + -79.01, + 43.27 + ], + [ + -79.17167355011186, + 43.46633942318426 + ], + [ + -78.72027991404235, + 43.62508942318493 + ], + [ + -77.7378850979577, + 43.62905558936328 + ], + [ + -76.82003414580558, + 43.628784288093755 + ], + [ + -76.50000000000001, + 44.01845889375865 + ], + [ + -76.375, + 44.09631 + ], + [ + -75.31821000000001, + 44.81645 + ], + [ + -74.86700000000002, + 45.000480000000096 + ], + [ + -73.34783, + 45.00738 + ], + [ + -71.50506, + 45.0082 + ], + [ + -71.405, + 45.254999999999995 + ], + [ + -71.08482000000004, + 45.30524000000014 + ], + [ + -70.66, + 45.46 + ], + [ + -70.305, + 45.915 + ], + [ + -69.99997, + 46.69307 + ], + [ + -69.237216, + 47.447781 + ], + [ + -68.90500000000003, + 47.18500000000006 + ], + [ + -68.23444, + 47.354859999999974 + ], + [ + -67.79046000000001, + 47.066359999999996 + ], + [ + -67.79134, + 45.70281000000001 + ], + [ + -67.13741, + 45.13753 + ], + [ + -66.96465999999998, + 44.809700000000134 + ], + [ + -68.03251999999998, + 44.325199999999995 + ], + [ + -69.05999999999995, + 43.980000000000075 + ], + [ + -70.11616999999995, + 43.68405000000013 + ], + [ + -70.64547563341102, + 43.09023834896402 + ], + [ + -70.81488999999999, + 42.865299999999934 + ], + [ + -70.82499999999999, + 42.33499999999998 + ], + [ + -70.49499999999995, + 41.80500000000001 + ], + [ + -70.07999999999998, + 41.78000000000003 + ], + [ + -70.185, + 42.145000000000095 + ], + [ + -69.88496999999995, + 41.92283000000009 + ], + [ + -69.96502999999996, + 41.63717000000014 + ], + [ + -70.63999999999999, + 41.47500000000002 + ], + [ + -71.12039000000004, + 41.49445000000014 + ], + [ + -71.8599999999999, + 41.32000000000005 + ], + [ + -72.29500000000002, + 41.26999999999998 + ], + [ + -72.87643000000003, + 41.220650000000035 + ], + [ + -73.71000000000004, + 40.93110235165449 + ], + [ + -72.24125999999995, + 41.119480000000124 + ], + [ + -71.94499999999988, + 40.930000000000064 + ], + [ + -73.34499999999997, + 40.63000000000005 + ], + [ + -73.98200000000003, + 40.62799999999993 + ], + [ + -73.95232499999997, + 40.75075000000004 + ], + [ + -74.25671, + 40.47351000000003 + ], + [ + -73.96243999999996, + 40.42763000000002 + ], + [ + -74.17838, + 39.70925999999997 + ], + [ + -74.90603999999996, + 38.93954000000002 + ], + [ + -74.98041, + 39.19640000000004 + ], + [ + -75.20002, + 39.248450000000105 + ], + [ + -75.52805000000001, + 39.49850000000009 + ], + [ + -75.32, + 38.960000000000036 + ], + [ + -75.07183476478986, + 38.782032230179254 + ], + [ + -75.05672999999996, + 38.40412000000009 + ], + [ + -75.37746999999996, + 38.015510000000006 + ], + [ + -75.94022999999999, + 37.21689000000009 + ], + [ + -76.03126999999995, + 37.25659999999999 + ], + [ + -75.72204999999985, + 37.93705000000011 + ], + [ + -76.23286999999999, + 38.319214999999986 + ], + [ + -76.35000000000002, + 39.14999999999998 + ], + [ + -76.54272499999996, + 38.71761500000008 + ], + [ + -76.32933000000003, + 38.08326000000005 + ], + [ + -76.98999793161352, + 38.23999176691336 + ], + [ + -76.30161999999996, + 37.91794499999992 + ], + [ + -76.25873999999999, + 36.96640000000008 + ], + [ + -75.97179999999997, + 36.89726000000002 + ], + [ + -75.8680399999999, + 36.55125000000004 + ], + [ + -75.72748999999999, + 35.55074000000013 + ], + [ + -76.36318, + 34.80854000000011 + ], + [ + -77.39763499999992, + 34.512009999999975 + ], + [ + -78.05496, + 33.92547000000002 + ], + [ + -78.55434999999989, + 33.86133000000012 + ], + [ + -79.06067000000002, + 33.493949999999984 + ], + [ + -79.20357000000001, + 33.158390000000054 + ], + [ + -80.30132499999996, + 32.509355000000085 + ], + [ + -80.86498, + 32.033300000000054 + ], + [ + -81.33629000000002, + 31.44049000000001 + ], + [ + -81.49041999999997, + 30.7299900000001 + ], + [ + -81.31371000000001, + 30.035520000000076 + ], + [ + -80.97999999999996, + 29.18000000000012 + ], + [ + -80.53558499999991, + 28.472129999999993 + ], + [ + -80.52999999999986, + 28.040000000000077 + ], + [ + -80.05653928497759, + 26.88000000000011 + ], + [ + -80.08801499999998, + 26.205764999999985 + ], + [ + -80.13155999999992, + 25.816775000000064 + ], + [ + -80.38103000000001, + 25.20616000000001 + ], + [ + -80.67999999999995, + 25.08000000000004 + ], + [ + -81.17212999999998, + 25.201260000000104 + ], + [ + -81.33000000000004, + 25.639999999999986 + ], + [ + -81.70999999999987, + 25.870000000000005 + ], + [ + -82.23999999999995, + 26.730000000000132 + ], + [ + -82.70515, + 27.495040000000074 + ], + [ + -82.85525999999999, + 27.886240000000043 + ], + [ + -82.64999999999998, + 28.550000000000125 + ], + [ + -82.92999999999995, + 29.10000000000008 + ], + [ + -83.70958999999999, + 29.936560000000043 + ], + [ + -84.09999999999997, + 30.09000000000009 + ], + [ + -85.10881999999998, + 29.636150000000043 + ], + [ + -85.28784000000002, + 29.68612000000013 + ], + [ + -85.7731, + 30.152610000000095 + ], + [ + -86.39999999999992, + 30.40000000000009 + ], + [ + -87.53035999999992, + 30.27433000000002 + ], + [ + -88.41781999999995, + 30.384900000000016 + ], + [ + -89.1804899999999, + 30.315980000000025 + ], + [ + -89.5938311784198, + 30.159994004836847 + ], + [ + -89.41373499999997, + 29.89418999999998 + ], + [ + -89.43, + 29.488639999999975 + ], + [ + -89.21767, + 29.291080000000022 + ], + [ + -89.40822999999995, + 29.159610000000043 + ], + [ + -89.77927999999997, + 29.307140000000118 + ], + [ + -90.15463, + 29.11743000000007 + ], + [ + -90.88022499999994, + 29.148535000000095 + ], + [ + -91.62678499999993, + 29.677000000000135 + ], + [ + -92.49905999999999, + 29.552300000000002 + ], + [ + -93.22636999999997, + 29.783750000000055 + ], + [ + -93.84841999999998, + 29.71363000000008 + ], + [ + -94.69, + 29.480000000000132 + ], + [ + -95.60025999999999, + 28.738630000000057 + ], + [ + -96.59403999999995, + 28.307480000000055 + ], + [ + -97.13999999999987, + 27.83000000000004 + ], + [ + -97.36999999999995, + 27.380000000000052 + ], + [ + -97.37999999999994, + 26.690000000000055 + ], + [ + -97.32999999999998, + 26.210000000000093 + ], + [ + -97.13999999999987, + 25.870000000000005 + ], + [ + -97.52999999999992, + 25.84000000000009 + ], + [ + -98.23999999999995, + 26.06000000000006 + ], + [ + -99.01999999999992, + 26.37000000000006 + ], + [ + -99.30000000000001, + 26.840000000000032 + ], + [ + -99.51999999999992, + 27.54000000000002 + ], + [ + -100.10999999999996, + 28.110000000000127 + ], + [ + -100.45584000000002, + 28.69612000000012 + ], + [ + -100.95759999999996, + 29.380710000000136 + ], + [ + -101.66239999999999, + 29.77930000000009 + ], + [ + -102.48000000000002, + 29.75999999999999 + ], + [ + -103.11000000000001, + 28.970000000000027 + ], + [ + -103.94, + 29.27000000000004 + ], + [ + -104.4569699999999, + 29.571960000000047 + ], + [ + -104.70574999999997, + 30.121730000000014 + ], + [ + -105.03737000000001, + 30.644019999999955 + ], + [ + -105.63159000000002, + 31.08383000000009 + ], + [ + -106.1429, + 31.399950000000047 + ], + [ + -106.50758999999988, + 31.754520000000014 + ], + [ + -108.24000000000001, + 31.754853718166373 + ], + [ + -108.24193999999994, + 31.342220000000054 + ], + [ + -109.03500000000003, + 31.341940000000136 + ], + [ + -111.02361000000002, + 31.334719999999948 + ], + [ + -113.30498, + 32.03914000000009 + ], + [ + -114.815, + 32.52528000000001 + ], + [ + -114.72138999999993, + 32.72082999999992 + ], + [ + -115.99134999999995, + 32.61239000000012 + ], + [ + -117.12775999999985, + 32.53533999999996 + ], + [ + -117.29593769127393, + 33.04622461520387 + ], + [ + -117.94400000000002, + 33.621236431201396 + ], + [ + -118.41060227589753, + 33.74090922312445 + ], + [ + -118.51989482279976, + 34.02778157757575 + ], + [ + -119.08100000000002, + 34.07799999999992 + ], + [ + -119.43884064201671, + 34.34847717828427 + ], + [ + -120.36777999999998, + 34.447110000000066 + ], + [ + -120.62286, + 34.60854999999998 + ], + [ + -120.74432999999999, + 35.15686000000011 + ], + [ + -121.71456999999992, + 36.161529999999914 + ], + [ + -122.54746999999998, + 37.551760000000115 + ], + [ + -122.51201000000003, + 37.78339000000011 + ], + [ + -122.95319, + 38.11371000000008 + ], + [ + -123.72720000000004, + 38.95166000000012 + ], + [ + -123.86516999999998, + 39.76699000000008 + ], + [ + -124.39807000000002, + 40.313199999999995 + ], + [ + -124.17885999999999, + 41.142020000000116 + ], + [ + -124.21370000000002, + 41.99964000000011 + ], + [ + -124.53283999999996, + 42.7659900000001 + ], + [ + -124.14213999999998, + 43.708380000000034 + ], + [ + -124.020535, + 44.615894999999966 + ], + [ + -123.89892999999995, + 45.52341000000007 + ], + [ + -124.079635, + 46.864750000000015 + ], + [ + -124.39566999999994, + 47.72017000000011 + ], + [ + -124.68721008300781, + 48.18443298339855 + ], + [ + -124.56610107421875, + 48.37971496582037 + ], + [ + -123.12, + 48.04000000000002 + ], + [ + -122.58735999999993, + 47.09600000000006 + ], + [ + -122.34000000000003, + 47.360000000000014 + ], + [ + -122.5, + 48.180000000000064 + ], + [ + -122.84000000000003, + 49.000000000000114 + ] + ] + ], + [ + [ + [ + -155.40214, + 20.07975 + ], + [ + -155.22452, + 19.99302 + ], + [ + -155.06226, + 19.8591 + ], + [ + -154.80741, + 19.50871 + ], + [ + -154.83147, + 19.453280000000003 + ], + [ + -155.22217, + 19.23972 + ], + [ + -155.54211, + 19.08348 + ], + [ + -155.68817, + 18.91619 + ], + [ + -155.93665, + 19.05939 + ], + [ + -155.90806, + 19.33888 + ], + [ + -156.07347000000001, + 19.70294 + ], + [ + -156.02368, + 19.81422 + ], + [ + -155.85008000000002, + 19.97729 + ], + [ + -155.91907, + 20.17395 + ], + [ + -155.86108000000002, + 20.267210000000002 + ], + [ + -155.78505, + 20.2487 + ], + [ + -155.40214, + 20.07975 + ] + ] + ], + [ + [ + [ + -155.99566000000002, + 20.76404 + ], + [ + -156.07926, + 20.643970000000003 + ], + [ + -156.41445, + 20.57241 + ], + [ + -156.58673, + 20.783 + ], + [ + -156.70167, + 20.8643 + ], + [ + -156.71054999999998, + 20.92676 + ], + [ + -156.61258, + 21.01249 + ], + [ + -156.25711, + 20.917450000000002 + ], + [ + -155.99566000000002, + 20.76404 + ] + ] + ], + [ + [ + [ + -156.75824, + 21.176840000000002 + ], + [ + -156.78933, + 21.068730000000002 + ], + [ + -157.32521, + 21.097770000000004 + ], + [ + -157.25027, + 21.219579999999997 + ], + [ + -156.75824, + 21.176840000000002 + ] + ] + ], + [ + [ + [ + -158.0252, + 21.71696 + ], + [ + -157.94161, + 21.65272 + ], + [ + -157.65283000000002, + 21.322170000000003 + ], + [ + -157.70703, + 21.26442 + ], + [ + -157.7786, + 21.27729 + ], + [ + -158.12667000000002, + 21.31244 + ], + [ + -158.2538, + 21.53919 + ], + [ + -158.29265, + 21.57912 + ], + [ + -158.0252, + 21.71696 + ] + ] + ], + [ + [ + [ + -159.36569, + 22.21494 + ], + [ + -159.34512, + 21.982000000000003 + ], + [ + -159.46372, + 21.88299 + ], + [ + -159.80051, + 22.065330000000003 + ], + [ + -159.74877, + 22.1382 + ], + [ + -159.5962, + 22.236179999999997 + ], + [ + -159.36569, + 22.21494 + ] + ] + ], + [ + [ + [ + -166.46779212142462, + 60.384169826897754 + ], + [ + -165.67442969466364, + 60.29360687930625 + ], + [ + -165.57916419173358, + 59.90998688418753 + ], + [ + -166.19277014876727, + 59.75444082298899 + ], + [ + -166.84833736882197, + 59.941406155020985 + ], + [ + -167.45527706609008, + 60.21306915957936 + ], + [ + -166.46779212142462, + 60.384169826897754 + ] + ] + ], + [ + [ + [ + -153.22872941792113, + 57.96896841087248 + ], + [ + -152.56479061583514, + 57.901427313866996 + ], + [ + -152.1411472239064, + 57.591058661522 + ], + [ + -153.00631405333692, + 57.11584219016593 + ], + [ + -154.0050902984581, + 56.734676825581076 + ], + [ + -154.51640275777004, + 56.99274892844669 + ], + [ + -154.67099280497118, + 57.46119578717253 + ], + [ + -153.7627795074415, + 57.81657461204373 + ], + [ + -153.22872941792113, + 57.96896841087248 + ] + ] + ], + [ + [ + [ + -140.98598761037601, + 69.71199839952635 + ], + [ + -140.986, + 69.712 + ], + [ + -140.9925, + 66.00003000000001 + ], + [ + -140.99778, + 60.30639000000001 + ], + [ + -140.013, + 60.27682000000001 + ], + [ + -139.03900000000002, + 60 + ], + [ + -138.34089, + 59.562110000000004 + ], + [ + -137.4525, + 58.905 + ], + [ + -136.47972000000004, + 59.46389000000005 + ], + [ + -135.47583, + 59.787780000000005 + ], + [ + -134.94500000000005, + 59.2705600000001 + ], + [ + -134.27111000000002, + 58.86111000000005 + ], + [ + -133.35556000000003, + 58.41028000000001 + ], + [ + -132.73042, + 57.692890000000006 + ], + [ + -131.70781, + 56.55212 + ], + [ + -130.00778000000003, + 55.915830000000085 + ], + [ + -129.98, + 55.285000000000004 + ], + [ + -130.53611, + 54.802780000000006 + ], + [ + -130.53610895273684, + 54.80275447679924 + ], + [ + -130.5361101894673, + 54.8027534043494 + ], + [ + -131.08581823797215, + 55.17890615500204 + ], + [ + -131.9672114671423, + 55.497775580459006 + ], + [ + -132.2500107428595, + 56.3699962428974 + ], + [ + -133.53918108435641, + 57.17888743756214 + ], + [ + -134.07806292029608, + 58.12306753196691 + ], + [ + -135.0382110322791, + 58.18771474876394 + ], + [ + -136.62806230995471, + 58.21220937767043 + ], + [ + -137.800006279686, + 58.49999542910376 + ], + [ + -139.867787041413, + 59.53776154238915 + ], + [ + -140.825273817133, + 59.727517401765056 + ], + [ + -142.57444353556446, + 60.08444651960497 + ], + [ + -143.9588809948799, + 59.999180406323376 + ], + [ + -145.92555681682788, + 60.45860972761426 + ], + [ + -147.11437394914665, + 60.884656073644635 + ], + [ + -148.22430620012761, + 60.67298940697714 + ], + [ + -148.01806555885082, + 59.97832896589364 + ], + [ + -148.57082251686086, + 59.914172675203304 + ], + [ + -149.72785783587585, + 59.70565827090553 + ], + [ + -150.60824337461642, + 59.368211168039466 + ], + [ + -151.7163927886833, + 59.15582103131993 + ], + [ + -151.85943315326722, + 59.744984035879554 + ], + [ + -151.40971900124717, + 60.72580272077937 + ], + [ + -150.3469414947325, + 61.03358755150987 + ], + [ + -150.62111080625704, + 61.2844249538544 + ], + [ + -151.89583919981683, + 60.727197984451266 + ], + [ + -152.57832984109558, + 60.061657212964235 + ], + [ + -154.01917212625764, + 59.35027944603428 + ], + [ + -153.28751135965317, + 58.86472768821977 + ], + [ + -154.23249243875847, + 58.14637360293051 + ], + [ + -155.3074914215102, + 57.727794501366304 + ], + [ + -156.30833472392305, + 57.422774359763594 + ], + [ + -156.55609737854638, + 56.97998484967064 + ], + [ + -158.11721655986779, + 56.46360809999419 + ], + [ + -158.43332129619714, + 55.99415355083852 + ], + [ + -159.60332739971741, + 55.56668610292013 + ], + [ + -160.28971961163427, + 55.643580634170576 + ], + [ + -161.22304765525777, + 55.364734605523495 + ], + [ + -162.23776607974105, + 55.02418691672011 + ], + [ + -163.06944658104638, + 54.68973704692712 + ], + [ + -164.78556922102717, + 54.40417308208214 + ], + [ + -164.94222632552007, + 54.57222483989534 + ], + [ + -163.84833960676565, + 55.03943146424609 + ], + [ + -162.87000139061595, + 55.34804311789321 + ], + [ + -161.80417497459607, + 55.89498647727038 + ], + [ + -160.5636047027812, + 56.00805451112501 + ], + [ + -160.07055986228448, + 56.41805532492873 + ], + [ + -158.6844429189195, + 57.01667511659787 + ], + [ + -158.46109737855403, + 57.21692129172885 + ], + [ + -157.72277035218391, + 57.57000051536306 + ], + [ + -157.55027442119362, + 58.328326321030204 + ], + [ + -157.04167497457698, + 58.91888458926172 + ], + [ + -158.19473120830554, + 58.61580231386978 + ], + [ + -158.51721798402303, + 58.78778148053732 + ], + [ + -159.0586061269288, + 58.42418610293163 + ], + [ + -159.71166704001737, + 58.93139028587632 + ], + [ + -159.98128882550017, + 58.572549140041644 + ], + [ + -160.3552711659965, + 59.07112335879361 + ], + [ + -161.3550034251151, + 58.670837714260756 + ], + [ + -161.96889360252632, + 58.67166453717738 + ], + [ + -162.05498653872465, + 59.26692536074745 + ], + [ + -161.8741707021354, + 59.63362132429057 + ], + [ + -162.51805904849212, + 59.98972361921386 + ], + [ + -163.8183414378202, + 59.79805573184336 + ], + [ + -164.66221757714652, + 60.26748444278263 + ], + [ + -165.3463877024748, + 60.50749563256238 + ], + [ + -165.3508318756519, + 61.073895168697504 + ], + [ + -166.12137915755602, + 61.50001902937623 + ], + [ + -165.73445187077058, + 62.074996853271784 + ], + [ + -164.9191786367179, + 62.63307648380794 + ], + [ + -164.56250790103934, + 63.14637848576302 + ], + [ + -163.75333248599708, + 63.21944896102377 + ], + [ + -163.06722449445786, + 63.05945872664802 + ], + [ + -162.26055538638175, + 63.54193573674115 + ], + [ + -161.53444983624863, + 63.455816962326764 + ], + [ + -160.7725066803211, + 63.766108100023246 + ], + [ + -160.9583351308426, + 64.22279857040274 + ], + [ + -161.51806840721218, + 64.40278758407527 + ], + [ + -160.77777767641481, + 64.78860382756642 + ], + [ + -161.39192623598765, + 64.77723501246231 + ], + [ + -162.4530500966689, + 64.55944468856819 + ], + [ + -162.75778601789415, + 64.33860545516876 + ], + [ + -163.54639421288428, + 64.5591604681905 + ], + [ + -164.96082984114514, + 64.44694509546883 + ], + [ + -166.42528825586447, + 64.68667206487066 + ], + [ + -166.8450042389391, + 65.08889557561452 + ], + [ + -168.11056006576715, + 65.66999705673675 + ], + [ + -166.70527116602193, + 66.08831777613938 + ], + [ + -164.47470964257548, + 66.5766600612975 + ], + [ + -163.65251176659564, + 66.5766600612975 + ], + [ + -163.78860165103623, + 66.07720734319668 + ], + [ + -161.67777442121013, + 66.11611969671242 + ], + [ + -162.48971452538004, + 66.73556509059512 + ], + [ + -163.71971696679117, + 67.11639455837008 + ], + [ + -164.4309913808565, + 67.61633820257777 + ], + [ + -165.39028683170673, + 68.04277212185025 + ], + [ + -166.76444068099605, + 68.35887685817966 + ], + [ + -166.20470740462667, + 68.88303091091615 + ], + [ + -164.43081051334346, + 68.91553538682774 + ], + [ + -163.1686136546145, + 69.37111481391287 + ], + [ + -162.930566169262, + 69.85806183539927 + ], + [ + -161.90889726463556, + 70.33332998318764 + ], + [ + -160.93479651593367, + 70.44768992784958 + ], + [ + -159.03917578838713, + 70.89164215766891 + ], + [ + -158.11972286683394, + 70.82472117785102 + ], + [ + -156.58082455139808, + 71.35776357694175 + ], + [ + -155.06779029032427, + 71.14777639432367 + ], + [ + -154.3441652089412, + 70.69640859647018 + ], + [ + -153.9000062733926, + 70.88998851183567 + ], + [ + -152.21000606993528, + 70.82999217394485 + ], + [ + -152.27000240782613, + 70.60000621202983 + ], + [ + -150.73999243874448, + 70.43001658800569 + ], + [ + -149.7200030181675, + 70.53001048449045 + ], + [ + -147.61336157935705, + 70.2140349392418 + ], + [ + -145.68998980022533, + 70.12000967068673 + ], + [ + -144.9200109590764, + 69.98999176704046 + ], + [ + -143.58944618042523, + 70.15251414659832 + ], + [ + -142.07251034871348, + 69.85193817817265 + ], + [ + -140.98598752156073, + 69.71199839952635 + ], + [ + -140.98598761037601, + 69.71199839952635 + ] + ] + ], + [ + [ + [ + -171.73165686753944, + 63.782515367275934 + ], + [ + -171.1144335602453, + 63.59219106714495 + ], + [ + -170.4911124339407, + 63.694975490973505 + ], + [ + -169.6825054596536, + 63.43111562769119 + ], + [ + -168.6894394603007, + 63.297506212000556 + ], + [ + -168.77194088445466, + 63.18859813094544 + ], + [ + -169.5294398672051, + 62.97693146427792 + ], + [ + -170.29055620021595, + 63.194437567794424 + ], + [ + -170.67138566799093, + 63.3758218451389 + ], + [ + -171.55306311753873, + 63.317789211675105 + ], + [ + -171.79111060289122, + 63.40584585230046 + ], + [ + -171.73165686753944, + 63.782515367275934 + ] + ] + ] + ] + }, + "properties": { + "continent": "North America", + "gdp_md_est": 18560000, + "iso_a3": "USA", + "name": "United States of America", + "pop_est": 326625791 + }, + "type": "Feature", + "bbox": [ + -171.79111060289122, + 18.91619, + -66.96465999999998, + 71.35776357694175 + ] + } + ] +} \ No newline at end of file diff --git a/internal/geoparquet/featurewriter.go b/internal/geoparquet/featurewriter.go index 5f9325a..04104d8 100644 --- a/internal/geoparquet/featurewriter.go +++ b/internal/geoparquet/featurewriter.go @@ -39,7 +39,7 @@ func NewFeatureWriter(config *WriterConfig) (*FeatureWriter, error) { geoMetadata := config.Metadata if geoMetadata == nil { - geoMetadata = DefaultMetadata() + geoMetadata = DefaultMetadata(config.WriteCoveringMetadata) } if config.ArrowSchema == nil { @@ -100,6 +100,23 @@ func (w *FeatureWriter) append(feature *geo.Feature, field arrow.Field, builder return w.appendGeometry(feature, field, builder) } + if name == DefaultBboxColumn || name == GetBboxColumnNameFromMetadata(w.geoMetadata) { + if feature.Bbox == nil { + if !field.Nullable { + return fmt.Errorf("field %q is required, but the property is missing in the feature", name) + } + builder.AppendNull() + return nil + } + bboxMap := map[string]any{ + "xmin": feature.Bbox.Min.X(), + "ymin": feature.Bbox.Min.Y(), + "xmax": feature.Bbox.Max.X(), + "ymax": feature.Bbox.Max.Y(), + } + return w.appendValue(name, bboxMap, builder) + } + value, ok := feature.Properties[name] if !ok || value == nil { if !field.Nullable { diff --git a/internal/geoparquet/geoparquet.go b/internal/geoparquet/geoparquet.go index a8ba7ec..efdc79b 100644 --- a/internal/geoparquet/geoparquet.go +++ b/internal/geoparquet/geoparquet.go @@ -197,7 +197,7 @@ type BboxColumnFieldNames struct { Ymax string } -func getBboxColumnFieldNames(metadata *Metadata) *BboxColumnFieldNames { +func GetBboxColumnFieldNames(metadata *Metadata) *BboxColumnFieldNames { // infer bbox struct field names fieldNames := &BboxColumnFieldNames{} @@ -225,6 +225,16 @@ type BboxColumn struct { BboxColumnFieldNames } +// Get Bbox column name from covering metadata only. For most cases, use the +// more robust function GetBboxColumn that infers the name even if metadata +// is not present. +func GetBboxColumnNameFromMetadata(geoMetadata *Metadata) string { + if geoMetadata.Columns[geoMetadata.PrimaryColumn].Covering != nil && len(geoMetadata.Columns[geoMetadata.PrimaryColumn].Covering.Bbox.Xmin) == 2 { + return geoMetadata.Columns[geoMetadata.PrimaryColumn].Covering.Bbox.Xmin[0] + } + return "" +} + // Returns a *BboxColumn struct that contains index, name and other data // that describe the bounding box column contained in the schema. // If there is no match for the standard name "bbox" in the schema, @@ -233,8 +243,8 @@ type BboxColumn struct { func GetBboxColumn(schema *schema.Schema, geoMetadata *Metadata) *BboxColumn { bboxCol := &BboxColumn{} // try standard name first - bboxCol.Name = "bbox" - bboxCol.Index = schema.Root().FieldIndexByName("bbox") + bboxCol.Name = DefaultBboxColumn + bboxCol.Index = schema.Root().FieldIndexByName(DefaultBboxColumn) // if no match, check covering metadata if bboxCol.Index == -1 { @@ -247,6 +257,6 @@ func GetBboxColumn(schema *schema.Schema, geoMetadata *Metadata) *BboxColumn { } bboxCol.BaseColumn = schema.ColumnIndexByName(geoMetadata.PrimaryColumn) - bboxCol.BboxColumnFieldNames = *getBboxColumnFieldNames(geoMetadata) + bboxCol.BboxColumnFieldNames = *GetBboxColumnFieldNames(geoMetadata) return bboxCol } diff --git a/internal/geoparquet/geoparquet_test.go b/internal/geoparquet/geoparquet_test.go index 5f3726e..a643ed7 100644 --- a/internal/geoparquet/geoparquet_test.go +++ b/internal/geoparquet/geoparquet_test.go @@ -277,7 +277,7 @@ func TestFromParquetWithoutDefaultGeometryColumn(t *testing.T) { } func TestMetadataClone(t *testing.T) { - metadata := geoparquet.DefaultMetadata() + metadata := geoparquet.DefaultMetadata(false) clone := metadata.Clone() assert.Equal(t, metadata.PrimaryColumn, clone.PrimaryColumn) diff --git a/internal/geoparquet/metadata.go b/internal/geoparquet/metadata.go index 9fd5e26..7b9e38b 100644 --- a/internal/geoparquet/metadata.go +++ b/internal/geoparquet/metadata.go @@ -10,12 +10,13 @@ import ( ) const ( - Version = "1.0.0" + Version = "1.1.0" MetadataKey = "geo" EdgesPlanar = "planar" EdgesSpherical = "spherical" OrientationCounterClockwise = "counterclockwise" DefaultGeometryColumn = "geometry" + DefaultBboxColumn = "bbox" DefaultGeometryEncoding = geo.EncodingWKB ) @@ -152,14 +153,23 @@ func getDefaultGeometryColumn() *GeometryColumn { } } -func DefaultMetadata() *Metadata { - return &Metadata{ +func DefaultMetadata(writeCovering bool) *Metadata { + metadata := &Metadata{ Version: Version, PrimaryColumn: DefaultGeometryColumn, Columns: map[string]*GeometryColumn{ DefaultGeometryColumn: getDefaultGeometryColumn(), }, } + if writeCovering { + metadata.Columns[metadata.PrimaryColumn].Covering = &Covering{Bbox: coveringBbox{ + Xmin: []string{"bbox", "xmin"}, + Ymin: []string{"bbox", "ymin"}, + Xmax: []string{"bbox", "xmax"}, + Ymax: []string{"bbox", "ymax"}, + }} + } + return metadata } var ErrNoMetadata = fmt.Errorf("missing %s metadata key", MetadataKey) diff --git a/internal/geoparquet/recordwriter.go b/internal/geoparquet/recordwriter.go index e4f3ab2..d4ceed7 100644 --- a/internal/geoparquet/recordwriter.go +++ b/internal/geoparquet/recordwriter.go @@ -11,9 +11,10 @@ import ( ) type RecordWriter struct { - fileWriter *pqarrow.FileWriter - metadata *Metadata - wroteGeoMetadata bool + fileWriter *pqarrow.FileWriter + metadata *Metadata + wroteGeoMetadata bool + writeCoveringMetadata bool } func NewRecordWriter(config *WriterConfig) (*RecordWriter, error) { @@ -41,8 +42,9 @@ func NewRecordWriter(config *WriterConfig) (*RecordWriter, error) { } writer := &RecordWriter{ - fileWriter: fileWriter, - metadata: config.Metadata, + fileWriter: fileWriter, + metadata: config.Metadata, + writeCoveringMetadata: config.WriteCoveringMetadata, } return writer, nil @@ -66,7 +68,7 @@ func (w *RecordWriter) Close() error { if !w.wroteGeoMetadata { metadata := w.metadata if metadata == nil { - metadata = DefaultMetadata() + metadata = DefaultMetadata(w.writeCoveringMetadata) } data, err := json.Marshal(metadata) if err != nil { diff --git a/internal/geoparquet/writer.go b/internal/geoparquet/writer.go index 5554118..11e3bc3 100644 --- a/internal/geoparquet/writer.go +++ b/internal/geoparquet/writer.go @@ -9,9 +9,10 @@ import ( ) type WriterConfig struct { - Writer io.Writer - Metadata *Metadata - ParquetWriterProps *parquet.WriterProperties - ArrowWriterProps *pqarrow.ArrowWriterProperties - ArrowSchema *arrow.Schema + Writer io.Writer + Metadata *Metadata + ParquetWriterProps *parquet.WriterProperties + ArrowWriterProps *pqarrow.ArrowWriterProperties + ArrowSchema *arrow.Schema + WriteCoveringMetadata bool } diff --git a/internal/pqutil/arrow.go b/internal/pqutil/arrow.go index 0f20c98..b68c5b4 100644 --- a/internal/pqutil/arrow.go +++ b/internal/pqutil/arrow.go @@ -39,6 +39,17 @@ func (b *ArrowSchemaBuilder) AddGeometry(name string, encoding string) error { return nil } +func (b *ArrowSchemaBuilder) AddBbox(name string) { + bboxFields := []arrow.Field{ + {Name: "xmin", Type: arrow.PrimitiveTypes.Float64, Nullable: false}, + {Name: "ymin", Type: arrow.PrimitiveTypes.Float64, Nullable: false}, + {Name: "xmax", Type: arrow.PrimitiveTypes.Float64, Nullable: false}, + {Name: "ymax", Type: arrow.PrimitiveTypes.Float64, Nullable: false}, + } + dataType := arrow.StructOf(bboxFields...) + b.fields[name] = &arrow.Field{Name: name, Type: dataType, Nullable: true} +} + func (b *ArrowSchemaBuilder) Add(record map[string]any) error { for name, value := range record { if b.fields[name] != nil { From 643c44ba1e02e43826fa988b01acd4b1ec4a84b9 Mon Sep 17 00:00:00 2001 From: Felix Schott Date: Thu, 27 Mar 2025 14:51:11 +0100 Subject: [PATCH 10/11] documentation --- cmd/gpq/command/convert.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cmd/gpq/command/convert.go b/cmd/gpq/command/convert.go index dd12574..b200dbe 100644 --- a/cmd/gpq/command/convert.go +++ b/cmd/gpq/command/convert.go @@ -35,7 +35,7 @@ type ConvertCmd struct { InputPrimaryColumn string `help:"Primary geometry column name when reading Parquet without metadata." default:"geometry"` Compression string `help:"Parquet compression to use. Possible values: ${enum}." enum:"uncompressed, snappy, gzip, brotli, zstd" default:"zstd"` RowGroupLength int `help:"Maximum number of rows per group when writing Parquet."` - AddBbox bool `help:"Compute the bounding box of features where not yet available and write to Parquet output."` + AddBbox bool `help:"Compute the bounding box of features where not present in GeoJSON input and write to Parquet output."` } type FormatType string @@ -149,6 +149,10 @@ func (c *ConvertCmd) Run() error { output = o } + if c.AddBbox && outputFormat != GeoParquetType { + return NewCommandError("--add-bbox is only available when converting to GeoParquet.") + } + if inputFormat == GeoJSONType { if outputFormat != ParquetType && outputFormat != GeoParquetType { return NewCommandError("GeoJSON input can only be converted to GeoParquet") From 26c3f6ae8dc55d9e1f1e6b340b8921376fb02fb3 Mon Sep 17 00:00:00 2001 From: Felix Schott Date: Fri, 11 Apr 2025 18:08:56 +0200 Subject: [PATCH 11/11] add comment --- internal/geoparquet/geoparquet.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/geoparquet/geoparquet.go b/internal/geoparquet/geoparquet.go index efdc79b..4c884ed 100644 --- a/internal/geoparquet/geoparquet.go +++ b/internal/geoparquet/geoparquet.go @@ -244,6 +244,8 @@ func GetBboxColumn(schema *schema.Schema, geoMetadata *Metadata) *BboxColumn { bboxCol := &BboxColumn{} // try standard name first bboxCol.Name = DefaultBboxColumn + + // NB: we can't do schema.ColumnIndexByName() in this case as it won't give the expected result for nested types like structs bboxCol.Index = schema.Root().FieldIndexByName(DefaultBboxColumn) // if no match, check covering metadata @@ -256,7 +258,7 @@ func GetBboxColumn(schema *schema.Schema, geoMetadata *Metadata) *BboxColumn { } } - bboxCol.BaseColumn = schema.ColumnIndexByName(geoMetadata.PrimaryColumn) + bboxCol.BaseColumn = schema.Root().FieldIndexByName(geoMetadata.PrimaryColumn) bboxCol.BboxColumnFieldNames = *GetBboxColumnFieldNames(geoMetadata) return bboxCol }