update to reflect changes to go-whosonfirst-spatial interfaces

Author: thisisaaronland

README.md

@@ -6,18 +6,13 @@ This is work in progress. It may change, probably has bugs and isn't properly do
 
 The goal is to have a package that conforms to the [database.SpatialDatabase](https://github.com/whosonfirst/go-whosonfirst-spatial#spatialdatabase) interface using [mattn/go-sqlite3](https://github.com/mattn/go-sqlite3) and SQLite's [RTree](https://www.sqlite.org/rtree.html) extension.
 
-Also, this is not as fast as it should be. This is largely with the way WOF records are inflated and passed around in order to support GeoJSON output. There is [an open ticket](https://github.com/whosonfirst/go-whosonfirst-spatial-sqlite/issues/2) to address this.
-
 ## Databases
 
 This code depends on (4) tables as indexed by the `go-whosonfirst-sqlite-features` package:
 
 * [rtree](https://github.com/whosonfirst/go-whosonfirst-sqlite-features#rtree) - this table is used to perform point-in-polygon spatial queries.
 * [spr](https://github.com/whosonfirst/go-whosonfirst-sqlite-features#spr) - this table is used to generate [standard place response](#) (SPR) results.
-* [geometry](https://github.com/whosonfirst/go-whosonfirst-sqlite-features#geometry) - this table is used to append geometries to GeoJSON-formatted results.
-* [properties](https://github.com/whosonfirst/go-whosonfirst-sqlite-features#properties) - this table is used to append extra properties (to the SPR response) for GeoJSON-formatted results.
-
-The `go-whosonfirst-sqlite-features` package also indexes a `geojson` table  but it turns out that retrieving, and parsing, properties and geometries from their own tables is faster.
+* [properties](https://github.com/whosonfirst/go-whosonfirst-sqlite-features#properties) - this table is used to append extra properties (to the SPR response) for `spatial.PropertiesResponseResults` responses.
 
 Here's an example of the creating a compatible SQLite database for all the [administative data in Canada](https://github.com/whosonfirst-data/whosonfirst-data-admin-ca) using the `wof-sqlite-index-features` tool which is part of the [go-whosonfirst-sqlite-features-index](https://github.com/whosonfirst/go-whosonfirst-sqlite-features-index) package:
 
@@ -61,18 +56,18 @@ $> ./bin/query \
 2020/12/15 15:32:06 Unable to parse placetype (alt) for ID 85633041, because 'Invalid placetype' - skipping placetype filters
 2020/12/15 15:32:06 Unable to parse placetype (alt) for ID 136251273, because 'Invalid placetype' - skipping placetype filters
 2020/12/15 15:32:06 Unable to parse placetype (alt) for ID 85633041, because 'Invalid placetype' - skipping placetype filters
-2020/12/15 15:32:06 Time to point in polygon, 596.579126ms
-      "wof:id": "1108955735",
+2020/12/16 13:25:32 Time to point in polygon, 395.201983ms
+      "wof:id": "85633041",
       "wof:id": "85874359",
+      "wof:id": "1108955735",
       "wof:id": "85874359",
-      "wof:id": "890458661",
       "wof:id": "85633041",
+      "wof:id": "890458661",
       "wof:id": "136251273",
-      "wof:id": "85633041",
-      "wof:id": "85633041",
       "wof:id": "136251273",
       "wof:id": "85633041",
       "wof:id": "136251273",
+      "wof:id": "85633041",
 ```
 
 _TBW: Indexing tables on start-up._
@@ -237,7 +232,7 @@ $> ./bin/query \
 Note: This assumes a database that was previously indexed using the [whosonfirst/go-whosonfirst-sqlite-features](https://github.com/whosonfirst/go-whosonfirst-sqlite-features) `wof-sqlite-index-features` tool. For example:
 
 ```
-$> ./bin/wof-sqlite-index-features -rtree -geojson -dsn /tmp/test.db -mode repo:// /usr/local/data/sfomuseum-data-architecture/
+$> ./bin/wof-sqlite-index-features -rtree -spr -properties -dsn /tmp/test.db -mode repo:// /usr/local/data/sfomuseum-data-architecture/
 ```
 
 ## See also

database.go

@@ -11,11 +11,11 @@ import (
 	"github.com/skelterjohn/geom"
 	wof_geojson "github.com/whosonfirst/go-whosonfirst-geojson-v2"
 	"github.com/whosonfirst/go-whosonfirst-log"
-	"github.com/whosonfirst/go-whosonfirst-spatial"	
+	"github.com/whosonfirst/go-whosonfirst-spatial"
 	"github.com/whosonfirst/go-whosonfirst-spatial/database"
 	"github.com/whosonfirst/go-whosonfirst-spatial/filter"
-	"github.com/whosonfirst/go-whosonfirst-spatial/timer"	
 	"github.com/whosonfirst/go-whosonfirst-spatial/geo"
+	"github.com/whosonfirst/go-whosonfirst-spatial/timer"
 	"github.com/whosonfirst/go-whosonfirst-spr"
 	"github.com/whosonfirst/go-whosonfirst-sqlite"
 	"github.com/whosonfirst/go-whosonfirst-sqlite-features/tables"
@@ -34,16 +34,14 @@ func init() {
 
 type SQLiteSpatialDatabase struct {
 	database.SpatialDatabase
-	Logger         *log.WOFLogger
-	mu             *sync.RWMutex
-	db             *sqlite_database.SQLiteDatabase
-	rtree_table    sqlite.Table
-	geometry_table sqlite.Table
-	spr_table      sqlite.Table
-	gocache        *gocache.Cache
-	dsn            string
-	strict         bool
-	timer          *timer.Timer
+	Logger      *log.WOFLogger
+	Timer       *timer.Timer
+	mu          *sync.RWMutex
+	db          *sqlite_database.SQLiteDatabase
+	rtree_table sqlite.Table
+	spr_table   sqlite.Table
+	gocache     *gocache.Cache
+	dsn         string
 }
 
 type RTreeSpatialIndex struct {
@@ -99,12 +97,6 @@ func NewSQLiteSpatialDatabase(ctx context.Context, uri string) (database.Spatial
 		return nil, err
 	}
 
-	geometry_table, err := tables.NewGeometryTableWithDatabase(sqlite_db)
-
-	if err != nil {
-		return nil, err
-	}
-
 	rtree_table, err := tables.NewRTreeTableWithDatabase(sqlite_db)
 
 	if err != nil {
@@ -117,12 +109,6 @@ func NewSQLiteSpatialDatabase(ctx context.Context, uri string) (database.Spatial
 		return nil, err
 	}
 
-	strict := true
-
-	if q.Get("strict") == "false" {
-		strict = false
-	}
-
 	logger := log.SimpleWOFLogger("index")
 
 	expires := 5 * time.Minute
@@ -135,16 +121,14 @@ func NewSQLiteSpatialDatabase(ctx context.Context, uri string) (database.Spatial
 	t := timer.NewTimer()
 
 	spatial_db := &SQLiteSpatialDatabase{
-		Logger:         logger,
-		db:             sqlite_db,
-		rtree_table:    rtree_table,
-		geometry_table: geometry_table,
-		spr_table:      spr_table,
-		gocache:        gc,
-		dsn:            dsn,
-		strict:         strict,
-		mu:             mu,
-		timer:          t,
+		Logger:      logger,
+		Timer:       t,
+		db:          sqlite_db,
+		rtree_table: rtree_table,
+		spr_table:   spr_table,
+		gocache:     gc,
+		dsn:         dsn,
+		mu:          mu,
 	}
 
 	return spatial_db, nil
@@ -170,12 +154,6 @@ func (r *SQLiteSpatialDatabase) IndexFeature(ctx context.Context, f wof_geojson.
 		return err
 	}
 
-	err = r.geometry_table.IndexRecord(r.db, f)
-
-	if err != nil {
-		return err
-	}
-
 	return nil
 }
 
@@ -218,12 +196,14 @@ func (r *SQLiteSpatialDatabase) PointInPolygon(ctx context.Context, coord *geom.
 		}
 	}
 
-	for label, timings := range r.timer.Timings {
+	/*
+		for label, timings := range r.Timer.Timings {
 
-		for _, tm := range timings {
-			golog.Printf("[%s] %s\n", label, tm)
+			for _, tm := range timings {
+				golog.Printf("[%s] %s\n", label, tm)
+			}
 		}
-	}
+	*/
 
 	spr_results := &SQLiteResults{
 		Places: results,
@@ -301,14 +281,14 @@ func (r *SQLiteSpatialDatabase) PointInPolygonCandidatesWithChannels(ctx context
 	for _, sp := range intersects {
 
 		bounds := sp.Bounds()
-		
+
 		c := &spatial.PointInPolygonCandidate{
-			Id: sp.Id,
-			WOFId: sp.WOFId,
+			Id:       sp.Id,
+			WOFId:    sp.WOFId,
 			AltLabel: sp.AltLabel,
-			Bounds: &bounds,
+			Bounds:   &bounds,
 		}
-		
+
 		rsp_ch <- c
 	}
 
@@ -441,7 +421,7 @@ func (r *SQLiteSpatialDatabase) inflateSpatialIndexWithChannels(ctx context.Cont
 	t1 := time.Now()
 
 	defer func() {
-		r.timer.Add(ctx, sp_id, "time to inflate", time.Since(t1))
+		r.Timer.Add(ctx, sp_id, "time to inflate", time.Since(t1))
 	}()
 
 	// have we already looked up the filters for this ID?
@@ -458,12 +438,12 @@ func (r *SQLiteSpatialDatabase) inflateSpatialIndexWithChannels(ctx context.Cont
 	t2 := time.Now()
 
 	// this needs to be sped up (20201216/thisisaaronland)
-	
+
 	var coords [][][]float64
 
 	err := json.Unmarshal([]byte(sp.geometry), &coords)
 
-	r.timer.Add(ctx, sp_id, "time to unmarshal geometry", time.Since(t2))
+	r.Timer.Add(ctx, sp_id, "time to unmarshal geometry", time.Since(t2))
 
 	if err != nil {
 		err_ch <- err
@@ -481,7 +461,7 @@ func (r *SQLiteSpatialDatabase) inflateSpatialIndexWithChannels(ctx context.Cont
 		return
 	}
 
-	r.timer.Add(ctx, sp_id, "time to perform contains test", time.Since(t3))
+	r.Timer.Add(ctx, sp_id, "time to perform contains test", time.Since(t3))
 
 	// there is at least one ring that contains the coord
 	// now we check the filters - whether or not they pass
@@ -501,7 +481,7 @@ func (r *SQLiteSpatialDatabase) inflateSpatialIndexWithChannels(ctx context.Cont
 		return
 	}
 
-	r.timer.Add(ctx, sp_id, "time to retrieve SPR", time.Since(t4))
+	r.Timer.Add(ctx, sp_id, "time to retrieve SPR", time.Since(t4))
 
 	if err != nil {
 		r.Logger.Error("Failed to retrieve feature cache for %s, %v", sp_id, err)
@@ -520,7 +500,7 @@ func (r *SQLiteSpatialDatabase) inflateSpatialIndexWithChannels(ctx context.Cont
 		}
 	}
 
-	r.timer.Add(ctx, sp_id, "time to filter SPR", time.Since(t5))
+	r.Timer.Add(ctx, sp_id, "time to filter SPR", time.Since(t5))
 
 	rsp_ch <- s
 }
@@ -532,7 +512,7 @@ func (r *SQLiteSpatialDatabase) retrieveSPR(ctx context.Context, uri_str string)
 	if ok {
 		return c.(*SQLiteStandardPlacesResult), nil
 	}
-	
+
 	id, uri_args, err := uri.ParseURI(uri_str)
 
 	if err != nil {

go.mod

@@ -4,16 +4,14 @@ go 1.12
 
 require (
 	github.com/patrickmn/go-cache v2.1.0+incompatible
-	github.com/paulmach/go.geojson v1.4.0
 	github.com/sfomuseum/go-flags v0.4.2
 	github.com/skelterjohn/geom v0.0.0-20180103142417-96f3e8a219c5
-	github.com/tidwall/gjson v1.6.4
 	github.com/whosonfirst/go-whosonfirst-flags v0.1.0
 	github.com/whosonfirst/go-whosonfirst-geojson-v2 v0.14.0
 	github.com/whosonfirst/go-whosonfirst-log v0.1.0
-	github.com/whosonfirst/go-whosonfirst-spatial v0.0.13
+	github.com/whosonfirst/go-whosonfirst-spatial v0.0.14
 	github.com/whosonfirst/go-whosonfirst-spr v0.1.0
 	github.com/whosonfirst/go-whosonfirst-sqlite v0.1.6
-	github.com/whosonfirst/go-whosonfirst-sqlite-features v0.6.2
+	github.com/whosonfirst/go-whosonfirst-sqlite-features v0.6.3
 	github.com/whosonfirst/go-whosonfirst-uri v0.2.0
 )

go.sum

@@ -141,6 +141,8 @@ github.com/whosonfirst/go-whosonfirst-spatial v0.0.12 h1:Hl++UT0fye7f4c9/IxI7pAX
 github.com/whosonfirst/go-whosonfirst-spatial v0.0.12/go.mod h1:PywJosSLB0RCNgTPyWgQ+gMjxRj8Uw3hX3zPa+JZKug=
 github.com/whosonfirst/go-whosonfirst-spatial v0.0.13 h1:q1jf79ahX+4yGfmwMwWx9TYScq9AQOR2HXjdVmfUoQ0=
 github.com/whosonfirst/go-whosonfirst-spatial v0.0.13/go.mod h1:+zpBqUkaRifbarW9f3GIahiD87H+aNkJU3RsfRA/RxA=
+github.com/whosonfirst/go-whosonfirst-spatial v0.0.14 h1:BwAuGx+zp793p7t9TDM5HC3HNe10MdX2FQaMiloQ4N0=
+github.com/whosonfirst/go-whosonfirst-spatial v0.0.14/go.mod h1:+zpBqUkaRifbarW9f3GIahiD87H+aNkJU3RsfRA/RxA=
 github.com/whosonfirst/go-whosonfirst-spr v0.1.0 h1:5qE629nCiucF2upy5NjPOEl9cFatsljykYY0l2JKgAk=
 github.com/whosonfirst/go-whosonfirst-spr v0.1.0/go.mod h1:R8GtEVz1GVSnwwOjzcoVUd172ZK26Q7hQSLI6SGG7lM=
 github.com/whosonfirst/go-whosonfirst-sqlite v0.1.6 h1:XhAlLoPm7y/4565du5H7R5Swjf/pBl+cuXHoAs6evLA=
@@ -149,6 +151,8 @@ github.com/whosonfirst/go-whosonfirst-sqlite-features v0.4.0 h1:hogsGiJCxzyHo7L+
 github.com/whosonfirst/go-whosonfirst-sqlite-features v0.4.0/go.mod h1:FG3V5mkagABuTB5hOeLm1LKq80pSJpW67TdC4vNPMaY=
 github.com/whosonfirst/go-whosonfirst-sqlite-features v0.6.2 h1:7eWyeIwsHZTeMmSdmRVuj6X9bvjEUamE2caDrOZ0uXw=
 github.com/whosonfirst/go-whosonfirst-sqlite-features v0.6.2/go.mod h1:jZXXBBarIOYmkFl/LiKPjE4dQEunUZjV4fGqQImnXYQ=
+github.com/whosonfirst/go-whosonfirst-sqlite-features v0.6.3 h1:1FH//dWDsBUr1ZVC6CbGj+Y7TDU44SvKs8UiZc3tr80=
+github.com/whosonfirst/go-whosonfirst-sqlite-features v0.6.3/go.mod h1:jZXXBBarIOYmkFl/LiKPjE4dQEunUZjV4fGqQImnXYQ=
 github.com/whosonfirst/go-whosonfirst-uri v0.1.0/go.mod h1:8eaDVcc4v+HHHEDaRbApdmhPwM4/JQllw2PktvZcPVs=
 github.com/whosonfirst/go-whosonfirst-uri v0.2.0 h1:iODHdyvW+8IXqHZTixZ/9GEZy1dVKGj6dMRg7fn0d2M=
 github.com/whosonfirst/go-whosonfirst-uri v0.2.0/go.mod h1:8eaDVcc4v+HHHEDaRbApdmhPwM4/JQllw2PktvZcPVs=