post index func rather than belongs to

cmd/wof-sqlite-index-example/main.go

@@ -12,8 +12,6 @@ import (
 	"github.com/whosonfirst/go-whosonfirst-sqlite-index"
 	"github.com/whosonfirst/go-whosonfirst-sqlite/database"
 	"github.com/whosonfirst/go-whosonfirst-sqlite/tables"
-	"github.com/whosonfirst/go-reader"
-	_ "github.com/whosonfirst/go-reader-http"	
 	"io"
 	"os"
 	"strings"
@@ -37,13 +35,10 @@ func main() {
 	live_hard := flag.Bool("live-hard-die-fast", true, "Enable various performance-related pragmas at the expense of possible (unlikely) database corruption")
 	timings := flag.Bool("timings", false, "Display timings during and after indexing")
 
-	index_belongsto := flag.Bool("index-belongs-to", false, "Index wof:belongsto records for each feature")
-	belongsto_reader_uri := flag.String("belongs-to-uri", "", "A valid whosonfirst/go-reader Reader URI for reading wof:belongsto records")
+	post_index := flag.Bool("post-index", false, "Enable post indexing callback function")
 
 	flag.Parse()
 
-	ctx := context.Background()
-
 	logger := log.SimpleWOFLogger()
 
 	stdout := io.Writer(os.Stdout)
@@ -76,7 +71,7 @@ func main() {
 
 	to_index = append(to_index, ex)
 
-	cb := func(ctx context.Context, fh io.Reader, args ...interface{}) (interface{}, error) {
+	record_func := func(ctx context.Context, fh io.Reader, args ...interface{}) (interface{}, error) {
 
 		now := time.Now()
 
@@ -88,23 +83,21 @@ func main() {
 	}
 
 	idx_opts := &index.SQLiteIndexerOptions{
-		DB: db,
-		Tables: to_index,
-		Callback: cb,
+		DB:             db,
+		Tables:         to_index,
+		LoadRecordFunc: record_func,
 	}
 
-	if *index_belongsto {
+	if *post_index {
 
-		r, err := reader.NewReader(ctx, *belongsto_reader_uri)
-
-		if err != nil {
-			logger.Fatal("Failed to create Reader (%s), %v", *belongsto_reader_uri, err)
+		post_func := func(ctx context.Context, db sqlite.Database, tables []sqlite.Table, record interface{}) error {
+			logger.Status("Post index func w/ %v", record)
+			return nil
 		}
 
-		idx_opts.IndexBelongsTo = true
-		idx_opts.BelongsToReader = r
+		idx_opts.PostIndexFunc = post_func
 	}
-	
+
 	idx, err := index.NewSQLiteIndexer(idx_opts)
 
 	if err != nil {

go.mod

@@ -3,14 +3,14 @@ module github.com/whosonfirst/go-whosonfirst-sqlite-index
 go 1.12
 
 require (
-	github.com/whosonfirst/go-reader v0.1.1
-	github.com/whosonfirst/go-reader-http v0.0.2
-	github.com/whosonfirst/go-whosonfirst-geojson-v2 v0.12.2
+	github.com/whosonfirst/go-reader v0.1.1 // indirect
+	github.com/whosonfirst/go-reader-http v0.0.2 // indirect
+	github.com/whosonfirst/go-whosonfirst-geojson-v2 v0.12.2 // indirect
 	github.com/whosonfirst/go-whosonfirst-index v0.2.3
 	github.com/whosonfirst/go-whosonfirst-index-csv v0.0.0-20191002171239-c6712fe20972
 	github.com/whosonfirst/go-whosonfirst-index-sqlite v0.0.3
 	github.com/whosonfirst/go-whosonfirst-log v0.1.0
-	github.com/whosonfirst/go-whosonfirst-reader v0.0.1
+	github.com/whosonfirst/go-whosonfirst-reader v0.0.1 // indirect
 	github.com/whosonfirst/go-whosonfirst-sqlite v0.1.5
-	github.com/whosonfirst/go-whosonfirst-sqlite-features v0.2.7
+	github.com/whosonfirst/go-whosonfirst-sqlite-features v0.2.7 // indirect
 )

index.go

@@ -2,22 +2,18 @@ package index
 
 import (
 	"context"
-	"fmt"
-	"github.com/whosonfirst/go-reader"
-	"github.com/whosonfirst/go-whosonfirst-geojson-v2"
-	"github.com/whosonfirst/go-whosonfirst-geojson-v2/properties/whosonfirst"
 	wof_index "github.com/whosonfirst/go-whosonfirst-index"
 	"github.com/whosonfirst/go-whosonfirst-log"
-	wof_reader "github.com/whosonfirst/go-whosonfirst-reader"
 	"github.com/whosonfirst/go-whosonfirst-sqlite"
-	wof_tables "github.com/whosonfirst/go-whosonfirst-sqlite-features/tables"
 	"io"
 	"sync"
 	"sync/atomic"
 	"time"
 )
 
-type SQLiteIndexerFunc func(context.Context, io.Reader, ...interface{}) (interface{}, error)
+type SQLiteIndexerPostIndexFunc func(context.Context, sqlite.Database, []sqlite.Table, interface{}) error
+
+type SQLiteIndexerLoadRecordFunc func(context.Context, io.Reader, ...interface{}) (interface{}, error)
 
 type SQLiteIndexer struct {
 	callback      wof_index.IndexerFunc
@@ -28,18 +24,17 @@ type SQLiteIndexer struct {
 }
 
 type SQLiteIndexerOptions struct {
-	DB              sqlite.Database
-	Tables          []sqlite.Table
-	Callback        SQLiteIndexerFunc
-	IndexBelongsTo  bool
-	BelongsToReader reader.Reader
+	DB             sqlite.Database
+	Tables         []sqlite.Table
+	LoadRecordFunc SQLiteIndexerLoadRecordFunc
+	PostIndexFunc  SQLiteIndexerPostIndexFunc
 }
 
 func NewSQLiteIndexer(opts *SQLiteIndexerOptions) (*SQLiteIndexer, error) {
 
 	db := opts.DB
 	tables := opts.Tables
-	callback := opts.Callback
+	record_func := opts.LoadRecordFunc
 
 	table_timings := make(map[string]time.Duration)
 	mu := new(sync.RWMutex)
@@ -54,7 +49,7 @@ func NewSQLiteIndexer(opts *SQLiteIndexerOptions) (*SQLiteIndexer, error) {
 			return err
 		}
 
-		record, err := callback(ctx, fh, args...)
+		record, err := record_func(ctx, fh, args...)
 
 		if err != nil {
 			logger.Warning("failed to load record (%s) because %s", path, err)
@@ -97,61 +92,13 @@ func NewSQLiteIndexer(opts *SQLiteIndexerOptions) (*SQLiteIndexer, error) {
 			mu.Unlock()
 		}
 
-		if opts.IndexBelongsTo {
+		if opts.PostIndexFunc != nil {
 
-			geojson_t, err := wof_tables.NewGeoJSONTable()
+			err := opts.PostIndexFunc(ctx, db, tables, record)
 
 			if err != nil {
 				return err
 			}
-
-			conn, err := db.Conn()
-
-			if err != nil {
-				return err
-			}
-
-			f := record.(geojson.Feature)
-			belongsto := whosonfirst.BelongsTo(f)
-
-			to_index := make([]geojson.Feature, 0)
-
-			for _, id := range belongsto {
-
-				sql := fmt.Sprintf("SELECT COUNT(id) FROM %s WHERE id=?", geojson_t.Name())
-				row := conn.QueryRow(sql, id)
-
-				var count int
-				err = row.Scan(&count)
-
-				if err != nil {
-					return err
-				}
-
-				if count == 0 {
-
-					ancestor, err := wof_reader.LoadFeatureFromID(ctx, opts.BelongsToReader, id)
-
-					if err != nil {
-						return err
-					}
-
-					to_index = append(to_index, ancestor)
-				}
-			}
-
-			for _, record := range to_index {
-
-				for _, t := range tables {
-
-					err = t.IndexRecord(db, record)
-
-					if err != nil {
-						return err
-					}
-				}
-			}
-
 		}
 
 		return nil