From b195bfc7882c3df1c50c48d9c0770fb6a5716af3 Mon Sep 17 00:00:00 2001 From: Omkar Mestry Date: Sat, 21 Nov 2020 13:29:05 +0530 Subject: [PATCH 1/3] Added changes to optimize points_within_polygons function --- tests/test_measurement.py | 14 ++------ turfpy/helper.py | 14 +++++--- turfpy/measurement.py | 76 ++++++++++++++++++++++----------------- turfpy/meta.py | 16 ++------- turfpy/transformation.py | 10 +++--- 5 files changed, 62 insertions(+), 68 deletions(-) diff --git a/tests/test_measurement.py b/tests/test_measurement.py index 4b905e5..ba302a5 100644 --- a/tests/test_measurement.py +++ b/tests/test_measurement.py @@ -84,14 +84,7 @@ def test_bbox_multi_polygon(): mp = MultiPolygon( [ ([(3.78, 9.28), (-130.91, 1.52), (35.12, 72.234), (3.78, 9.28)],), - ( - [ - (23.18, -34.29), - (-1.31, -4.61), - (3.41, 77.91), - (23.18, -34.29), - ], - ), + ([(23.18, -34.29), (-1.31, -4.61), (3.41, 77.91), (23.18, -34.29),],), ] ) bb = bbox(mp) @@ -178,10 +171,7 @@ def test_rhumb_destination(): distance = 50 bearing = 90 dest = rhumb_destination( - start, - distance, - bearing, - {"units": "mi", "properties": {"marker-color": "F00"}}, + start, distance, bearing, {"units": "mi", "properties": {"marker-color": "F00"}}, ) assert dest["geometry"]["coordinates"] == [-74.398553, 39.984] diff --git a/turfpy/helper.py b/turfpy/helper.py index 7203028..b338622 100644 --- a/turfpy/helper.py +++ b/turfpy/helper.py @@ -59,11 +59,15 @@ def get_coord(coord): raise Exception("coord is required") if not isinstance(coord, list): - if coord.type == "Feature" and coord.geometry and coord.geometry.type == "Point": - return coord.geometry.coordinates - - if coord.type == "Point": - return coord.coordinates + if ( + coord["type"] == "Feature" + and coord["geometry"] + and coord["geometry"]["type"] == "Point" + ): + return coord["geometry"]["coordinates"] + + if coord["type"] == "Point": + return coord["coordinates"] if ( isinstance(coord, list) diff --git a/turfpy/measurement.py b/turfpy/measurement.py index 1c31eaa..b842ce8 100644 --- a/turfpy/measurement.py +++ b/turfpy/measurement.py @@ -4,8 +4,11 @@ This is mainly inspired by turf.js. link: http://turfjs.org/ """ +import concurrent.futures +from functools import partial from math import asin, atan2, cos, degrees, log, pi, pow, radians, sin, sqrt, tan -from typing import Optional, Union +from multiprocessing import Manager +from typing import Optional, Union, List from geojson import ( Feature, @@ -801,11 +804,7 @@ def explode(geojson): def _callback_feature_each(feature, feature_index): def _callback_coord_each( - coord, - coord_index, - feature_index, - multi_feature_index, - geometry_index, + coord, coord_index, feature_index, multi_feature_index, geometry_index, ): nonlocal points point = Point(coord) @@ -817,11 +816,7 @@ def _callback_coord_each( else: def _callback_coord_each( - coord, - coord_index, - feature_index, - multi_feature_index, - geometry_index, + coord, coord_index, feature_index, multi_feature_index, geometry_index, ): nonlocal points, geojson point = Point(coord) @@ -878,9 +873,7 @@ def polygon_tangents(point, polygon): ltan = poly_coords[0][nearest_pt_index] eprev = _is_left( - poly_coords[0][0], - poly_coords[0][len(poly_coords[0]) - 1], - point_coords, + poly_coords[0][0], poly_coords[0][len(poly_coords[0]) - 1], point_coords, ) out = process_polygon(poly_coords[0], point_coords, eprev, enext, rtan, ltan) rtan = out[0] @@ -1143,8 +1136,7 @@ def rhumb_destination(origin, distance, bearing, options: dict = {}) -> Feature: coords = get_coord(origin) destination_point = _calculate_rhumb_destination(coords, distance_in_meters, bearing) return Feature( - geometry=Point(destination_point), - properties=options.get("properties", ""), + geometry=Point(destination_point), properties=options.get("properties", ""), ) @@ -1294,7 +1286,9 @@ def square(bbox: list): def points_within_polygon( - points: Union[Feature, FeatureCollection], polygons: Union[Feature, FeatureCollection] + points: Union[Feature, FeatureCollection], + polygons: Union[Feature, FeatureCollection], + chunk_size: int = 1, ) -> FeatureCollection: """Find Point(s) that fall within (Multi)Polygon(s). @@ -1308,26 +1302,42 @@ def points_within_polygon( :param points: A single GeoJSON ``Point`` feature or FeatureCollection of Points. :param polygons: A Single GeoJSON Polygon/MultiPolygon or FeatureCollection of Polygons/MultiPolygons. + :param chunk_size: Number of chunks each process to handle. The default value is + 1, for a large number of features please use `chunk_size` greater than 1 + to get better results in terms of performance. :return: A :class:`geojson.FeatureCollection` of Points. """ - results = [] + if not points: + raise Exception("Points cannot be empty") - def __callback_feature_each(feature, feature_index): - contained = False + if points["type"] == "Point": + points = FeatureCollection([Feature(geometry=points)]) - def __callback_geom_each( - current_geometry, feature_index, feature_properties, feature_bbox, feature_id - ): - if boolean_point_in_polygon(feature, current_geometry): - nonlocal contained - contained = True + if points["type"] == "Feature": + points = FeatureCollection([points]) + + manager = Manager() + results: List[dict] = manager.list() + + part_func = partial(check_each_point, polygons=polygons, results=results,) - if contained: - nonlocal results - results.append(feature) + with concurrent.futures.ProcessPoolExecutor() as executor: + for _ in executor.map(part_func, points["features"], chunksize=chunk_size): + pass + + return FeatureCollection(list(results)) + + +def check_each_point(point, polygons, results): + def __callback_geom_each( + current_geometry, feature_index, feature_properties, feature_bbox, feature_id + ): + contained = False + if boolean_point_in_polygon(point, current_geometry): + contained = True - geom_each(polygons, __callback_geom_each) - return True + if contained: + nonlocal results + results.append(point) - feature_each(points, __callback_feature_each) - return FeatureCollection(results) + geom_each(polygons, __callback_geom_each) diff --git a/turfpy/meta.py b/turfpy/meta.py index 0aef4f2..6ae1133 100644 --- a/turfpy/meta.py +++ b/turfpy/meta.py @@ -10,11 +10,7 @@ def geom_reduce(geojson, initial_value_param): previous_value = initial_value_param def callback_geom_each( - current_geometry, - feature_index, - feature_properties, - feature_bbox, - feature_id, + current_geometry, feature_index, feature_properties, feature_bbox, feature_id, ): nonlocal initial_value nonlocal previous_value @@ -81,11 +77,7 @@ def geom_each(geojson, callback): if not geometry: if not callback( - None, - feature_index, - feature_properties, - feature_b_box, - feature_id, + None, feature_index, feature_properties, feature_b_box, feature_id, ): return False continue @@ -305,9 +297,7 @@ def coord_each(geojson, callback, excludeWrapCoord=None): elif geom_type == "GeometryCollection": for j in range(0, len(geometry["geometries"])): if not coord_each( - geometry["geometries"][j], - callback, - excludeWrapCoord, + geometry["geometries"][j], callback, excludeWrapCoord, ): return False else: diff --git a/turfpy/transformation.py b/turfpy/transformation.py index 90be9a4..2384edd 100644 --- a/turfpy/transformation.py +++ b/turfpy/transformation.py @@ -678,10 +678,7 @@ def _callback_coord_each( def transform_scale( - features, - factor: float, - origin: Union[str, list] = "centroid", - mutate: bool = False, + features, factor: float, origin: Union[str, list] = "centroid", mutate: bool = False, ): """ Scale a GeoJSON from a given @@ -814,7 +811,10 @@ def tesselate(poly: Feature) -> FeatureCollection: ... [21, 15], [11, 11], [11, 0]]], "type": "Polygon"}) >>> tesselate(polygon) """ - if poly.geometry.type != "Polygon" and poly.geometry.type != "MultiPolygon": + if ( + poly["geometry"]["type"] != "Polygon" + and poly["geometry"]["type"] != "MultiPolygon" + ): raise ValueError("Geometry must be Polygon or MultiPolygon") fc = FeatureCollection([]) From db5fe9fd788dafc87709c913cae8fc6069289f39 Mon Sep 17 00:00:00 2001 From: Omkar Mestry Date: Sat, 21 Nov 2020 13:32:10 +0530 Subject: [PATCH 2/3] Added changes to optimize points_within_polygons function --- measurements.md | 1 + 1 file changed, 1 insertion(+) diff --git a/measurements.md b/measurements.md index 28f830c..64b4535 100644 --- a/measurements.md +++ b/measurements.md @@ -427,6 +427,7 @@ square(bbox) | ------- | ---------------------------------------------------------- | ---------------------------------------------- | | `points` | Feature/FeatureCollection of Points | FeatureCollection of Points to find | | `polygons` | Feature/FeatureCollection of Polygon(s)/MultiPolygon(s) | FeatureCollection of Polygon(s)/MultiPolygon(s)| +| `chunk_size` | int | Number of chunks each process to handle. The default value is 1, for a large number of features please use `chunk_size` greater than 1 to get better results in terms of performance.| | Return | Type | Description | | ----------- | ------------------ | ----------------------------------------------------------------- | From 31fa99214bb8531495e5dcf2b746fa94fcd7f6bc Mon Sep 17 00:00:00 2001 From: Omkar Mestry Date: Sat, 21 Nov 2020 13:38:43 +0530 Subject: [PATCH 3/3] Changes to fix lint issue --- tests/test_measurement.py | 14 ++++++++++++-- turfpy/measurement.py | 27 +++++++++++++++++++++------ turfpy/meta.py | 16 +++++++++++++--- turfpy/transformation.py | 5 ++++- 4 files changed, 50 insertions(+), 12 deletions(-) diff --git a/tests/test_measurement.py b/tests/test_measurement.py index ba302a5..4b905e5 100644 --- a/tests/test_measurement.py +++ b/tests/test_measurement.py @@ -84,7 +84,14 @@ def test_bbox_multi_polygon(): mp = MultiPolygon( [ ([(3.78, 9.28), (-130.91, 1.52), (35.12, 72.234), (3.78, 9.28)],), - ([(23.18, -34.29), (-1.31, -4.61), (3.41, 77.91), (23.18, -34.29),],), + ( + [ + (23.18, -34.29), + (-1.31, -4.61), + (3.41, 77.91), + (23.18, -34.29), + ], + ), ] ) bb = bbox(mp) @@ -171,7 +178,10 @@ def test_rhumb_destination(): distance = 50 bearing = 90 dest = rhumb_destination( - start, distance, bearing, {"units": "mi", "properties": {"marker-color": "F00"}}, + start, + distance, + bearing, + {"units": "mi", "properties": {"marker-color": "F00"}}, ) assert dest["geometry"]["coordinates"] == [-74.398553, 39.984] diff --git a/turfpy/measurement.py b/turfpy/measurement.py index b842ce8..42680a9 100644 --- a/turfpy/measurement.py +++ b/turfpy/measurement.py @@ -8,7 +8,7 @@ from functools import partial from math import asin, atan2, cos, degrees, log, pi, pow, radians, sin, sqrt, tan from multiprocessing import Manager -from typing import Optional, Union, List +from typing import List, Optional, Union from geojson import ( Feature, @@ -804,7 +804,11 @@ def explode(geojson): def _callback_feature_each(feature, feature_index): def _callback_coord_each( - coord, coord_index, feature_index, multi_feature_index, geometry_index, + coord, + coord_index, + feature_index, + multi_feature_index, + geometry_index, ): nonlocal points point = Point(coord) @@ -816,7 +820,11 @@ def _callback_coord_each( else: def _callback_coord_each( - coord, coord_index, feature_index, multi_feature_index, geometry_index, + coord, + coord_index, + feature_index, + multi_feature_index, + geometry_index, ): nonlocal points, geojson point = Point(coord) @@ -873,7 +881,9 @@ def polygon_tangents(point, polygon): ltan = poly_coords[0][nearest_pt_index] eprev = _is_left( - poly_coords[0][0], poly_coords[0][len(poly_coords[0]) - 1], point_coords, + poly_coords[0][0], + poly_coords[0][len(poly_coords[0]) - 1], + point_coords, ) out = process_polygon(poly_coords[0], point_coords, eprev, enext, rtan, ltan) rtan = out[0] @@ -1136,7 +1146,8 @@ def rhumb_destination(origin, distance, bearing, options: dict = {}) -> Feature: coords = get_coord(origin) destination_point = _calculate_rhumb_destination(coords, distance_in_meters, bearing) return Feature( - geometry=Point(destination_point), properties=options.get("properties", ""), + geometry=Point(destination_point), + properties=options.get("properties", ""), ) @@ -1319,7 +1330,11 @@ def points_within_polygon( manager = Manager() results: List[dict] = manager.list() - part_func = partial(check_each_point, polygons=polygons, results=results,) + part_func = partial( + check_each_point, + polygons=polygons, + results=results, + ) with concurrent.futures.ProcessPoolExecutor() as executor: for _ in executor.map(part_func, points["features"], chunksize=chunk_size): diff --git a/turfpy/meta.py b/turfpy/meta.py index 6ae1133..0aef4f2 100644 --- a/turfpy/meta.py +++ b/turfpy/meta.py @@ -10,7 +10,11 @@ def geom_reduce(geojson, initial_value_param): previous_value = initial_value_param def callback_geom_each( - current_geometry, feature_index, feature_properties, feature_bbox, feature_id, + current_geometry, + feature_index, + feature_properties, + feature_bbox, + feature_id, ): nonlocal initial_value nonlocal previous_value @@ -77,7 +81,11 @@ def geom_each(geojson, callback): if not geometry: if not callback( - None, feature_index, feature_properties, feature_b_box, feature_id, + None, + feature_index, + feature_properties, + feature_b_box, + feature_id, ): return False continue @@ -297,7 +305,9 @@ def coord_each(geojson, callback, excludeWrapCoord=None): elif geom_type == "GeometryCollection": for j in range(0, len(geometry["geometries"])): if not coord_each( - geometry["geometries"][j], callback, excludeWrapCoord, + geometry["geometries"][j], + callback, + excludeWrapCoord, ): return False else: diff --git a/turfpy/transformation.py b/turfpy/transformation.py index 2384edd..f04ca2e 100644 --- a/turfpy/transformation.py +++ b/turfpy/transformation.py @@ -678,7 +678,10 @@ def _callback_coord_each( def transform_scale( - features, factor: float, origin: Union[str, list] = "centroid", mutate: bool = False, + features, + factor: float, + origin: Union[str, list] = "centroid", + mutate: bool = False, ): """ Scale a GeoJSON from a given