Merge pull request #2 from CartoDB/2.3.x.twkb

2.3.x.twkb

plugins/input/postgis/README.md

@@ -0,0 +1,39 @@
+# PostGIS Driver Parameters
+
+* `table`: Either the name of a table to map from or a SQL query to attempt to map (required)
+* `geometry_table`: In the case of a SQL query in the `table` parameter, the table in the query that is the source of the geometry
+* `geometry_field`: The name of the column in the table or SQL query that will return the geometry
+* `key_field`: The name of the column in the table or SQL query that contains a unique key
+* `autodetect_key_field`: Try and figure out the keyfield from metadata tables. (default: false)
+* `cursor_size`: If non-zero, the maximum number of rows to fetch while processing a query. For memory constrained situations, this limits the amount of data held in memory during query processing. (default: 0)
+* `row_limit`: If non-zero, apply a this `LIMIT` to the query, effectively placing a maximum on the number of features to be mapped. (default: 0)
+* `srid`: If non-zero, the EPSG SRID to use for the geometries coming from this query, notwithstanding the SRID that is actually on them. (default: 0)
+* `host`, `port`, `dbname`, `user`, `password`: PostgreSQL database connection parameters (required)
+* `connect_timeout`: Database connection timeout. (default: 4)
+* `pool_max_size`: How big to make the connection pool. (default: 10)
+* `persist_connection`: Keep connections open between requests? (default: true)
+* `estimate_extent`: Try and estimate extent of data? (default: false)
+* `extent_from_subquery`: If trying to estimate extent, and SQL is subquery, estimate on that? (default: false)
+* `max_async_connection`: If > 1, turn on the asyncronous connection feature. (default: 1)
+* `simplify_geometries`: Use PostGIS ST_Simplify call on the geometry to make input data smaller. (default: false)
+* `simplify_dp_ratio`: Use a PostGIS simplification factor of this proportion of the ground size of a pixel. For example, if pixels are 40m square for this rendering, and the ratio is set to 1/20, then `ST_Simplify(geom, 2)` would be used. (default: 1/20)
+
+## 2.3.x.twkb
+
+* `simplify_snap_ratio`: Use PostGIS `ST_SnapToGrid` call to make the input data smaller. This is applied to geometry **before** any `ST_Simplify` is called, so should use a tolerance smaller than the simplify tolerance. Tolerance is expressed as a proportion of a pixel, as with `simplify_dp_ratio`. (default: 1/40)
+
+These features all require PostGIS 2.2+, as they use features that do not appear in earlier PostGIS versions.
+
+* `twkb_encoding`: Use TWKB to encode geometries for transport from the database to the renderer instead of standard WKB? (default: false)
+* `simplify_dp_preserve`: Set the `preserve` option in `ST_Simplify` when in `simplify_geometries` mode? This will ensure that features that get simplified down to nothing aren't dropped but are retained in point-like form. Useful for rendering to avoid gaps where small feature "disappear". (default: false)
+* `simplify_clip_resolution`: In non-zero, sets the map scale at which geometries start getting clipped to the rendering window. (default: 0.0)
+
+## SQL Tokens
+
+SQL tokens are strings that are replaced in the input SQL before the SQL is sent to the database for execution. This is information that Mapnik knows as the result of the request, and can be useful in forming complex SQL queries.
+
+* `!bbox!`
+* `!scale_denominator!`
+* `!pixel_width!`
+* `!pixel_height!`
+

plugins/input/postgis/build.py

@@ -31,6 +31,7 @@
   """
   %(PLUGIN_NAME)s_datasource.cpp
   %(PLUGIN_NAME)s_featureset.cpp
+  %(PLUGIN_NAME)s_utils.cpp
   """ % locals()
 )
 

plugins/input/postgis/postgis_datasource.cpp

@@ -23,6 +23,7 @@
 #include "connection_manager.hpp"
 #include "postgis_datasource.hpp"
 #include "postgis_featureset.hpp"
+#include "postgis_utils.hpp"
 #include "asyncresultset.hpp"
 
 
@@ -85,6 +86,15 @@ postgis_datasource::postgis_datasource(parameters const& params)
       extent_from_subquery_(*params.get<mapnik::boolean>("extent_from_subquery", false)),
       max_async_connections_(*params_.get<int>("max_async_connection", 1)),
       asynchronous_request_(false),
+      twkb_encoding_(false),
+      simplify_snap_ratio_(*params_.get<value_double>("simplify_snap_ratio", 1.0/40.0)),
+      // 1/20 of pixel seems to be a good compromise to avoid
+      // drop of collapsed polygons.
+      // See https://github.com/mapnik/mapnik/issues/1639
+      // See http://trac.osgeo.org/postgis/ticket/2093
+      simplify_dp_ratio_(*params_.get<value_double>("simplify_dp_ratio", 1.0/20.0)),
+      simplify_dp_preserve_(false),
+      simplify_clip_resolution_(*params_.get<value_double>("simplify_clip_resolution", 0.0)),
       // params below are for testing purposes only and may be removed at any time
       intersect_min_scale_(*params.get<int>("intersect_min_scale", 0)),
       intersect_max_scale_(*params.get<int>("intersect_max_scale", 0))
@@ -123,6 +133,11 @@ postgis_datasource::postgis_datasource(parameters const& params)
     estimate_extent_ = estimate_extent && *estimate_extent;
     boost::optional<mapnik::boolean> simplify_opt = params.get<mapnik::boolean>("simplify_geometries", false);
     simplify_geometries_ = simplify_opt && *simplify_opt;
+    boost::optional<mapnik::boolean> twkb_opt = params.get<mapnik::boolean>("twkb_encoding", false);
+    twkb_encoding_ = twkb_opt && *twkb_opt;
+
+    boost::optional<mapnik::boolean> simplify_preserve_opt = params.get<mapnik::boolean>("simplify_dp_preserve", false);
+    simplify_dp_preserve_ = simplify_preserve_opt && *simplify_preserve_opt;
 
     ConnectionManager::instance().registerPool(creator_, *initial_size, pool_max_size_);
     CnxPool_ptr pool = ConnectionManager::instance().getPool(creator_.id());
@@ -162,7 +177,7 @@ postgis_datasource::postgis_datasource(parameters const& params)
             if (geometryColumn_.empty() || srid_ == 0)
             {
 #ifdef MAPNIK_STATS
-                mapnik::progress_timer __stats2__(std::clog, "postgis_datasource::init(get_srid_and_geometry_column)");
+                mapnik::progress_timer __stats2__(std::clog, "postgis_datasource::init(get_srid_and_geometry_column) <start>");
 #endif
                 std::ostringstream s;
 
@@ -235,13 +250,16 @@ postgis_datasource::postgis_datasource(parameters const& params)
                     }
                     rs->close();
                 }
+#ifdef MAPNIK_STATS
+                mapnik::progress_timer __stats2__(std::clog, "postgis_datasource::init(get_srid_and_geometry_column) <end>");
+#endif
             }
 
             // detect primary key
             if (*autodetect_key_field && key_field_.empty())
             {
 #ifdef MAPNIK_STATS
-                mapnik::progress_timer __stats2__(std::clog, "postgis_datasource::bind(get_primary_key)");
+                mapnik::progress_timer __stats2__(std::clog, "postgis_datasource::bind(get_primary_key) <start>");
 #endif
 
                 std::ostringstream s;
@@ -305,6 +323,10 @@ postgis_datasource::postgis_datasource(parameters const& params)
                     }
                 }
                 rs_key->close();
+
+#ifdef MAPNIK_STATS
+                mapnik::progress_timer __stats2__(std::clog, "postgis_datasource::bind(get_primary_key) <end>");
+#endif
             }
 
             // if a globally unique key field/primary key is required
@@ -330,7 +352,7 @@ postgis_datasource::postgis_datasource(parameters const& params)
 
             // collect attribute desc
 #ifdef MAPNIK_STATS
-            mapnik::progress_timer __stats2__(std::clog, "postgis_datasource::bind(get_column_description)");
+            mapnik::progress_timer __stats2__(std::clog, "postgis_datasource::bind(get_column_description) <start>");
 #endif
 
             std::ostringstream s;
@@ -427,6 +449,9 @@ postgis_datasource::postgis_datasource(parameters const& params)
                     }
                 }
             }
+#ifdef MAPNIK_STATS
+            mapnik::progress_timer __stats2__(std::clog, "postgis_datasource::bind(get_column_description) <end>");
+#endif
 
             rs->close();
 
@@ -482,21 +507,18 @@ std::string postgis_datasource::sql_bbox(box2d<double> const& env) const
 {
     std::ostringstream b;
 
-    if (srid_ > 0)
-    {
-        b << "ST_SetSRID(";
-    }
-
-    b << "'BOX3D(";
+    b << "ST_MakeEnvelope(";
     b << std::setprecision(16);
-    b << env.minx() << " " << env.miny() << ",";
-    b << env.maxx() << " " << env.maxy() << ")'::box3d";
+    b << env.minx() << "," << env.miny() << ",";
+    b << env.maxx() << "," << env.maxy();
 
     if (srid_ > 0)
     {
-        b << ", " << srid_ << ")";
+        b << "," << srid_;
     }
 
+    b << ")";
+
     return b.str();
 }
 
@@ -673,7 +695,7 @@ featureset_ptr postgis_datasource::features_with_context(query const& q,processo
 {
 
 #ifdef MAPNIK_STATS
-    mapnik::progress_timer __stats__(std::clog, "postgis_datasource::features_with_context");
+    mapnik::progress_timer __stats__(std::clog, "postgis_datasource::features_with_context<start>");
 #endif
 
 
@@ -733,24 +755,78 @@ featureset_ptr postgis_datasource::features_with_context(query const& q,processo
 
         const double px_gw = 1.0 / boost::get<0>(q.resolution());
         const double px_gh = 1.0 / boost::get<1>(q.resolution());
+        const double px_sz = std::min(px_gw, px_gh);
 
-        s << "SELECT ST_AsBinary(";
+        if (twkb_encoding_) {
+            s << "SELECT ST_AsTWKB(";
+        }
+        else {
+            s << "SELECT ST_AsBinary(";
+        }
 
         if (simplify_geometries_) {
-          s << "ST_Simplify(";
+            s << "ST_Simplify(";
+        }
+
+        if (simplify_clip_resolution_ > 0.0 && simplify_clip_resolution_ > px_sz) {
+            s << "ST_ClipByBox2D(";
         }
 
+        if (simplify_geometries_) {
+            s<< "ST_SnapToGrid(";
+        }
+
+        // Geometry column!
         s << "\"" << geometryColumn_ << "\"";
 
+        // ! ST_SnapToGrid()
         if (simplify_geometries_) {
           // 1/20 of pixel seems to be a good compromise to avoid
           // drop of collapsed polygons.
           // See https://github.com/mapnik/mapnik/issues/1639
-          const double tolerance = std::min(px_gw, px_gh) / 20.0;
-          s << ", " << tolerance << ")";
+          // See http://trac.osgeo.org/postgis/ticket/2093
+          const double grid_tolerance = px_sz * simplify_snap_ratio_;
+          s << ", " << grid_tolerance << ")";
         }
 
-        s << ") AS geom";
+        // ! ST_ClipByBox2D()
+        if (simplify_clip_resolution_ > 0.0 && simplify_clip_resolution_ > px_sz) {
+            s << "," << sql_bbox(box) << ")";
+        }
+
+        // ! ST_Simplify()
+        if (simplify_geometries_) {
+          const double tolerance = px_sz * simplify_dp_ratio_;
+          s << ", " << tolerance;
+          // Add parameter to ST_Simplify to keep collapsed geometries
+          if (simplify_dp_preserve_) {
+            s << ", true";
+          }
+          s << ")";
+        }
+
+        // ! ST_TWKB()
+        if ( twkb_encoding_ ) {
+            // Depending on where resolution falls relative to rounding levels,
+            // we get a rounding to either 1 pixel down to 1/10 of a pixel.
+            // (When rouding levels jump by factors of 10, hard to choose a "perfect" 
+            // rounding level)
+            const double tolerance = px_sz;
+            // Figure out number of decimals of rounding that implies
+            if ( tolerance > 0 ) {
+                const int i = -1 * lround(log10(tolerance) + 0.5) + 1;
+                // Write the SQL
+                s << "," << i << ") AS geom";
+            }
+            // Hopefully we're never fed a negative tolerance...
+            else {
+                s << ") AS geom";
+            }
+        }
+        // ! ST_AsBinary()
+        else {
+            s << ") AS geom";
+        }
 
         mapnik::context_ptr ctx = boost::make_shared<mapnik::context_type>();
         std::set<std::string> const& props = q.property_names();
@@ -790,10 +866,18 @@ featureset_ptr postgis_datasource::features_with_context(query const& q,processo
         }
 
         boost::shared_ptr<IResultSet> rs = get_resultset(conn, s.str(), pool, proc_ctx);
-        return boost::make_shared<postgis_featureset>(rs, ctx, desc_.get_encoding(), !key_field_.empty());
+
+#ifdef MAPNIK_STATS
+        mapnik::progress_timer __stats__(std::clog, "postgis_datasource::features_with_context<end>");
+#endif
+        return boost::make_shared<postgis_featureset>(rs, ctx, desc_.get_encoding(), !key_field_.empty(), twkb_encoding_);
 
     }
 
+#ifdef MAPNIK_STATS
+    mapnik::progress_timer __stats__(std::clog, "postgis_datasource::features_with_context<end>");
+#endif
+
     return featureset_ptr();
 }
 
@@ -873,7 +957,7 @@ featureset_ptr postgis_datasource::features_at_point(coord2d const& pt, double t
             }
 
             boost::shared_ptr<IResultSet> rs = get_resultset(conn, s.str(), pool);
-            return boost::make_shared<postgis_featureset>(rs, ctx, desc_.get_encoding(), !key_field_.empty());
+            return boost::make_shared<postgis_featureset>(rs, ctx, desc_.get_encoding(), !key_field_.empty(), twkb_encoding_);
         }
     }
 

plugins/input/postgis/postgis_datasource.hpp

@@ -58,6 +58,7 @@ using mapnik::feature_ptr;
 using mapnik::query;
 using mapnik::parameters;
 using mapnik::coord2d;
+using mapnik::value_double;
 
 typedef boost::shared_ptr< ConnectionManager::PoolType> CnxPool_ptr;
 
@@ -113,6 +114,11 @@ class postgis_datasource : public datasource
     bool estimate_extent_;
     int max_async_connections_;
     bool asynchronous_request_;
+    bool twkb_encoding_;
+    mapnik::value_double simplify_snap_ratio_;
+    mapnik::value_double simplify_dp_ratio_;
+    bool simplify_dp_preserve_;
+    mapnik::value_double simplify_clip_resolution_;
     int intersect_min_scale_;
     int intersect_max_scale_;
 };

plugins/input/postgis/postgis_featureset.cpp

@@ -21,6 +21,7 @@
  *****************************************************************************/
 
 #include "postgis_featureset.hpp"
+#include "postgis_utils.hpp"
 #include "resultset.hpp"
 #include "cursorresultset.hpp"
 
@@ -47,19 +48,22 @@
 using mapnik::geometry_type;
 using mapnik::byte;
 using mapnik::geometry_utils;
+using mapnik::postgis_utils;
 using mapnik::feature_factory;
 using mapnik::context_ptr;
 
 postgis_featureset::postgis_featureset(boost::shared_ptr<IResultSet> const& rs,
                                        context_ptr const& ctx,
                                        std::string const& encoding,
-                                       bool key_field)
+                                       bool key_field,
+                                       bool twkb_encoding)
     : rs_(rs),
       ctx_(ctx),
       tr_(new transcoder(encoding)),
       totalGeomSize_(0),
       feature_id_(1),
-      key_field_(key_field)
+      key_field_(key_field),
+      twkb_encoding_(twkb_encoding)
 {
 }
 
@@ -128,8 +132,14 @@ feature_ptr postgis_featureset::next()
         int size = rs_->getFieldLength(0);
         const char *data = rs_->getValue(0);
 
-        if (!geometry_utils::from_wkb(feature->paths(), data, size))
-            continue;
+        if ( twkb_encoding_ ) {
+            if (!postgis_utils::from_twkb(feature->paths(), data, size))
+                continue;
+        }
+        else {            
+            if (!geometry_utils::from_wkb(feature->paths(), data, size))
+                continue;
+        }
 
         totalGeomSize_ += size;
         unsigned num_attrs = ctx_->size() + 1;

plugins/input/postgis/postgis_featureset.hpp

@@ -47,7 +47,8 @@ class postgis_featureset : public mapnik::Featureset
     postgis_featureset(boost::shared_ptr<IResultSet> const& rs,
                        context_ptr const& ctx,
                        std::string const& encoding,
-                       bool key_field = false);
+                       bool key_field = false,
+                       bool twkb_encoding = false);
     feature_ptr next();
     ~postgis_featureset();
 
@@ -58,6 +59,7 @@ class postgis_featureset : public mapnik::Featureset
     unsigned totalGeomSize_;
     mapnik::value_integer feature_id_;
     bool key_field_;
+    bool twkb_encoding_;
 };
 
 #endif // POSTGIS_FEATURESET_HPP