Embed Size (px)
DESCRIPTION
My EuroDjangoCon talk about how EveryBlock has used Mapnik and GeoDjango to create our own maps.
Citation preview
The Web map stack on Django
Paul Smithhttp://www.pauladamsmith.com/ @paulsmith
EveryBlockEuroDjangoCon ‘09
Data types
11 metros
● Boston
● Charlotte
● Chicago
● Los Angeles
● Miami
● New York
● Philadelphia
● San Francisco
● San Jose
● Seattle
● Washington, DC
… and growing
Open sourceThis summer
Why?
Control design
Prioritize visualizations
The Web map stack
The Web map stack
The Web map stack
The Web map stack
The Web map stack
GeoDjango + Mapnikexample app
“Your Political Footprint”
Mapnik overview
# models.py
from django.contrib.gis.db import models
class CongressionalDistrict(models.Model): state = models.ForeignKey(State) name = models.CharField(max_length=32) # ex. 1st, 25th, at-large number = models.IntegerField() # 0 if at-large district = models.MultiPolygonField(srid=4326) objects = models.GeoManager()
def __unicode__(self): return '%s %s' % (self.state.name, self.name)
class Footprint(models.Model): location = models.CharField(max_length=200) point = models.PointField(srid=4326) cong_dist = models.ForeignKey(CongressionalDistrict) objects = models.GeoManager()
def __unicode__(self): return '%s in %s' % (self.location, self.cong_dist)
GET /tile/?bbox=-112.5,22.5,-90,45
# urls.py
from django.conf import settingsfrom django.conf.urls.defaults import *from edc_demo.footprint import views
urlpatterns = patterns('', (r'^footprint/', views.political_footprint), (r'^tile/', views.map_tile))
# views.py
from mapnik import *from django.http import HttpResponse, Http404from django.conf import settingsfrom edc_demo.footprint.models import CongressionalDistrict
TILE_WIDTH = TILE_HEIGHT = 256TILE_MIMETYPE = 'image/png'LIGHT_GREY = '#C0CCC4'PGIS_DB_CONN = dict( host=settings.DATABASE_HOST, dbname=settings.DATABASE_NAME, user=settings.DATABASE_USER, password=settings.DATABASE_PASSWORD)
def map_tile(request): if request.GET.has_key('bbox'): bbox = [float(x) for x in request.GET['bbox'].split(',')] tile = Map(TILE_WIDTH, TILE_HEIGHT) rule = Rule() rule.symbols.append(LineSymbolizer(Color(LIGHT_GREY), 1.0)) style = Style() style.rules.append(rule) tile.append_style('cong_dist', style) layer = Layer('cong_dists') db_table = CongressionalDistrict._meta.db_table layer.datasource = PostGIS(table=db_table, **PGIS_DB_CONN) layer.styles.append('cong_dist') tile.layers.append(layer) tile.zoom_to_box(Envelope(*bbox)) img = Image(tile.width, tile.height) render(tile, img) img_bytes = img.tostring(TILE_MIMETYPE.split('/')[1]) return HttpResponse(img_bytes, mimetype=TILE_MIMETYPE) else: raise Http404()
# views.py cont'd
from django.shortcuts import render_to_responsefrom edc_demo.footprint.geocoder import geocode
def political_footprint(request): context = {} if request.GET.has_key('location'): point = geocode(request.GET['location']) cd = CongressionalDistrict.objects.get(district__contains=point) footprint = Footprint.objects.create( location = request.GET['location'], point = point, cong_dist = cd ) context['footprint'] = footprint context['cd_bbox'] = cong_dist.district.extent return render_to_response('footprint.html', context)
// footprint.html<script type="text/javascript">var map;var TileLayerClass = OpenLayers.Class(OpenLayers.Layer.TMS, { initialize: function(footprint_id) {
var name = "tiles";var url = "http://127.0.0.1:8000/tile/";
var args = [];args.push(name, url, {}, {});
OpenLayers.Layer.Grid.prototype.initialize.apply(this, args);this.footprint_id = footprint_id;
},
getURL: function(bounds) {var url = this.url + "?bbox=" + bounds.toBBOX();if (this.footprint_id)
url += "&fp_id=" + this.footprint_id; return url; }});function onload() { var options = {
minScale: 19660800,numZoomLevels: 14,units: "degrees"
}; map = new OpenLayers.Map("map"); {% if not footprint %}
var bbox = new OpenLayers.Bounds(-126.298828, 17.578125, -64.775391, 57.128906); var tileLayer = new TileLayerClass(); {% else %} var bbox = new OpenLayers.Bounds({{ cd_bbox|join:", " }}); var tileLayer = new TileLayerClass({{ footprint.id }}); {% endif %} map.addLayer(tileLayer); map.zoomToExtent(bbox);}
# views.py
from edc_demo.footprint.models import Footprint
def map_tile(request): if request.GET.has_key('bbox'): bbox = [float(x) for x in request.GET['bbox'].split(',')] tile = Map(TILE_WIDTH, TILE_HEIGHT) rule = Rule() rule.symbols.append(LineSymbolizer(Color(LIGHT_GREY), 1.0)) style = Style() style.rules.append(rule) if request.GET.has_key('fp_id'): footprint = Footprint.objects.get(pk=request.GET['fp_id']) rule = Rule() rule.symbols.append(LineSymbolizer(Color(GREEN), 1.0)) rule.symbols.append(PolygonSymbolizer(Color(LIGHT_GREEN))) rule.filter = Filter('[id] = ' + str(footprint.cong_dist.id)) style.rules.append(rule) tile.append_style('cong_dist', style) layer = Layer('cong_dists') db_table = CongressionalDistrict._meta.db_table layer.datasource = PostGIS(table=db_table, **PGIS_DB_CONN) layer.styles.append('cong_dist') tile.layers.append(layer) if request.GET.has_key('fp_id'): add_footprint_layer(tile, footprint) tile.zoom_to_box(Envelope(*bbox)) img = Image(tile.width, tile.height) render(tile, img) img_bytes = img.tostring(TILE_MIMETYPE.split('/')[1]) return HttpResponse(img_bytes, mimetype=TILE_MIMETYPE) else: raise Http404()
# views.py cont'd
def add_footprint_layer(tile, footprint): rule = Rule() rule.symbols.append(
PointSymbolizer(os.path.join(settings.STATIC_MEDIA_DIR, 'img', 'footprint.png'),'png', 46, 46)
) rule.filter = Filter('[id] = ' + str(footprint.id)) style = Style() style.rules.append(rule) tile.append_style('footprint', style) layer = Layer('footprint') layer.datasource = PostGIS(table=Footprint._meta.db_table, **PGIS_DB_CONN) layer.styles.append('footprint') tile.layers.append(layer)
Serving tiles
Zoom levels
Tile example
z: 5, x: 2384, y: 1352
TileCache
pro
● Cache population integrated with request/response cycle
● Flexible storage
con
● Python overhead (rendering, serving)
Pre-render + custom nginx mod
pro
● Fast responses
● Parallelizable, offline rendering
con
● Render everything in advance
● C module inflexibility (esp. storage backends)
Tile rendering
for each zoom level z:
for each column x:
for each row y:
render tile (x, y, z)
Tile rendering
for each zoom level z:
for each column x:
for each row y:
render tile (x, y, z)
Tile rendering
for each zoom level z:
for each column x:
for each row y:
render tile (x, y, z)
Tile rendering
for each zoom level z:
for each column x:
for each row y:
render tile (x, y, z)
# nginx.conf
server { server_name tile.example.com root /var/www/maptiles; expires max; location ~* ^/[^/]+/\w+/\d+/\d+,\d+\.(jpg|gif|png)$ { tilecache; }}
// ngx_tilecache_mod.c
/* * This struct holds the attributes that uniquely identify a map tile. */typedef struct { u_char *version; u_char *name; int x; int y; int z; u_char *ext;} tilecache_tile_t;
/* * The following regex pattern matches the request URI for a tile and * creates capture groups for the tile attributes. Example request URI: * * /1.0/main/8/654,23.png * * would map to the following attributes: * * version: 1.0 * name: main * z: 8 * x: 654 * y: 23 * extension: png */static ngx_str_t tile_request_pat = ngx_string("^/([^/]+)/([^/]+)/([0-9]+)/([0-9]+),([0-9]+)\\.([a-z]+)$");
// ngx_tilecache_mod.c
u_char *get_disk_key(u_char *s, u_char *name, int x, int y, int z, u_char *ext){ u_int a, b, c, d, e, f;
a = x / 100000; b = (x / 1000) % 1000; c = x % 1000; d = y / 100000; e = (y / 1000) % 1000; f = y % 1000;
return ngx_sprintf(s, "/%s/%02d/%03d/%03d/%03d/%03d/%03d/%03d.%s", name, z, a, b, c, d, e, f, ext);}
static ngx_int_tngx_tilecache_handler(ngx_http_request_t *r){ // ... snip ... sub_uri.data = ngx_pcalloc(r->pool, len + 1); if (sub_uri.data == NULL) { return NGX_ERROR; }
get_disk_key(sub_uri.data, tile->name, tile->x, tile->y, tile->z, tile->ext); sub_uri.len = ngx_strlen(sub_uri.data);
return ngx_http_internal_redirect(r, &sub_uri, &r->args);}
Custom tile cache
technique
● Far-future expiry header expires max;
responsibility
● Tile versions for cache invalidation
// everyblock.js
eb.TileLayer = OpenLayers.Class(OpenLayers.Layer.TMS, { version: null, // see eb.TILE_VERSION layername: null, // lower-cased: "main", "locator" type: null, // i.e., mime-type extension: "png", "jpg", "gif"
initialize: function(name, url, options) { var args = []; args.push(name, url, {}, options); OpenLayers.Layer.TMS.prototype.initialize.apply(this, args); },
// Returns an object with the x, y, and z of a tile for a given bounds getCoordinate: function(bounds) { bounds = this.adjustBounds(bounds);
var res = this.map.getResolution();var x = Math.round((bounds.left - this.tileOrigin.lon) / (res * this.tileSize.w));var y = Math.round((bounds.bottom - this.tileOrigin.lat) / (res * this.tileSize.h));var z = this.map.getZoom();return {x: x, y: y, z: z};
},
getPath: function(x, y, z) { return this.version + "/" + this.layername + "/" + z + "/" + x + "," + y + "." +
this.type; },
getURL: function(bounds) {var coord = this.getCoordinate(bounds);
var path = this.getPath(coord.x, coord.y, coord.z); var url = this.url; if (url instanceof Array) url = this.selectUrl(path, url); return url + path; },
CLASS_NAME: "eb.TileLayer"});
Clustering
# cluster.py
import mathfrom everyblock.maps.clustering.models import Bunch
def euclidean_distance(a, b): return math.hypot(a[0] - b[0], a[1] - b[1])
def buffer_cluster(objects, radius, dist_fn=euclidean_distance): bunches = [] buffer_ = radius for key, point in objects.iteritems(): bunched = False for bunch in bunches: if dist_fn(point, bunch.center) <= buffer_: bunch.add_obj(key, point) bunched = True break if not bunched: bunches.append(Bunch(key, point)) return bunches
# bunch.py
class Bunch(object): def __init__(self, obj, point): self.objects = [] self.points = [] self.center = (0, 0) self.add_obj(obj, point)
def add_obj(self, obj, point): self.objects.append(obj) self.points.append(point) self.update_center(point)
def update_center(self, point): xs = [p[0] for p in self.points] ys = [p[1] for p in self.points] self.center = (sum(xs) * 1.0 / len(self.objects), sum(ys) * 1.0 / len(self.objects))
# cluster_scale.py
from everyblock.maps import utilsfrom everyblock.maps.clustering import cluster
def cluster_by_scale(objs, radius, scale, extent=(-180, -90, 180, 90)): resolution = utils.get_resolution(scale) # Translate from lng/lat into coordinate system of the display. objs = dict([(k, utils.px_from_lnglat(v, resolution, extent)) for k, v in objs.iteritems()]) bunches = [] for bunch in cluster.buffer_cluster(objs, radius): # Translate back into lng/lat. bunch.center = utils.lnglat_from_px(bunch.center, resolution, extent) bunches.append(bunch) return bunches
Sneak peek
Sneak peek
Sneak peek
Thank you
http://www.pauladamsmith.com/@paulsmith
Further exploration:“How to Lie with Maps”
Mark Monmonier
