This is an automated email from the git hooks/post-receive script. sebastic pushed a commit to branch master in repository tilestache.
commit 35c6dd9ae5497ecda62dc2a501afdc5ce364b794 Author: Bas Couwenberg <[email protected]> Date: Wed Jan 25 21:23:16 2017 +0100 Imported Upstream version 1.51.5 --- .travis.yml | 3 +- CHANGELOG | 5 +++ README.md | 2 +- TileStache/Caches.py | 14 +++---- TileStache/Config.py | 70 +++++++++------------------------ TileStache/Core.py | 67 +++++++++++++++++++++++++++---- TileStache/Geography.py | 7 ++-- TileStache/Goodies/VecTiles/client.py | 20 ++++++++-- TileStache/Goodies/VecTiles/geojson.py | 25 ++++++------ TileStache/Goodies/VecTiles/mvt.py | 6 ++- TileStache/Goodies/VecTiles/server.py | 19 +++++++-- TileStache/Goodies/VecTiles/topojson.py | 6 +-- TileStache/Goodies/VecTiles/wkb.py | 6 ++- TileStache/Mapnik.py | 5 ++- TileStache/Memcache.py | 9 ++++- TileStache/Pixels.py | 6 ++- TileStache/Providers.py | 16 ++++++-- TileStache/VERSION | 2 +- TileStache/Vector/Arc.py | 2 +- TileStache/Vector/__init__.py | 17 +++++--- TileStache/__init__.py | 46 ++++++++++++++++------ requirements.txt | 3 +- setup.py | 2 +- tests/cache_tests.py | 17 +++++--- tests/provider_tests.py | 11 +++--- tests/servers/dummy-response-server.py | 5 ++- tests/utils.py | 15 ++++--- tests/vectiles_tests.py | 47 +++++++++++----------- tests/vector_tests.py | 10 ++--- 29 files changed, 287 insertions(+), 176 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4300763..75450f2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,12 +8,13 @@ dist: trusty python: - "2.7" + - "3.4" addons: postgresql: "9.4" apt: packages: - - postgresql-9.4-postgis-2.2 + - postgresql-9.4-postgis-2.3 - python-dev - libgdal1-dev - python-werkzeug diff --git a/CHANGELOG b/CHANGELOG index 360a1e7..236e613 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,8 @@ +2017-01-25: 1.51.5 +- Fix to be compliant with mapnik 3 grid renderer +- Fix of preview protocol +- Moving towards python 3 compliance + 2016-12-21: 1.51.4 - Support for GDAL 2.x types OFTInteger64 and OFTInteger64 on Vector Tiles diff --git a/README.md b/README.md index d23aa28..8c893e5 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ _a stylish alternative for caching your map tiles_ -[](https://travis-ci.org/TileStache/TileStache) +[](https://travis-ci.org/TileStache/TileStache) **TileStache** is a Python-based server application that can serve up map tiles based on rendered geographic data. You might be familiar with [TileCache](http://tilecache.org), diff --git a/TileStache/Caches.py b/TileStache/Caches.py index ffd113b..eae4e5b 100644 --- a/TileStache/Caches.py +++ b/TileStache/Caches.py @@ -183,7 +183,7 @@ class Disk: "http://example.com/tilestache.cfg";, the path *must* be an unambiguous filesystem path, e.g. "file:///tmp/cache" """ - def __init__(self, path, umask=0022, dirs='safe', gzip='txt text json xml'.split()): + def __init__(self, path, umask=0o022, dirs='safe', gzip='txt text json xml'.split()): self.cachepath = path self.umask = int(umask) self.dirs = dirs @@ -271,9 +271,9 @@ class Disk: # Oh - no they didn't. pass - os.makedirs(lockpath, 0777&~self.umask) + os.makedirs(lockpath, 0o777&~self.umask) break - except OSError, e: + except OSError as e: if e.errno != 17: raise time.sleep(.2) @@ -300,7 +300,7 @@ class Disk: try: os.remove(fullpath) - except OSError, e: + except OSError as e: # errno=2 means that the file does not exist, which is fine if e.errno != 2: raise @@ -332,8 +332,8 @@ class Disk: try: umask_old = os.umask(self.umask) - os.makedirs(dirname(fullpath), 0777&~self.umask) - except OSError, e: + os.makedirs(dirname(fullpath), 0o777&~self.umask) + except OSError as e: if e.errno != 17: raise finally: @@ -359,7 +359,7 @@ class Disk: os.unlink(fullpath) os.rename(tmp_path, fullpath) - os.chmod(fullpath, 0666&~self.umask) + os.chmod(fullpath, 0o666&~self.umask) class Multi: """ Caches tiles to multiple, ordered caches. diff --git a/TileStache/Config.py b/TileStache/Config.py index 18c91ef..2dfa6f6 100644 --- a/TileStache/Config.py +++ b/TileStache/Config.py @@ -60,11 +60,19 @@ documentation for TileStache.Providers, TileStache.Core, and TileStache.Geograph import sys import logging -from sys import stderr, modules +from sys import modules from os.path import realpath, join as pathjoin -from urlparse import urljoin, urlparse +try: + from urllib.parse import urljoin, urlparse +except ImportError: + # Python 2 + from urlparse import urljoin, urlparse from mimetypes import guess_type -from urllib import urlopen +try: + from urllib.request import urlopen +except ImportError: + # Python 2 + from urllib import urlopen from json import dumps try: @@ -75,11 +83,11 @@ except ImportError: from ModestMaps.Geo import Location from ModestMaps.Core import Coordinate -import Core -import Caches -import Providers -import Geography -import PixelEffects +from . import Core +from . import Caches +from . import Providers +from . import Geography +from . import PixelEffects class Configuration: """ A complete site configuration, with a collection of Layer objects. @@ -315,7 +323,7 @@ def _parseConfigCache(cache_dict, dirpath): raise Exception('Unknown cache: %s' % cache_dict['name']) elif 'class' in cache_dict: - _class = loadClassPath(cache_dict['class']) + _class = Core.loadClassPath(cache_dict['class']) kwargs = cache_dict.get('kwargs', {}) kwargs = dict( [(str(k), v) for (k, v) in kwargs.items()] ) @@ -449,7 +457,7 @@ def _parseConfigLayer(layer_dict, config, dirpath): provider_kwargs = _class.prepareKeywordArgs(provider_dict) elif 'class' in provider_dict: - _class = loadClassPath(provider_dict['class']) + _class = Core.loadClassPath(provider_dict['class']) provider_kwargs = provider_dict.get('kwargs', {}) provider_kwargs = dict( [(str(k), v) for (k, v) in provider_kwargs.items()] ) @@ -467,45 +475,3 @@ def _parseConfigLayer(layer_dict, config, dirpath): layer.pixel_effect = pixel_effect return layer - -def loadClassPath(classpath): - """ Load external class based on a path. - - Example classpath: "Module.Submodule:Classname". - - Equivalent soon-to-be-deprecated classpath: "Module.Submodule.Classname". - """ - if ':' in classpath: - # - # Just-added support for "foo:blah"-style classpaths. - # - modname, objname = classpath.split(':', 1) - - try: - __import__(modname) - module = modules[modname] - _class = eval(objname, module.__dict__) - - if _class is None: - raise Exception('eval(%(objname)s) in %(modname)s came up None' % locals()) - - except Exception, e: - raise Core.KnownUnknown('Tried to import %s, but: %s' % (classpath, e)) - - else: - # - # Support for "foo.blah"-style classpaths, TODO: deprecate this in v2. - # - classpath = classpath.split('.') - - try: - module = __import__('.'.join(classpath[:-1]), fromlist=str(classpath[-1])) - except ImportError, e: - raise Core.KnownUnknown('Tried to import %s, but: %s' % ('.'.join(classpath), e)) - - try: - _class = getattr(module, classpath[-1]) - except AttributeError, e: - raise Core.KnownUnknown('Tried to import %s, but: %s' % ('.'.join(classpath), e)) - - return _class diff --git a/TileStache/Core.py b/TileStache/Core.py index 2b60e6c..163da22 100644 --- a/TileStache/Core.py +++ b/TileStache/Core.py @@ -144,12 +144,21 @@ The preview can be accessed through a URL like /<layer name>/preview.html: """ import logging +from sys import modules from wsgiref.headers import Headers -from StringIO import StringIO -from urlparse import urljoin +try: + from io import BytesIO +except ImportError: + # Python 2 + from StringIO import StringIO as BytesIO +try: + from urllib.parse import urljoin +except ImportError: + # Python 2 + from urlparse import urljoin from time import time -from Pixels import load_palette, apply_palette, apply_palette256 +from .Pixels import load_palette, apply_palette, apply_palette256 try: from PIL import Image @@ -382,7 +391,7 @@ class Layer: # Start by checking for a tile in the cache. try: body = cache.read(self, coord, format) - except TheTileLeftANote, e: + except TheTileLeftANote as e: headers = e.headers status_code = e.status_code body = e.content @@ -417,12 +426,12 @@ class Layer: if body is None: # No one else wrote the tile, do it here. - buff = StringIO() + buff = BytesIO() try: tile = self.render(coord, format) save = True - except NoTileLeftBehind, e: + except NoTileLeftBehind as e: tile = e.tile save = False status_code = 404 @@ -445,7 +454,7 @@ class Layer: tile_from = 'layer.render()' - except TheTileLeftANote, e: + except TheTileLeftANote as e: headers = e.headers status_code = e.status_code body = e.content @@ -733,7 +742,7 @@ def _preview(layer): <html> <head> <title>TileStache Preview: %(layername)s</title> - <script src="http://cdn.rawgit.com/stamen/modestmaps-js/v1.0.0-beta1/modestmaps.min.js"; type="text/javascript"></script> + <script src="//cdn.rawgit.com/stamen/modestmaps-js/v1.0.0-beta1/modestmaps.min.js" type="text/javascript"></script> <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=0"> <style type="text/css"> html, body, #map { @@ -818,3 +827,45 @@ def _rummy(): '##Mh@M . ...:;;,:@A#@@@@@@@@@@@#@@@@@@#MMHAB@@@@#G#@@#: i@@ r@@#MMM#######@@@@#@@@@@@#####M#####@@', '#H3#@3. ,. ... :@@&@@@@@@@@@@@@@#@@#@@@MMBHGA@H&;:@@i :B@@@B .@@#MM####@@@##@@@#@@@@@#######M##M#@@@', 'M&AM5i;.,. ..,,rA@@MH@@@@@@@@@@@@@##@@@@@MMMBB#@h9hH#s;3######, .A@#MMM#####@@@@@##@@@#@@#####M#####M39B'] + +def loadClassPath(classpath): + """ Load external class based on a path. + + Example classpath: "Module.Submodule:Classname". + + Equivalent soon-to-be-deprecated classpath: "Module.Submodule.Classname". + """ + if ':' in classpath: + # + # Just-added support for "foo:blah"-style classpaths. + # + modname, objname = classpath.split(':', 1) + + try: + __import__(modname) + module = modules[modname] + _class = eval(objname, module.__dict__) + + if _class is None: + raise Exception('eval(%(objname)s) in %(modname)s came up None' % locals()) + + except Exception as e: + raise KnownUnknown('Tried to import %s, but: %s' % (classpath, e)) + + else: + # + # Support for "foo.blah"-style classpaths, TODO: deprecate this in v2. + # + classpath = classpath.split('.') + + try: + module = __import__('.'.join(classpath[:-1]), fromlist=str(classpath[-1])) + except ImportError as e: + raise KnownUnknown('Tried to import %s, but: %s' % ('.'.join(classpath), e)) + + try: + _class = getattr(module, classpath[-1]) + except AttributeError as e: + raise KnownUnknown('Tried to import %s, but: %s' % ('.'.join(classpath), e)) + + return _class diff --git a/TileStache/Geography.py b/TileStache/Geography.py index 62160b3..366ecb3 100644 --- a/TileStache/Geography.py +++ b/TileStache/Geography.py @@ -36,8 +36,7 @@ from ModestMaps.Core import Point, Coordinate from ModestMaps.Geo import deriveTransformation, MercatorProjection, LinearProjection, Location from math import log as _log, pi as _pi -import Core -import Config +from . import Core class SphericalMercator(MercatorProjection): """ Spherical mercator projection for most commonly-used web map tile scheme. @@ -143,6 +142,6 @@ def getProjectionByName(name): else: try: - return Config.loadClassPath(name) - except Exception, e: + return Core.loadClassPath(name) + except Exception as e: raise Core.KnownUnknown('Failed projection in configuration: "%s" - %s' % (name, e)) diff --git a/TileStache/Goodies/VecTiles/client.py b/TileStache/Goodies/VecTiles/client.py index 49e16ea..efde249 100644 --- a/TileStache/Goodies/VecTiles/client.py +++ b/TileStache/Goodies/VecTiles/client.py @@ -32,10 +32,22 @@ See also: ''' from math import pi, log as _log from threading import Thread, Lock as _Lock -from httplib import HTTPConnection +try: + from http.client import HTTPConnection +except ImportError: + # Python 2 + from httplib import HTTPConnection from itertools import product -from StringIO import StringIO -from urlparse import urlparse +try: + from io import StringIO +except ImportError: + # Python 2 + from StringIO import StringIO +try: + from urllib.parse import urlparse +except ImportError: + # Python 2 + from urlparse import urlparse from gzip import GzipFile import logging @@ -259,7 +271,7 @@ class Datasource (PythonDatasource): if self.sort: logging.debug('Sorting by %s %s' % (self.sort, 'descending' if self.reverse else 'ascending')) - key_func = lambda (wkb, props): props.get(self.sort, None) + key_func = lambda wkb_props: wkb_props[1].get(self.sort, None) features.sort(reverse=self.reverse, key=key_func) if len(features) == 0: diff --git a/TileStache/Goodies/VecTiles/geojson.py b/TileStache/Goodies/VecTiles/geojson.py index 6638388..6bbde69 100644 --- a/TileStache/Goodies/VecTiles/geojson.py +++ b/TileStache/Goodies/VecTiles/geojson.py @@ -34,7 +34,7 @@ def get_tiles(names, config, coord): if bad_mimes: raise KnownUnknown('%s.get_tiles encountered a non-JSON mime-type in %s sub-layer: "%s"' % ((__name__, ) + bad_mimes[0])) - geojsons = map(json.loads, bodies) + geojsons = [json.loads(body.decode('utf8')) for body in bodies] bad_types = [(name, topo['type']) for (topo, name) in zip(geojsons, names) if topo['type'] != 'FeatureCollection'] if bad_types: @@ -42,10 +42,11 @@ def get_tiles(names, config, coord): return geojsons -def mercator((x, y)): +def mercator(xy): ''' Project an (x, y) tuple to spherical mercator. ''' - x, y = pi * x/180, pi * y/180 + _x, _y = xy + x, y = pi * _x/180, pi * _y/180 y = log(tan(0.25 * pi + 0.5 * y)) return 6378137 * x, 6378137 * y @@ -98,14 +99,13 @@ def encode(file, features, zoom, is_clipped): for token in encoded: if charfloat_pat.match(token): # in python 2.7, we see a character followed by a float literal - file.write(token[0] + flt_fmt % float(token[1:])) - + piece = token[0] + flt_fmt % float(token[1:]) elif float_pat.match(token): # in python 2.6, we see a simple float literal - file.write(flt_fmt % float(token)) - + piece = flt_fmt % float(token) else: - file.write(token) + piece = token + file.write(piece.encode('utf8')) def merge(file, names, config, coord): ''' Retrieve a list of GeoJSON tile responses and merge them into one. @@ -122,11 +122,10 @@ def merge(file, names, config, coord): for token in encoded: if charfloat_pat.match(token): # in python 2.7, we see a character followed by a float literal - file.write(token[0] + flt_fmt % float(token[1:])) - + piece = token[0] + flt_fmt % float(token[1:]) elif float_pat.match(token): # in python 2.6, we see a simple float literal - file.write(flt_fmt % float(token)) - + piece = flt_fmt % float(token) else: - file.write(token) + piece = token + file.write(piece.encode('utf8')) diff --git a/TileStache/Goodies/VecTiles/mvt.py b/TileStache/Goodies/VecTiles/mvt.py index 73dc62c..88fc952 100644 --- a/TileStache/Goodies/VecTiles/mvt.py +++ b/TileStache/Goodies/VecTiles/mvt.py @@ -36,7 +36,11 @@ By default, encode() approximates the floating point precision of WKB geometry to 26 bits for a significant compression improvement and no visible impact on rendering at zoom 18 and lower. ''' -from StringIO import StringIO +try: + from io import StringIO +except ImportError: + # Python 2 + from StringIO import StringIO from zlib import decompress as _decompress, compress as _compress from struct import unpack as _unpack, pack as _pack import json diff --git a/TileStache/Goodies/VecTiles/server.py b/TileStache/Goodies/VecTiles/server.py index 9326239..afc5288 100644 --- a/TileStache/Goodies/VecTiles/server.py +++ b/TileStache/Goodies/VecTiles/server.py @@ -8,8 +8,16 @@ For a more general implementation, try the Vector provider: http://tilestache.org/doc/#vector-provider ''' from math import pi -from urlparse import urljoin, urlparse -from urllib import urlopen +try: + from urllib.parse import urljoin, urlparse +except ImportError: + # Python 2 + from urlparse import urljoin, urlparse +try: + from urllib.request import urlopen +except ImportError: + # Python 2 + from urllib import urlopen from os.path import exists try: @@ -17,7 +25,7 @@ try: from psycopg2 import connect from psycopg2.extensions import TransactionRollbackError -except ImportError, err: +except ImportError as err: # Still possible to build the documentation without psycopg2 def connect(*args, **kwargs): @@ -182,7 +190,10 @@ class Provider: if query not in self.columns: self.columns[query] = query_columns(self.dbinfo, self.srid, query, bounds) - + + if not self.columns[query]: + return EmptyResponse(bounds) + tolerance = self.simplify * tolerances[coord.zoom] if coord.zoom < self.simplify_until else None return Response(self.dbinfo, self.srid, query, self.columns[query], bounds, tolerance, coord.zoom, self.clip, coord, self.layer.name(), self.padding) diff --git a/TileStache/Goodies/VecTiles/topojson.py b/TileStache/Goodies/VecTiles/topojson.py index 04c775b..9bcd13b 100644 --- a/TileStache/Goodies/VecTiles/topojson.py +++ b/TileStache/Goodies/VecTiles/topojson.py @@ -23,7 +23,7 @@ def get_tiles(names, config, coord): if bad_mimes: raise KnownUnknown('%s.get_tiles encountered a non-JSON mime-type in %s sub-layer: "%s"' % ((__name__, ) + bad_mimes[0])) - topojsons = map(json.loads, bodies) + topojsons = [json.loads(body.decode('utf8')) for body in bodies] bad_types = [(name, topo['type']) for (topo, name) in zip(topojsons, names) if topo['type'] != 'Topology'] if bad_types: @@ -188,7 +188,7 @@ def encode(file, features, bounds, is_clipped): 'arcs': arcs } - json.dump(result, file, separators=(',', ':')) + file.write(json.dumps(result, separators=(',', ':')).encode('utf8')) def merge(file, names, config, coord): ''' Retrieve a list of TopoJSON tile responses and merge them into one. @@ -214,4 +214,4 @@ def merge(file, names, config, coord): for geometry in object['geometries']: update_arc_indexes(geometry, output['arcs'], input['arcs']) - json.dump(output, file, separators=(',', ':')) + file.write(json.dumps(output, separators=(',', ':')).encode('utf8')) diff --git a/TileStache/Goodies/VecTiles/wkb.py b/TileStache/Goodies/VecTiles/wkb.py index 51c5d97..0ec954d 100644 --- a/TileStache/Goodies/VecTiles/wkb.py +++ b/TileStache/Goodies/VecTiles/wkb.py @@ -14,7 +14,11 @@ See also: ''' from struct import unpack -from StringIO import StringIO +try: + from io import StringIO +except ImportError: + # Python 2 + from StringIO import StringIO # # wkbByteOrder diff --git a/TileStache/Mapnik.py b/TileStache/Mapnik.py index 8c14299..ef762b2 100644 --- a/TileStache/Mapnik.py +++ b/TileStache/Mapnik.py @@ -285,8 +285,9 @@ class GridProvider: for (index, fields) in self.layers: datasource = self.mapnik.layers[index].datasource fields = (type(fields) is list) and map(str, fields) or datasource.fields() - - grid = mapnik.render_grid(self.mapnik, index, resolution=self.scale, fields=fields) + grid = mapnik.Grid(width, height) + mapnik.render_layer(self.mapnik, grid, layer=index, fields=fields) + grid = grid.encode('utf', resolution=self.scale, features=True) for key in grid['data']: grid['data'][key][self.layer_id_key] = self.mapnik.layers[index].name diff --git a/TileStache/Memcache.py b/TileStache/Memcache.py index 43f5494..e1455a9 100644 --- a/TileStache/Memcache.py +++ b/TileStache/Memcache.py @@ -33,6 +33,7 @@ Memcache cache parameters: """ from __future__ import absolute_import from time import time as _time, sleep as _sleep +from base64 import b64encode, b64decode # We enabled absolute_import because case insensitive filesystems # cause this file to be loaded twice (the name of this file @@ -109,7 +110,10 @@ class Cache: value = mem.get(key) mem.disconnect_all() - return value + if value is None: + return None + + return b64decode(value.encode('ascii')) def save(self, body, layer, coord, format): """ Save a cached tile. @@ -117,5 +121,8 @@ class Cache: mem = Client(self.servers) key = tile_key(layer, coord, format, self.revision, self.key_prefix) + if body is not None: + body = b64encode(body).decode('ascii') + mem.set(key, body, layer.cache_lifespan or 0) mem.disconnect_all() diff --git a/TileStache/Pixels.py b/TileStache/Pixels.py index 399d0d3..dc652e7 100644 --- a/TileStache/Pixels.py +++ b/TileStache/Pixels.py @@ -23,7 +23,11 @@ in the lookup table. If the final byte is 0xFFFF, there is no transparency. """ from struct import unpack, pack from math import sqrt, ceil, log -from urllib import urlopen +try: + from urllib.request import urlopen +except ImportError: + # Python 2 + from urllib import urlopen from operator import add try: diff --git a/TileStache/Providers.py b/TileStache/Providers.py index 30139fa..ca669c4 100644 --- a/TileStache/Providers.py +++ b/TileStache/Providers.py @@ -73,9 +73,17 @@ For an example of a non-image provider, see TileStache.Vector.Provider. import os import logging -from StringIO import StringIO +try: + from io import BytesIO +except ImportError: + # Python 2 + from StringIO import StringIO as BytesIO from string import Template -import urllib2 +try: + import urllib.request as urllib2 +except ImportError: + # Python 2 + import urllib2 import urllib try: @@ -87,7 +95,7 @@ except ImportError: import ModestMaps from ModestMaps.Core import Point, Coordinate -import Geography +from . import Geography # This import should happen inside getProviderByName(), but when testing # on Mac OS X features are missing from output. Wierd-ass C libraries... @@ -140,7 +148,7 @@ class Verbatim: ''' Wrapper for PIL.Image that saves raw input bytes if modes and formats match. ''' def __init__(self, bytes): - self.buffer = StringIO(bytes) + self.buffer = BytesIO(bytes) self.format = None self._image = None diff --git a/TileStache/VERSION b/TileStache/VERSION index 858461d..9d343dd 100644 --- a/TileStache/VERSION +++ b/TileStache/VERSION @@ -1 +1 @@ -1.51.4 +1.51.5 diff --git a/TileStache/Vector/Arc.py b/TileStache/Vector/Arc.py index 958a62e..ae7364c 100644 --- a/TileStache/Vector/Arc.py +++ b/TileStache/Vector/Arc.py @@ -2,7 +2,7 @@ """ from operator import add -from TileStache.Core import KnownUnknown +from ..Core import KnownUnknown geometry_types = { 'Point': 'esriGeometryPoint', diff --git a/TileStache/Vector/__init__.py b/TileStache/Vector/__init__.py index d64681f..0a53348 100644 --- a/TileStache/Vector/__init__.py +++ b/TileStache/Vector/__init__.py @@ -154,7 +154,11 @@ you can save yourself a world of trouble by using this definition: """ from re import compile -from urlparse import urlparse, urljoin +try: + from urllib.parse import urljoin, urlparse +except ImportError: + # Python 2 + from urlparse import urljoin, urlparse try: from json import JSONEncoder, loads as json_loads @@ -163,9 +167,9 @@ except ImportError: from osgeo import ogr, osr -from TileStache.Core import KnownUnknown -from TileStache.Geography import getProjectionByName -from Arc import reserialize_to_arc, pyamf_classes +from ..Core import KnownUnknown +from ..Geography import getProjectionByName +from .Arc import reserialize_to_arc, pyamf_classes class VectorResponse: """ Wrapper class for Vector response that makes it behave like a PIL.Image object. @@ -220,9 +224,10 @@ class VectorResponse: for atom in encoded: if float_pat.match(atom): - out.write(('%%.%if' % self.precision) % float(atom)) + piece = ('%%.%if' % self.precision) % float(atom) else: - out.write(atom) + piece = atom + out.write(piece.encode('utf8')) elif format in ('GeoBSON', 'ArcBSON'): import bson diff --git a/TileStache/__init__.py b/TileStache/__init__.py index fb9374c..ef0a501 100644 --- a/TileStache/__init__.py +++ b/TileStache/__init__.py @@ -8,6 +8,7 @@ designers and cartographers. Documentation available at http://tilestache.org/doc/ """ +from __future__ import print_function import os.path __version__ = open(os.path.join(os.path.dirname(__file__), 'VERSION')).read().strip() @@ -19,16 +20,32 @@ try: from urlparse import parse_qs except ImportError: from cgi import parse_qs -from StringIO import StringIO +try: + from io import StringIO +except ImportError: + # Python 2 + from StringIO import StringIO from os.path import dirname, join as pathjoin, realpath from datetime import datetime, timedelta -from urlparse import urljoin, urlparse +try: + from urllib.parse import urljoin, urlparse +except ImportError: + # Python 2 + from urlparse import urljoin, urlparse from wsgiref.headers import Headers -from urllib import urlopen +try: + from urllib.request import urlopen +except ImportError: + # Python 2 + from urllib import urlopen from os import getcwd from time import time -import httplib +try: + import http.client as httplib +except ImportError: + # Python 2 + import httplib import logging try: @@ -43,8 +60,8 @@ from ModestMaps.Core import Coordinate # dictionary of configuration objects for requestLayer(). _previous_configs = {} -import Core -import Config +from . import Core +from . import Config # regular expression for PATH_INFO _pathinfo_pat = re.compile(r'^/?(?P<l>\w.+)/(?P<z>\d+)/(?P<x>-?\d+)/(?P<y>-?\d+)\.(?P<e>\w+)$') @@ -102,13 +119,18 @@ def parseConfig(configHandle): config_dict = configHandle dirpath = '.' else: - config_dict = json_load(urlopen(configHandle)) scheme, host, path, p, q, f = urlparse(configHandle) - + if scheme == '': scheme = 'file' path = realpath(path) + if scheme == 'file': + with open(path) as file: + config_dict = json_load(file) + else: + config_dict = json_load(urlopen(configHandle)) + dirpath = '%s://%s%s' % (scheme, host, dirname(path).rstrip('/') + '/') return Config.buildConfiguration(config_dict, dirpath) @@ -275,7 +297,7 @@ def requestHandler2(config_hint, path_info, query_string=None, script_name=''): headers.setdefault('Expires', expires.strftime('%a, %d %b %Y %H:%M:%S GMT')) headers.setdefault('Cache-Control', 'public, max-age=%d' % layer.max_cache_age) - except Core.KnownUnknown, e: + except Core.KnownUnknown as e: out = StringIO() print >> out, 'Known unknown!' @@ -351,7 +373,7 @@ class WSGITileServer: try: self.config = parseConfig(config) except: - print "Error loading Tilestache config:" + print("Error loading Tilestache config:") raise else: @@ -369,12 +391,12 @@ class WSGITileServer: if self.autoreload: # re-parse the config file on every request try: self.config = parseConfig(self.config_path) - except Exception, e: + except Exception as e: raise Core.KnownUnknown("Error loading Tilestache config file:\n%s" % str(e)) try: layer, coord, ext = splitPathInfo(environ['PATH_INFO']) - except Core.KnownUnknown, e: + except Core.KnownUnknown as e: return self._response(start_response, 400, str(e)) # diff --git a/requirements.txt b/requirements.txt index 370c8da..fe6f600 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,8 @@ -ModestMaps +ModestMaps==1.4.7 simplejson shapely pillow psycopg2 python-memcached mapbox-vector-tile==0.5.0 +Werkzeug==0.11.13 \ No newline at end of file diff --git a/setup.py b/setup.py index 385984f..8fe24aa 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ def is_installed(name): return False -requires = ['ModestMaps >=1.3.0','simplejson', 'Werkzeug', 'Pillow'] +requires = ['ModestMaps >=1.3.0','simplejson', 'Werkzeug == 0.11.13', 'Pillow'] setup(name='TileStache', diff --git a/tests/cache_tests.py b/tests/cache_tests.py index 66e2e68..fda08f9 100644 --- a/tests/cache_tests.py +++ b/tests/cache_tests.py @@ -1,5 +1,6 @@ import os from unittest import TestCase, skipIf +from base64 import b64decode import memcache from . import utils @@ -36,8 +37,10 @@ class CacheTests(TestCase): tile_mimetype, tile_content = utils.request(config_file_content, "memcache_osm", "png", 0, 0, 0) self.assertEqual(tile_mimetype, "image/png") - self.assertEqual(self.mc.get('/4/memcache_osm/0/0/0.PNG'), tile_content, - 'Contents of memcached and value returned from TileStache do not match') + memcache_content = b64decode(self.mc.get('/4/memcache_osm/0/0/0.PNG').encode('ascii')) + + self.assertEqual(memcache_content, tile_content, + 'Contents of memcached and value returned from TileStache should match') def test_memcache_keyprefix(self): '''Fetch tile and check the existence of key with prefix in memcached''' @@ -64,8 +67,10 @@ class CacheTests(TestCase): tile_mimetype, tile_content = utils.request(config_file_content, "memcache_osm", "png", 0, 0, 0) self.assertEqual(tile_mimetype, "image/png") - self.assertEqual(self.mc.get('cool_prefix/1/memcache_osm/0/0/0.PNG'), tile_content, - 'Contents of memcached and value returned from TileStache do not match') + memcache_content = b64decode(self.mc.get('cool_prefix/1/memcache_osm/0/0/0.PNG').encode('ascii')) + + self.assertEqual(memcache_content, tile_content, + 'Contents of memcached and value returned from TileStache should match') - self.assertEqual(self.mc.get('/1/memcache_osm/0/0/0.PNG'), None, - 'Memcache returned a value even though it should have been empty') + self.assertIsNone(self.mc.get('/1/memcache_osm/0/0/0.PNG'), + 'Memcache value should be empty') diff --git a/tests/provider_tests.py b/tests/provider_tests.py index 181f271..130b552 100644 --- a/tests/provider_tests.py +++ b/tests/provider_tests.py @@ -30,7 +30,7 @@ class ProviderTests(TestCase): tile_mimetype, tile_content = utils.request(config_file_content, "osm", "png", 0, 0, 0) self.assertEqual(tile_mimetype, "image/png") - self.assertTrue(tile_content[:4] in '\x89\x50\x4e\x47') #check it is a png based on png magic number + self.assertTrue(tile_content[:4] in b'\x89\x50\x4e\x47') #check it is a png based on png magic number def test_url_template_wgs84(self): @@ -55,12 +55,12 @@ class ProviderTests(TestCase): tile_mimetype, tile_content = utils.request(config_file_content, "osgeo_wms", "png", 0, 0, 0) self.assertEqual(tile_mimetype, "image/png") - self.assertTrue(tile_content[:4] in '\x89\x50\x4e\x47') #check it is a png based on png magic number + self.assertTrue(tile_content[:4] in b'\x89\x50\x4e\x47') #check it is a png based on png magic number #in WGS84 we typically have two tiles at zoom level 0. Get the second tile tile_mimetype, tile_content = utils.request(config_file_content, "osgeo_wms", "png", 0, 1, 0) self.assertEqual(tile_mimetype, "image/png") - self.assertTrue(tile_content[:4] in '\x89\x50\x4e\x47') #check it is a png based on png magic number + self.assertTrue(tile_content[:4] in b'\x89\x50\x4e\x47') #check it is a png based on png magic number class ProviderWithDummyResponseServer(TestCase): @@ -71,8 +71,9 @@ class ProviderWithDummyResponseServer(TestCase): ''' def setUp(self): - #create custom binary file that pretends to be a png and a server that always returns the same response - self.response_content = '\x89\x50\x4e\x47Meh, I am a custom file that loves utf8 chars like éøæ®!!!' + # Create custom binary file that pretends to be a png and a server that always returns the same response + # Smallest PNG from http://garethrees.org/2007/11/14/pngcrush/ + self.response_content = b'\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01\x01\x00\x00\x00\x007n\xf9$\x00\x00\x00\nIDATx\x9cc`\x00\x00\x00\x02\x00\x01H\xaf\xa4q\x00\x00\x00\x00IEND\xaeB`\x82' self.response_mimetype = 'image/png' self.temp_file_name = utils.create_temp_file(self.response_content) diff --git a/tests/servers/dummy-response-server.py b/tests/servers/dummy-response-server.py index 74d2514..2bd6902 100755 --- a/tests/servers/dummy-response-server.py +++ b/tests/servers/dummy-response-server.py @@ -1,3 +1,4 @@ +from __future__ import print_function import argparse from werkzeug.wrappers import Request, Response @@ -17,14 +18,14 @@ if __name__ == '__main__': args = parser.parse_args() #read file into buffer - print 'Response Content: ' + args.response_file + print('Response Content:', args.response_file) global response_content f = open(args.response_file, 'rb') response_content = f.read() f.close() #set mimetype - print 'Response Mimetype: ' + args.response_mimetype + print('Response Mimetype:', args.response_mimetype) global response_mimetype response_mimetype = args.response_mimetype diff --git a/tests/utils.py b/tests/utils.py index 46c17c8..346c8b5 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,3 +1,4 @@ +from __future__ import print_function from tempfile import mkstemp import os import inspect @@ -22,8 +23,11 @@ def request(config_content, layer_name, format, row, column, zoom): Helper method to write config_file to disk and do request ''' + if sys.version_info.major == 2: + is_string = isinstance(config_content, basestring) + else: + is_string = isinstance(config_content, (str, bytes)) - is_string = isinstance(config_content, basestring) if is_string: absolute_file_name = create_temp_file(config_content) config = parseConfig(absolute_file_name) @@ -46,7 +50,7 @@ def create_temp_file(buffer): for deleting file once done ''' fd, absolute_file_name = mkstemp(text=True) - file = os.fdopen(fd, 'w+b') + file = os.fdopen(fd, 'wb' if (type(buffer) is bytes) else 'w') file.write(buffer) file.close() return absolute_file_name @@ -58,8 +62,7 @@ def create_dummy_server(file_with_content, mimetype): mimetype specified ''' - # see http://stackoverflow.com/questions/50499/in-python-how-do-i-get-the-path-and-name-of-the-file-that-is-currently-executin - current_script_dir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) + current_script_dir = os.path.dirname(os.path.abspath(__file__)) #start new process using our dummy-response-server.py script dummy_server_file = os.path.join(current_script_dir, 'servers', 'dummy-response-server.py') @@ -81,7 +84,7 @@ def create_dummy_server(file_with_content, mimetype): t.daemon = True # thread dies with the program t.start() - server_output = '' + server_output = b'' # read line and enter busy loop until the server says it is ok while True: @@ -97,7 +100,7 @@ def create_dummy_server(file_with_content, mimetype): continue else: # got line server_output += line - if "Running on http://"; in server_output: + if b"Running on http://"; in server_output: break; #server is running, get out of here else: continue diff --git a/tests/vectiles_tests.py b/tests/vectiles_tests.py index 95adb1b..22918d8 100644 --- a/tests/vectiles_tests.py +++ b/tests/vectiles_tests.py @@ -1,3 +1,5 @@ +from __future__ import print_function + import os from unittest import TestCase, skipIf from collections import namedtuple @@ -19,7 +21,8 @@ from . import utils def get_topo_transform(topojson): ''' ''' - def xform((x, y)): + def xform(xy): + x, y = xy lon = topojson['transform']['scale'][0] * x + topojson['transform']['translate'][0] lat = topojson['transform']['scale'][1] * y + topojson['transform']['translate'][1] @@ -94,12 +97,11 @@ def decoded_pbf_asshape(feature, extent, srid=4326): 3: "Polygon" } if feature['type'] in (1, 2): - coords = [ - trans_coord(3857, srid, *coord2merc(*g, extent=extent)) for g in feature['geometry']] + coords = [trans_coord(3857, srid, *coord2merc(x, y, extent=extent)) + for (x, y) in feature['geometry']] elif feature['type'] == 3: - coords = [ - [trans_coord(3857, srid, *coord2merc(*g, extent=extent)) for g in feature['geometry'][0]] - ] + coords = [[trans_coord(3857, srid, *coord2merc(x, y, extent=extent)) + for (x, y) in feature['geometry'][0]]] geoint = { 'type': TYPES_MAP.get(feature['type']), 'coordinates': coords, @@ -235,7 +237,7 @@ class VectorProviderTest(PostGISVectorTestBase, TestCase): # northwest quadrant should return San Francisco and Lima tile_mimetype, tile_content = utils.request(self.config_file_content, "vectile_test", "json", 0, 0, 1) - geojson_result = json.loads(tile_content) + geojson_result = json.loads(tile_content.decode('utf8')) self.assertTrue(tile_mimetype.endswith('/json')) self.assertEqual(geojson_result['type'], 'FeatureCollection') @@ -261,7 +263,7 @@ class VectorProviderTest(PostGISVectorTestBase, TestCase): # northeast quadrant should return Berlin tile_mimetype, tile_content = utils.request(self.config_file_content, "vectile_test", "json", 0, 1, 1) - geojson_result = json.loads(tile_content) + geojson_result = json.loads(tile_content.decode('utf8')) self.assertTrue(tile_mimetype.endswith('/json')) self.assertEqual(geojson_result['type'], 'FeatureCollection') @@ -282,7 +284,7 @@ class VectorProviderTest(PostGISVectorTestBase, TestCase): tile_mimetype, tile_content = utils.request(self.config_file_content, "vectile_test", "json", 0, 0, 0) self.assertTrue(tile_mimetype.endswith('/json')) - geojson_result = json.loads(tile_content) + geojson_result = json.loads(tile_content.decode('utf8')) west_hemisphere_geometry = asShape(geojson_result['features'][0]['geometry']) expected_geometry = LineString([(-180, 32), (180, 32)]) self.assertTrue(expected_geometry.almost_equals(west_hemisphere_geometry)) @@ -305,7 +307,7 @@ class VectorProviderTest(PostGISVectorTestBase, TestCase): tile_mimetype, tile_content = utils.request(self.config_file_content, "vectile_test", "json", 0, 0, 0) self.assertTrue(tile_mimetype.endswith('/json')) - geojson_result = json.loads(tile_content) + geojson_result = json.loads(tile_content.decode('utf8')) result_geom = asShape(geojson_result['features'][0]['geometry']) expected_geom = Polygon( [(-180, -85.05), (180, -85.05), (180, 85.05), (-180, 85.05), (-180, -85.05)]) @@ -346,7 +348,7 @@ class VectorProviderTest(PostGISVectorTestBase, TestCase): tile_mimetype, tile_content = utils.request(self.config_file_content, "vectile_multi", "json", 0, 0, 0) self.assertTrue(tile_mimetype.endswith('/json')) - geojson_result = json.loads(tile_content) + geojson_result = json.loads(tile_content.decode('utf8')) feature1, feature2 = geojson_result['vectile_test'], geojson_result['vectile_copy'] self.assertEqual(feature1['type'], 'FeatureCollection') @@ -382,10 +384,10 @@ class VectorProviderTest(PostGISVectorTestBase, TestCase): # northwest quadrant should return San Francisco and Lima tile_mimetype, tile_content = utils.request(self.config_file_content, "vectile_test", "topojson", 0, 0, 1) - topojson_result = json.loads(tile_content) + topojson_result = json.loads(tile_content.decode('utf8')) self.assertTrue(tile_mimetype.endswith('/json')) - print topojson_result + print(topojson_result) self.assertEqual(topojson_result['type'], 'Topology') self.assertEqual(len(topojson_result['objects']['vectile']['geometries']), 2) @@ -404,7 +406,7 @@ class VectorProviderTest(PostGISVectorTestBase, TestCase): elif feature['properties']['name'] == 'Lima': cities.append(feature['properties']['name']) - print feature['coordinates'] + print(feature['coordinates']) self.assertTrue(hypot(point_lima.x - lon, point_lima.y - lat) < 1) self.assertTrue('San Francisco' in cities) @@ -414,7 +416,7 @@ class VectorProviderTest(PostGISVectorTestBase, TestCase): # northeast quadrant should return Berlin tile_mimetype, tile_content = utils.request(self.config_file_content, "vectile_test", "topojson", 0, 1, 1) - topojson_result = json.loads(tile_content) + topojson_result = json.loads(tile_content.decode('utf8')) self.assertTrue(tile_mimetype.endswith('/json')) self.assertEqual(topojson_result['type'], 'Topology') @@ -435,7 +437,7 @@ class VectorProviderTest(PostGISVectorTestBase, TestCase): tile_mimetype, tile_content = utils.request(self.config_file_content, "vectile_test", "topojson", 0, 0, 0) self.assertTrue(tile_mimetype.endswith('/json')) - topojson_result = json.loads(tile_content) + topojson_result = json.loads(tile_content.decode('utf8')) topojson_xform = get_topo_transform(topojson_result) parts = [topojson_result['arcs'][arc] for arc in topojson_result['objects']['vectile']['geometries'][0]['arcs']] @@ -467,7 +469,7 @@ class VectorProviderTest(PostGISVectorTestBase, TestCase): tile_mimetype, tile_content = utils.request(self.config_file_content, "vectile_test", "topojson", 0, 0, 0) self.assertTrue(tile_mimetype.endswith('/json')) - topojson_result = json.loads(tile_content) + topojson_result = json.loads(tile_content.decode('utf8')) topojson_xform = get_topo_transform(topojson_result) parts = [topojson_result['arcs'][arc[0]] for arc in topojson_result['objects']['vectile']['geometries'][0]['arcs']] @@ -513,7 +515,7 @@ class VectorProviderTest(PostGISVectorTestBase, TestCase): tile_mimetype, tile_content = utils.request(self.config_file_content, "vectile_multi", "topojson", 0, 0, 0) self.assertTrue(tile_mimetype.endswith('/json')) - topojson_result = json.loads(tile_content) + topojson_result = json.loads(tile_content.decode('utf8')) self.assertEqual(topojson_result['type'], 'Topology') self.assertEqual(topojson_result['objects']['vectile_test']['type'], 'GeometryCollection') @@ -593,7 +595,7 @@ class VectorProviderTest(PostGISVectorTestBase, TestCase): '''Create a line that goes from west to east (clip on) (pbf)''' self.defineGeometry('LINESTRING') - geom = LineString([(-180, 32), (180, 32)]) + geom = LineString([(-179, 32), (179, 32)]) self.insertTestRow(geom.wkt) @@ -607,11 +609,10 @@ class VectorProviderTest(PostGISVectorTestBase, TestCase): extent = tile_bounds_mercator(0, 0, 0) west_hemisphere_geometry = decoded_pbf_asshape(layer_result['features'][0], extent) - # order of points returned are different - expected_geometry = LineString([(180, 32), (-180, 32)]) + expected_geometry = LineString([(-179, 32), (179, 32)]) for returned, expected in zip(west_hemisphere_geometry.coords, expected_geometry.coords): - self.assertTrue(round(returned[0]) == expected[0]) - self.assertTrue(round(returned[1]) == expected[1]) + self.assertEqual(round(returned[0]), expected[0]) + self.assertEqual(round(returned[1]), expected[1]) def test_polygon_pbf(self): ''' diff --git a/tests/vector_tests.py b/tests/vector_tests.py index 38082a2..980d18c 100644 --- a/tests/vector_tests.py +++ b/tests/vector_tests.py @@ -103,7 +103,7 @@ class VectorProviderTest(PostGISVectorTestBase, TestCase): # western hemisphere should return San Francisco and Lima tile_mimetype, tile_content = utils.request(self.config_file_content, "vector_test", "geojson", 0, 0, 0) - geojson_result = json.loads(tile_content) + geojson_result = json.loads(tile_content.decode('utf8')) self.assertTrue(tile_mimetype.endswith('/json')) self.assertEqual(geojson_result['type'], 'FeatureCollection') @@ -129,7 +129,7 @@ class VectorProviderTest(PostGISVectorTestBase, TestCase): # eastern hemisphere should return Berlin tile_mimetype, tile_content = utils.request(self.config_file_content, "vector_test", "geojson", 0, 1, 0) - geojson_result = json.loads(tile_content) + geojson_result = json.loads(tile_content.decode('utf8')) self.assertTrue(tile_mimetype.endswith('/json')) self.assertEqual(geojson_result['type'], 'FeatureCollection') @@ -151,7 +151,7 @@ class VectorProviderTest(PostGISVectorTestBase, TestCase): # for western hemisphere.... tile_mimetype, tile_content = utils.request(self.config_file_content, "vector_test", "geojson", 0, 0, 0) self.assertTrue(tile_mimetype.endswith('/json')) - geojson_result = json.loads(tile_content) + geojson_result = json.loads(tile_content.decode('utf8')) west_hemisphere_geometry = asShape(geojson_result['features'][0]['geometry']) expected_geometry = LineString([(-180, 32), (0, 32)]) self.assertTrue(expected_geometry.almost_equals(west_hemisphere_geometry)) @@ -159,7 +159,7 @@ class VectorProviderTest(PostGISVectorTestBase, TestCase): # for eastern hemisphere.... tile_mimetype, tile_content = utils.request(self.config_file_content, "vector_test", "geojson", 0, 1, 0) self.assertTrue(tile_mimetype.endswith('/json')) - geojson_result = json.loads(tile_content) + geojson_result = json.loads(tile_content.decode('utf8')) east_hemisphere_geometry = asShape(geojson_result['features'][0]['geometry']) expected_geometry = LineString([(0, 32), (180, 32)]) self.assertTrue(expected_geometry.almost_equals(east_hemisphere_geometry)) @@ -182,7 +182,7 @@ class VectorProviderTest(PostGISVectorTestBase, TestCase): tile_mimetype, tile_content = utils.request(self.config_file_content, "vector_test", "geojson", 0, 0, 0) self.assertTrue(tile_mimetype.endswith('/json')) - geojson_result = json.loads(tile_content) + geojson_result = json.loads(tile_content.decode('utf8')) result_geom = asShape(geojson_result['features'][0]['geometry']) expected_geom = Polygon( [(-180, -90), (0, -90), (0, 90), (-180, 90), (-180, -90)]) -- Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-grass/tilestache.git _______________________________________________ Pkg-grass-devel mailing list [email protected] http://lists.alioth.debian.org/cgi-bin/mailman/listinfo/pkg-grass-devel
