Revision: 4701 http://sourceforge.net/p/jump-pilot/code/4701 Author: elnico Date: 2015-12-27 17:32:15 +0000 (Sun, 27 Dec 2015) Log Message: ----------- GeoPackage layout for spatial metadata support added: blob gpkg read with code from DBQuery plugin Date and datetime columns mapped to String (TODO: use Joda to try to parse a valid date ?)
Modified Paths: -------------- core/trunk/src/com/vividsolutions/jump/datastore/spatialdatabases/SpatialDatabasesDSMetadata.java core/trunk/src/com/vividsolutions/jump/datastore/spatialite/GeometryColumnsLayout.java core/trunk/src/com/vividsolutions/jump/datastore/spatialite/SpatialiteDSMetadata.java core/trunk/src/com/vividsolutions/jump/datastore/spatialite/SpatialiteSQLBuilder.java core/trunk/src/com/vividsolutions/jump/datastore/spatialite/SpatialiteValueConverterFactory.java Modified: core/trunk/src/com/vividsolutions/jump/datastore/spatialdatabases/SpatialDatabasesDSMetadata.java =================================================================== --- core/trunk/src/com/vividsolutions/jump/datastore/spatialdatabases/SpatialDatabasesDSMetadata.java 2015-12-27 17:29:08 UTC (rev 4700) +++ core/trunk/src/com/vividsolutions/jump/datastore/spatialdatabases/SpatialDatabasesDSMetadata.java 2015-12-27 17:32:15 UTC (rev 4701) @@ -236,8 +236,6 @@ final ResultSetBlock resultSetBlock = new ResultSetBlock() { public void yield(ResultSet resultSet) throws Exception { if (resultSet.next()) { - // hack for Oracle: todo: find common method: overload in oracle impl ? - // try a binary reader, then defaults to a WKT reader byte[] bytes = null; Geometry geom = null; try { @@ -260,9 +258,6 @@ JDBCUtil.execute(conn.getJdbcConnection(), (sql2), resultSetBlock); } } catch (Exception ex1) { - // If attributeName is indexed but indexed has not been initialized - // (no VACUUM ANALYZE) ST_Estimated_Extent may throw an exception. - // In this case, try the second method using ST_Extent if (sql2 != null) { // some drivers do not support a second SQL query for extent: /// sqlite w/o spatialite for instance Modified: core/trunk/src/com/vividsolutions/jump/datastore/spatialite/GeometryColumnsLayout.java =================================================================== --- core/trunk/src/com/vividsolutions/jump/datastore/spatialite/GeometryColumnsLayout.java 2015-12-27 17:29:08 UTC (rev 4700) +++ core/trunk/src/com/vividsolutions/jump/datastore/spatialite/GeometryColumnsLayout.java 2015-12-27 17:32:15 UTC (rev 4701) @@ -6,5 +6,5 @@ * @author nicolas */ public enum GeometryColumnsLayout { - FDO_LAYOUT, OGC_OGR_LAYOUT,OGC_SPATIALITE_LAYOUT, NO_LAYOUT; + FDO_LAYOUT, OGC_OGR_LAYOUT,OGC_SPATIALITE_LAYOUT, OGC_GEOPACKAGE_LAYOUT, NO_LAYOUT; } Modified: core/trunk/src/com/vividsolutions/jump/datastore/spatialite/SpatialiteDSMetadata.java =================================================================== --- core/trunk/src/com/vividsolutions/jump/datastore/spatialite/SpatialiteDSMetadata.java 2015-12-27 17:29:08 UTC (rev 4700) +++ core/trunk/src/com/vividsolutions/jump/datastore/spatialite/SpatialiteDSMetadata.java 2015-12-27 17:32:15 UTC (rev 4701) @@ -21,6 +21,7 @@ public class SpatialiteDSMetadata extends SpatialDatabasesDSMetadata { public static String GC_COLUMN_NAME = "geometry_columns"; + public static String GPKG_GC_COLUMN_NAME = "gpkg_geometry_columns"; //TODO= variables for all SQL code + String.format. /** @@ -40,10 +41,16 @@ * The map of geometric columns types (WKB, WKT, SPATIALITE) */ private Map<String, GeometricColumnType> geoColTypesdMap = null; + + /** + * The query to get the list of geometric columns data types, used to build a + * suitable SQL query OJ can read + */ + private String geoColumnTypesQuery = null; /** - * - * @param con + * + * @param con */ public SpatialiteDSMetadata(DataStoreConnection con) { conn = con; @@ -54,37 +61,59 @@ checkSpatialiteLoaded(); setGeoColLayout(); + + // formats queries to use for this connection according to the detected layout + geoColumnTypesQuery = "select f_table_name, f_geometry_column, \"SPATIALITE\" as geometry_format from geometry_columns"; + if (this.getGeometryColumnsLayout() == GeometryColumnsLayout.FDO_LAYOUT) { + // MD table contains a geometry_format column: query it + geoColumnTypesQuery = "select f_table_name, f_geometry_column, geometry_format from geometry_columns"; + } else if (this.getGeometryColumnsLayout() == GeometryColumnsLayout.OGC_GEOPACKAGE_LAYOUT) { + // MD table contains a geometry_format column: query it + geoColumnTypesQuery = "select table_name as f_table_name, column_name as " + + "f_geometry_columns, \"SPATIALITE\" as geometry_format from gpkg_geometry_columns"; + } + // done here as every connection needs it getGeoColumnType(); - + // TODO: use bind parameters to avoid SQL injection datasetNameQuery = "SELECT DISTINCT '' as f_table_schema, f_table_name FROM geometry_columns"; + if (this.getGeometryColumnsLayout() == GeometryColumnsLayout.OGC_GEOPACKAGE_LAYOUT) { + // MD table contains a geometry_format column: query it + datasetNameQuery = "SELECT DISTINCT '' as f_table_schema, table_name as f_table_name FROM gpkg_geometry_columns"; + } + defaultSchemaName = ""; spatialDbName = isSpatialiteLoaded() ? "Spatialite" : "SQLite"; spatialExtentQuery1 = "SELECT %s from %s"; // no second query for spatialite spatialExtentQuery2 = null; - sridQuery = "SELECT srid FROM geometry_columns where f_table_name = '%s' and f_geometry_column = '%s'"; + if (this.geometryColumnsLayout == GeometryColumnsLayout.OGC_GEOPACKAGE_LAYOUT) { + sridQuery = "SELECT srs_id FROM gpkg_geometry_columns where table_name = '%s' and column_name = '%s'"; + } else { + sridQuery = "SELECT srid FROM geometry_columns where f_table_name = '%s' and f_geometry_column = '%s'"; + } // geo column query needs to be built occording to geometryColumnsLayout - if (this.geometryColumnsLayout == GeometryColumnsLayout.FDO_LAYOUT || - this.geometryColumnsLayout == GeometryColumnsLayout.OGC_OGR_LAYOUT) { - geoColumnsQuery = "SELECT f_geometry_column, srid,\n" + - " case\n" + - " when geometry_type = 1 then 'POINT'\n" + - " when geometry_type = 2 then 'LINESTRING'\n" + - " when geometry_type = 3 then 'POLYGON'\n" + - " when geometry_type = 4 then 'MULTIPOINT'\n" + - " when geometry_type = 5 then 'MULTILINESTRING'\n" + - " when geometry_type = 6 then 'MULTIPOLYGON'\n" + - " when geometry_type = 7 then 'GEOMETRY COLLECTION'\n" + - " else geometry_type end as geometry_type\n" + - "FROM geometry_columns where f_table_name = '%s'"; + if (this.geometryColumnsLayout == GeometryColumnsLayout.FDO_LAYOUT + || this.geometryColumnsLayout == GeometryColumnsLayout.OGC_OGR_LAYOUT) { + geoColumnsQuery = "SELECT f_geometry_column, srid,\n" + + " case\n" + + " when geometry_type = 1 then 'POINT'\n" + + " when geometry_type = 2 then 'LINESTRING'\n" + + " when geometry_type = 3 then 'POLYGON'\n" + + " when geometry_type = 4 then 'MULTIPOINT'\n" + + " when geometry_type = 5 then 'MULTILINESTRING'\n" + + " when geometry_type = 6 then 'MULTIPOLYGON'\n" + + " when geometry_type = 7 then 'GEOMETRY COLLECTION'\n" + + " else geometry_type end as geometry_type\n" + + "FROM geometry_columns where f_table_name = '%s'"; } else if (this.geometryColumnsLayout == GeometryColumnsLayout.OGC_SPATIALITE_LAYOUT) { geoColumnsQuery = "SELECT f_geometry_column, srid, type FROM geometry_columns where f_table_name = '%s'"; + } else if (this.geometryColumnsLayout == GeometryColumnsLayout.OGC_GEOPACKAGE_LAYOUT) { + geoColumnsQuery = "SELECT column_name, srs_id, geometry_type_name FROM gpkg_geometry_columns where table_name = '%s'"; } else { geoColumnsQuery = "SELECT '' "; } - } @Override @@ -154,14 +183,14 @@ ResultSet rs = stmt.executeQuery("select spatialite_version()"); rs.next(); this.setSpatialiteVersion(rs.getString(1)); - + JUMPWorkbench.getInstance().getFrame().log( - "SpatialDatabasesPlugin: Spatialite extension loaded for this connexion, version: " - + this.getSpatialiteVersion(), this.getClass()); + "SpatialDatabasesPlugin: Spatialite extension loaded for this connexion, version: " + + this.getSpatialiteVersion(), this.getClass()); } catch (Exception e) { JUMPWorkbench.getInstance().getFrame().log( - "SpatialDatabasesPlugin: CANNOT load Spatialite Extention (mod_spatialite), reason:" - + e.getMessage(), this.getClass()); + "SpatialDatabasesPlugin: CANNOT load Spatialite Extention (mod_spatialite), reason:" + + e.getMessage(), this.getClass()); } finally { try { stmt.close(); @@ -173,65 +202,87 @@ } /** - * Sets the geometry_column layout in this sqlite database: either FDO or - * OGC or no layout. Also tries to build the geo col type if geometry_columns - * table contains such info TODO: generic mechanism to get geo col type for - * Spatialite + * Sets the geometry_column layout in this sqlite database: either FDO or OGC + * or GeoPkg or no layout. Also tries to build the geo col type if + * geometry_columns table contains such info TODO: generic mechanism to get + * geo col type for Spatialite */ private void setGeoColLayout() { DatabaseMetaData dbMd = null; try { dbMd = this.conn.getJdbcConnection().getMetaData(); - ResultSet rs = dbMd.getTables(null, null, SpatialiteDSMetadata.GC_COLUMN_NAME, null); + + // GeoPackage test: + ResultSet rs = dbMd.getTables(null, null, SpatialiteDSMetadata.GPKG_GC_COLUMN_NAME, null); if (rs.next()) { - // tableName is third column in this metadata resultSet - String col = rs.getString(3); - boolean isGC = SpatialiteDSMetadata.GC_COLUMN_NAME.equalsIgnoreCase(col); + // no need to look at table layout: table name found is enough to say its geoPackage layout + geometryColumnsLayout = GeometryColumnsLayout.OGC_GEOPACKAGE_LAYOUT; + rs.close(); + } else { - // gc layout - if (isGC) { - rs = dbMd.getColumns(null, null, SpatialiteDSMetadata.GC_COLUMN_NAME, null); - int i = 0; - // geometry_columns table may have 2 layouts according to ogr2ogr - // options used to create the table: - // 1\xB0) the "FDO provider for spatialite (https://trac.osgeo.org/fdo/wiki/FDORfc16)", as used in "regular sqlite database" (cf.ogr spatialite format doc): - // f_table_name TEXT - // f_geometry_column TEXT - // geometry_type INTEGER - // coord_dimension INTEGER - // srid INTEGER - // geometry_format TEXT - // 2\xB0) the "OGC" flavour, as understood by qgis for instance, as used in spatialite-enabled sqlite database: - // f_table_name VARCHAR - // f_geometry_column VARCHAR - // type VARCHAR - // coord_dimension INTEGER - // srid INTEGER - // spatial_index_enabled INTEGER - i = 0; - String geoTypeCol = ""; - String extraInfoCol = ""; - while (rs.next()) { + // OGC/FDO layout + rs = dbMd.getTables(null, null, SpatialiteDSMetadata.GC_COLUMN_NAME, null); + if (rs.next()) { + // tableName is third column in this metadata resultSet + String col = rs.getString(3); + + // TODO: clean-up the JDBC metadata use... + boolean isGC = (SpatialiteDSMetadata.GC_COLUMN_NAME.equalsIgnoreCase(col) + || SpatialiteDSMetadata.GPKG_GC_COLUMN_NAME.equalsIgnoreCase(col)); + + // gc layout + if (isGC) { + rs = dbMd.getColumns(null, null, SpatialiteDSMetadata.GC_COLUMN_NAME, null); + int i = 0; + // geometry_columns table may have 23 layouts according to ogr2ogr + // options used to create the table or using a geo package (http://www.geopackage.org/) layout + // 1\xB0) the "FDO provider for spatialite (https://trac.osgeo.org/fdo/wiki/FDORfc16)", as used in "regular sqlite database" (cf.ogr spatialite format doc): + // f_table_name TEXT + // f_geometry_column TEXT + // geometry_type INTEGER + // coord_dimension INTEGER + // srid INTEGER + // geometry_format TEXT + // 2\xB0) the "OGC" flavour, as understood by qgis for instance, as used in spatialite-enabled sqlite database: + // f_table_name VARCHAR + // f_geometry_column VARCHAR + // type VARCHAR + // coord_dimension INTEGER + // srid INTEGER + // spatial_index_enabled INTEGER + // 3\xB0) the "GeoPackage" flavour, + // table_name TEXT NOT NULL, + // column_name TEXT NOT NULL, + // geometry_type_name TEXT NOT NULL, + // srs_id INTEGER NOT NULL, + // z INTEGER NOT NULL, + // m INTEGER NOT NULL, + + i = 0; + String geoTypeCol = ""; + String extraInfoCol = ""; + while (rs.next()) { // assume columns order is respected when gc table is created. - // TODO: enhance this - if (i == 2) { - geoTypeCol = rs.getString(4); + // TODO: enhance this + if (i == 2) { + geoTypeCol = rs.getString(4); + } + if (i == 5) { + extraInfoCol = rs.getString(4); + } + i++; } - if (i == 5) { - extraInfoCol = rs.getString(4); - } - i++; + if (geoTypeCol.equalsIgnoreCase("geometry_type") && extraInfoCol.equalsIgnoreCase("geometry_format")) { + geometryColumnsLayout = GeometryColumnsLayout.FDO_LAYOUT; + } else if (geoTypeCol.equalsIgnoreCase("type") && extraInfoCol.equalsIgnoreCase("spatial_index_enabled")) { + geometryColumnsLayout = GeometryColumnsLayout.OGC_SPATIALITE_LAYOUT; + } else if (geoTypeCol.equalsIgnoreCase("geometry_type") && extraInfoCol.equalsIgnoreCase("spatial_index_enabled")) { + geometryColumnsLayout = GeometryColumnsLayout.OGC_OGR_LAYOUT; + } else { + geometryColumnsLayout = GeometryColumnsLayout.NO_LAYOUT; + }; + rs.close(); } - if (geoTypeCol.equalsIgnoreCase("geometry_type") && extraInfoCol.equalsIgnoreCase("geometry_format")) { - geometryColumnsLayout = GeometryColumnsLayout.FDO_LAYOUT; - } else if (geoTypeCol.equalsIgnoreCase("type") && extraInfoCol.equalsIgnoreCase("spatial_index_enabled")) { - geometryColumnsLayout = GeometryColumnsLayout.OGC_SPATIALITE_LAYOUT; - } else if (geoTypeCol.equalsIgnoreCase("geometry_type") && extraInfoCol.equalsIgnoreCase("spatial_index_enabled")) { - geometryColumnsLayout = GeometryColumnsLayout.OGC_OGR_LAYOUT; - } else { - geometryColumnsLayout = GeometryColumnsLayout.NO_LAYOUT; - }; - rs.close(); } } @@ -244,22 +295,16 @@ /** * builds the map of geometric columns database type: WKB, WKT, SPATIALITE to - * be able to build custom queries for extent and geo type retrieval. - * The geometry_format column of the metadata will be queries to find geometry type - * (column only detected in the FDO_LAYOUT format). - * For other layout, will default to SPATIALITE type + * be able to build custom queries for extent and geo type retrieval. The + * geometry_format column of the metadata will be queries to find geometry + * type (column only detected in the FDO_LAYOUT format). For other layout, + * will default to SPATIALITE type */ private void getGeoColumnType() { - // Default query gets a hard-coded value for spatialite type - String query = "select f_table_name, f_geometry_column, \"SPATIALITE\" from geometry_columns"; - if (this.getGeometryColumnsLayout() == GeometryColumnsLayout.FDO_LAYOUT) { - // MD table contains a geometry_format column: query it - query = "select f_table_name, f_geometry_column, geometry_format from geometry_columns"; - } try { JDBCUtil.execute( conn.getJdbcConnection(), - query, + this.geoColumnTypesQuery, new ResultSetBlock() { public void yield(ResultSet resultSet) throws SQLException { while (resultSet.next()) { Modified: core/trunk/src/com/vividsolutions/jump/datastore/spatialite/SpatialiteSQLBuilder.java =================================================================== --- core/trunk/src/com/vividsolutions/jump/datastore/spatialite/SpatialiteSQLBuilder.java 2015-12-27 17:29:08 UTC (rev 4700) +++ core/trunk/src/com/vividsolutions/jump/datastore/spatialite/SpatialiteSQLBuilder.java 2015-12-27 17:32:15 UTC (rev 4701) @@ -39,9 +39,9 @@ //System.out.println(qs); String s = String.format(ret, cols, this.datasetName, bbox, and, lim); - JUMPWorkbench.getInstance().getFrame().log( - "SQL query to get Spatial table features:\n\t" - + s, this.getClass()); +// JUMPWorkbench.getInstance().getFrame().log( +// "SQL query to get Spatial table features:\n\t" +// + s, this.getClass()); return s; } Modified: core/trunk/src/com/vividsolutions/jump/datastore/spatialite/SpatialiteValueConverterFactory.java =================================================================== --- core/trunk/src/com/vividsolutions/jump/datastore/spatialite/SpatialiteValueConverterFactory.java 2015-12-27 17:29:08 UTC (rev 4700) +++ core/trunk/src/com/vividsolutions/jump/datastore/spatialite/SpatialiteValueConverterFactory.java 2015-12-27 17:32:15 UTC (rev 4701) @@ -43,22 +43,27 @@ // gets concerned tableName and column name to be able to detect geom column as text String tableName = rsm.getTableName(columnIndex).toLowerCase(); String columnName = rsm.getColumnName(columnIndex).toLowerCase(); - - GeometricColumnType gcType = metadata.getGeoColTypesdMap().get(tableName+"."+columnName); + + GeometricColumnType gcType = metadata.getGeoColTypesdMap().get(tableName + "." + columnName); if (gcType == null) { - // not a geo column + // not a geo column, handle date/datetime columns by forcing text mode + // todo: use Joda to parse date string ? + if ("DATETIME".equalsIgnoreCase(dbTypeName) || "DATE".equalsIgnoreCase(dbTypeName)) { + return ValueConverterFactory.STRING_MAPPER; + } ValueConverter stdConverter = ValueConverterFactory.getConverter(rsm, columnIndex); if (stdConverter != null) { return stdConverter; - } + } // default - can always show it as a string! return ValueConverterFactory.STRING_MAPPER; } else if (gcType == GeometricColumnType.WKB) { - return WKB_GEOMETRY_MAPPER; + return WKB_GEOMETRY_MAPPER; } else if (gcType == GeometricColumnType.WKT) { - return WKT_GEOMETRY_MAPPER; - } else if (gcType == GeometricColumnType.SPATIALITE) { - return SPATIALITE_GEOMETRY_MAPPER; + return WKT_GEOMETRY_MAPPER; + } else if (gcType == GeometricColumnType.SPATIALITE + || gcType == GeometricColumnType.NATIVE) { + return SPATIALITE_GEOMETRY_MAPPER; } else { return ValueConverterFactory.STRING_MAPPER; } @@ -81,7 +86,15 @@ //no FDO info for this table, try native spatialite blob encoding byte[] geometryBytes = rs.getBytes(columnIndex); - returnGeometry = getNativeGeometryFromBlob(geometryBytes); + if (geometryBytes != null) { + if (appearsToBeGeopackageGeometry(geometryBytes)) { + returnGeometry = getGeopackageGeometryFromBlob(geometryBytes); + } else { + returnGeometry = getNativeGeometryFromBlob(geometryBytes); + } + } else { + returnGeometry = wktReader.read("GEOMETRYCOLLECTION EMPTY"); + } return returnGeometry; } @@ -107,6 +120,132 @@ return returnGeometry; } + /** + * From DB Query plugin: TODO: factorize code + * + * @param blobAsBytes + * @return + * @throws Exception + */ + private Geometry getGeopackageGeometryFromBlob(byte[] blobAsBytes) throws IOException, ParseException { + Geometry returnGeometry; + + //first two bytes are GP.. + //Third byte is version + //Fourth byte is flags + byte flags = blobAsBytes[3]; + //Bytes 5-8 are SRS ID + + int evelopeSize = 0; + + //FIXME do something with this like Create empty geometry collection?? + // 0b00100000 == 0X20 + boolean emptyGeometry = (flags & 0X20) != 0; + + int envelopSize = getEnvelopeSize(flags); + + int headerSize = 8 + envelopSize; + + byte[] wkb = new byte[blobAsBytes.length - headerSize]; + System.arraycopy(blobAsBytes, headerSize, wkb, 0, blobAsBytes.length - headerSize); + WKBReader wkbReader = new WKBReader(); + returnGeometry = wkbReader.read(wkb); + + if (returnGeometry == null) { + throw new IOException("Unable to parse WKB"); + } + + return returnGeometry; + } + + /** + * From DB Query plugin: TODO: factorize code + * + * @param flags + * @return + * @throws Exception + */ + private int getEnvelopeSize(byte flags) throws IOException { + //0b0000001 == 0x01 + boolean littleEndian = (flags & 0x01) != 0; + + //0b00001110 == 0x0E + int envelopeCode = (flags & 0x0E) >>> 1; + + //spec says the endian bit sets byte order for "header" values + //Not sure what 'header" is in this context, but using it with + //geonames sample provided by Jukka results in bad parsing of BLOB +// if(littleEndian) +// { +// envelopeCode = 0; +// if( (flags & 0b00001000) != 0) +// { +// envelopeCode += 1; +// } +// +// if( (flags & 0b00000100) != 0) +// { +// envelopeCode += 2; +// } +// +// if( (flags & 0b00000010) != 0) +// { +// envelopeCode += 4; +// } +// } + int envelopeSize; + switch (envelopeCode) { + case 0: + envelopeSize = 0; + break; + case 1: + envelopeSize = 32; + break; + case 2: + case 3: + envelopeSize = 48; + break; + case 4: + envelopeSize = 64; + break; + default: + //Envelope codes 5-7 are invalid + throw new IOException("Invalid envelope code " + envelopeCode); + } + + return envelopeSize; + } + } + private boolean appearsToBeGeopackageGeometry(byte[] geometryAsBytes) { + + //From http://opengis.github.io/geopackage/#gpb_format + //Geopackage blobs start with "gp", contain some other header + //info, and are followed by a WKB + return (geometryAsBytes.length > 2 + && geometryAsBytes[0] == (byte) 0x47 //G + && geometryAsBytes[1] == (byte) 0x50 //P + ); + } + + private boolean appearsToBeNativeGeometry(byte[] geometryAsBytes) { + boolean blobIsGeometry = false; + + //From http://www.gaia-gis.it/spatialite-2.1/SpatiaLite-manual.html + //Spatialite geometry blobs are WKB-like, with some specifics to + //spatialite: For our purposes, this should be good enough: + //the 39th byte must be 0x7C (marks MBR end) + //and the blob must end with 0xFE + int numBytes = geometryAsBytes.length; + + if (numBytes > 39 + && geometryAsBytes[38] == (byte) 0x7C + && geometryAsBytes[numBytes - 1] == (byte) 0xFE) { + blobIsGeometry = true; + } + + return blobIsGeometry; + } + } ------------------------------------------------------------------------------ _______________________________________________ Jump-pilot-devel mailing list Jump-pilot-devel@lists.sourceforge.net https://lists.sourceforge.net/lists/listinfo/jump-pilot-devel