diff options
| -rw-r--r-- | config.yml | 81 | ||||
| -rw-r--r-- | export_mvt.py | 110 |
2 files changed, 174 insertions, 17 deletions
@@ -2709,7 +2709,10 @@ layers: - AvvSasong - AvvHa - AvverkningsanmalanKlass - publish: anmald + publish: + anmald: + fields: + age: Inkomdatum 'sks:avverk_utford': # https://geodpags.skogsstyrelsen.se/geodataport/feeds/UtfordAvverk.xml @@ -2799,7 +2802,10 @@ layers: Beteckn: - replace: 'Visas ej' with: null - publish: utford + publish: + utford: + fields: + age: Avvdatum 'sametinget:betesomrade': description: 'Samebyarnas betesområden: Renbetesområden' @@ -3217,30 +3223,44 @@ layers: minzoom: 4 where: | "Raderat" IS FALSE AND "Statuskod" = 4 + fields: + age: SenasteUppdaterat station_processed: minzoom: 4 where: | "Raderat" IS FALSE AND "Statuskod" = 1 + fields: + age: SenasteUppdaterat station_approved: minzoom: 4 where: | "Raderat" IS FALSE AND "Statuskod" = 3 + fields: + age: SenasteUppdaterat station_revoked: minzoom: 4 where: | "Raderat" IS FALSE AND ("Statuskod" = 6 OR "Statuskod" = 9) + fields: + age: SenasteUppdaterat station_rejected: minzoom: 4 where: | "Raderat" IS FALSE AND "Statuskod" = 7 + fields: + age: SenasteUppdaterat station_dismounted: minzoom: 4 where: | "Raderat" IS FALSE AND "Statuskod" = 5 + fields: + age: SenasteUppdaterat station_appealed: minzoom: 4 where: | "Raderat" IS FALSE AND "Statuskod" = 8 + fields: + age: SenasteUppdaterat 'vbk:projekteringsomraden': description: Vindbrukskollen landbaserade projekteringsområden (Länsstyrelsen) @@ -3395,9 +3415,13 @@ layers: area_current: where: | "Raderat" IS FALSE AND "EjAktuell" IS FALSE + fields: + age: SenasteUppdaterat area_notcurrent: where: | "Raderat" IS FALSE AND "EjAktuell" IS NOT FALSE + fields: + age: SenasteUppdaterat 'vbk:havsbaserad_vindkraft': description: Vindbrukskollen havsbaserad vindkraft (Länsstyrelsen) @@ -3613,30 +3637,48 @@ layers: offshore_completed: where: | "Raderat" IS FALSE AND "Projektstatus" = 'Uppförd' + fields: + age: SenasteUppdaterat offshore_approved: where: | "Raderat" IS FALSE AND "Projektstatus" = 'Tillståndsansökan beviljad' + fields: + age: SenasteUppdaterat offshore_amended: where: | "Raderat" IS FALSE AND "Projektstatus" = 'Ändringsansökan' + fields: + age: SenasteUppdaterat offshore_rejected: where: | "Raderat" IS FALSE AND "Projektstatus" = 'Tillståndsansökan avslagen' + fields: + age: SenasteUppdaterat offshore_appealed: where: | "Raderat" IS FALSE AND "Projektstatus" = 'Överklagad' + fields: + age: SenasteUppdaterat offshore_applied: where: | "Raderat" IS FALSE AND "Projektstatus" = 'Tillståndsansökan inlämnad' + fields: + age: SenasteUppdaterat offshore_consultation: where: | "Raderat" IS FALSE AND "Projektstatus" = 'Samråd inför tillståndsansökan' + fields: + age: SenasteUppdaterat offshore_investigation: where: | "Raderat" IS FALSE AND "Projektstatus" = 'Inledande undersökningar' + fields: + age: SenasteUppdaterat offshore_revoked: where: | "Raderat" IS FALSE AND "Projektstatus" = 'Inte aktuell eller återkallad' + fields: + age: SenasteUppdaterat 'mrr:ut_metaller_industrimineral_ansokta': # https://resource.sgu.se/dokument/produkter/mineralrattigheter-beskrivning.pdf @@ -3678,7 +3720,10 @@ layers: path: mineralrattigheter.gpkg format: GPKG layername: ut_metaller_industrimineral_ansokta - publish: appl_met + publish: + appl_met: + fields: + age: appl_date 'mrr:ut_diamant_ansokta': # https://resource.sgu.se/dokument/produkter/mineralrattigheter-beskrivning.pdf @@ -3720,7 +3765,10 @@ layers: path: mineralrattigheter.gpkg format: GPKG layername: ut_diamant_ansokta - publish: appl_ogd + publish: + appl_ogd: + fields: + age: appl_date 'mrr:bearbetningskoncessioner_ansokta': # https://resource.sgu.se/dokument/produkter/mineralrattigheter-beskrivning.pdf @@ -3762,7 +3810,10 @@ layers: path: mineralrattigheter.gpkg format: GPKG layername: bearbetningskoncessioner_ansokta - publish: appl_ec + publish: + appl_ec: + fields: + age: appl_date 'mrr:markanvisningar_bk_ansokta': # https://resource.sgu.se/dokument/produkter/mineralrattigheter-beskrivning.pdf @@ -3861,7 +3912,10 @@ layers: licenceid: - replace: '-' with: null - publish: appr_met + publish: + appr_met: + fields: + age: dec_date 'mrr:ut_diamant_beviljade': # https://resource.sgu.se/dokument/produkter/mineralrattigheter-beskrivning.pdf @@ -3924,7 +3978,10 @@ layers: licenceid: - replace: '-' with: null - publish: appr_ogd + publish: + appr_ogd: + fields: + age: dec_date 'mrr:bearbetningskoncessioner_beviljade': # https://resource.sgu.se/dokument/produkter/mineralrattigheter-beskrivning.pdf @@ -3988,7 +4045,10 @@ layers: licenceid: - replace: '-' with: null - publish: appr_ec + publish: + appr_ec: + fields: + age: dec_date 'mrr:markanvisningar_bk_beviljade': # https://resource.sgu.se/dokument/produkter/mineralrattigheter-beskrivning.pdf @@ -4038,7 +4098,10 @@ layers: licenceid: - replace: '-' with: null - publish: appr_dl + publish: + appr_dl: + fields: + age: dec_date 'mrr:ut_metaller_industrimineral_forbud': # https://resource.sgu.se/dokument/produkter/mineralrattigheter-beskrivning.pdf diff --git a/export_mvt.py b/export_mvt.py index d19909c..31c7044 100644 --- a/export_mvt.py +++ b/export_mvt.py @@ -121,6 +121,7 @@ def exportSourceLayer(ds_src : gdal.Dataset, lyr_src : ogr.Layer, lyr_dst : ogr.Layer, layerdef : dict[str,Any], + fieldMap : tuple[list[str],list[int]], extent : ogr.Geometry|None = None) -> int: """Export a source layer.""" count0 = -1 @@ -168,7 +169,7 @@ def exportSourceLayer(ds_src : gdal.Dataset, spatialFilter = getSpatialFilterFromGeometry(extent, srs_src) transform_geometry = layerdef.get('transform-geometry', None) - columns = [ 'm.' + escape_identifier(lyr_src.GetFIDColumn()) ] + columns = [ 'm.' + escape_identifier(lyr_src.GetFIDColumn()) ] + fieldMap[0] geomFieldName_esc = escape_identifier(geomField.GetName()) if transform_geometry is None: columns.append('m.' + geomFieldName_esc) @@ -194,16 +195,26 @@ def exportSourceLayer(ds_src : gdal.Dataset, logging.debug('Source layer "%s" has %d features, of which %d are to be exported', layername, count0, count1) + fieldMap = fieldMap[1] + logging.debug('Field map: %s', str(fieldMap)) + + geom_type = lyr_src.GetGeomType() + bFlatten = geom_type == ogr.wkbUnknown or ogr.GT_HasM(geom_type) or ogr.GT_HasZ(geom_type) + bTransform = bFlatten or ct is not None + feature_count = 0 defn_dst = lyr_dst.GetLayerDefn() feature = lyr_src.GetNextFeature() while feature is not None: - geom = feature.GetGeometryRef().Clone() - if ct is not None and geom.Transform(ct) != ogr.OGRERR_NONE: - raise RuntimeError('Could not apply coordinate transformation') - geom.FlattenTo2D() feature2 = ogr.Feature(defn_dst) - feature2.SetGeometryDirectly(geom) + feature2.SetFromWithMap(feature, False, fieldMap) + if bTransform: + geom = feature2.GetGeometryRef() + if ct is not None and geom.Transform(ct) != ogr.OGRERR_NONE: + raise RuntimeError('Could not apply coordinate transformation') + if bFlatten: + geom.FlattenTo2D() + feature2.SetGeometryDirectly(geom) feature2.SetFID(feature.GetFID()) if lyr_dst.CreateFeature(feature2) != ogr.OGRERR_NONE: raise RuntimeError(f'Could not transfer source feature #{feature.GetFID()}') @@ -351,7 +362,85 @@ def exportMetadata(basedir : Path, data : dict[str,Any], finally: os.close(fd) -# pylint: disable-next=too-many-branches, too-many-statements +def getFieldMap(lyr_dst : ogr.Layer, lyr_src : ogr.Layer, + drv_src : gdal.Driver, + fieldMap : dict[str,str]|None) -> tuple[list[str],list[int]]: + """Create fields on the destination MVT layer, and return a list of + column statements along with a field map for the MVT export.""" + if fieldMap is None or len(fieldMap) == 0: + return [], [] + + if not lyr_dst.TestCapability(ogr.OLCCreateField): + raise RuntimeError(f'Destination layer "{lyr_dst.GetName()}" lacks ' + 'field creation capability') + + columns = {} + defn_src = lyr_src.GetLayerDefn() + for fld_dst, fld_src in fieldMap.items(): + idx_src = defn_src.GetFieldIndex(fld_src) + if idx_src < 0: + raise RuntimeError(f'Source layer "{lyr_src.GetName()}" has no field named "{fld_src}"') + + defn_dst = ogr.FieldDefn() + defn_src_fld = defn_src.GetFieldDefn(idx_src) + if fld_dst == 'age': + if defn_src_fld.GetType() not in (ogr.OFTDate, ogr.OFTDateTime): + raise RuntimeError(f'Field "{fld_src}" of source layer "{lyr_src.GetName()}"' + ' has type ' + ogr.GetFieldTypeName(defn_src_fld.GetType()) + + ' (Date or DateTime expected)') + defn_dst.SetType(ogr.OFTInteger) + # signed int16 allows expressing dates from 1880-04-15 to 2059-09-18 + # which should be more than enough (it's not clear if the MVT format takes + # advantage of the reduced storage though) + defn_dst.SetSubType(ogr.OFSTInt16) + + # TODO[GDAL >=3.9] use lyr_src.GetDataset().GetDriver() + if drv_src.ShortName == 'PostgreSQL': + column = 'CAST(m.' + escape_identifier(fld_src) + column += ' - date \'1970-01-01\' AS smallint)' + elif drv_src.ShortName in ('SQLite', 'GPKG'): + column = 'CAST(floor(julianday(m.' + escape_identifier(fld_src) + ')' + column += ' - 2440587.5) AS smallint)' + else: + raise NotImplementedError(f'Unsupported source driver {drv_src.ShortName} for ' + f'field "{fld_src}" (MVT field "{fld_dst}")') + + else: + raise NotImplementedError(f'Destination MVT field "{fld_dst}"') + + columns[fld_dst] = column + + defn_dst.SetName(fld_dst) + defn_dst.SetNullable(defn_src_fld.IsNullable()) + logging.debug('Create output field "%s" with type=%s, subtype=%s, nullable=%d', + defn_dst.GetName(), + ogr.GetFieldTypeName(defn_dst.GetType()), + ogr.GetFieldSubTypeName(defn_dst.GetSubType()), + defn_dst.IsNullable()) + + if lyr_dst.CreateField(defn_dst, approx_ok=False) != gdal.CE_None: + raise RuntimeError(f'Could not create field "{fld_dst}" ' + f'in destination MVT layer "{lyr_dst.GetName()}"') + + indices = {} + defn_dst = lyr_dst.GetLayerDefn() + for i in range(defn_dst.GetFieldCount()): + fld = defn_dst.GetFieldDefn(i) + name = fld.GetName() + if name in columns: + indices[name] = i + else: + logging.warning('Destination layer has unknown field #%d "%s"', i, name) + + ret = [None] * len(columns) + fieldMap = [-1] * defn_dst.GetFieldCount() + for idx, name in enumerate(columns.keys()): + i = indices[name] # intentionally crash if we didn't create that field + fieldMap[i] = idx + ret[idx] = columns[name] + ' AS ' + escape_identifier(name) + return (ret, fieldMap) + +# pylint: disable-next=too-many-branches, too-many-locals, too-many-statements def exportMVT(ds : gdal.Dataset, layers : dict[str,dict[str,Any]], sources : dict[str,Any], @@ -436,10 +525,15 @@ def exportMVT(ds : gdal.Dataset, if lyr_dst is None: raise RuntimeError(f'Could not create destination layer "{layername}"') + fieldMap = getFieldMap(lyr_dst, lyr_src, drv_src=ds.GetDriver(), + fieldMap=layerdef.get('fields', None)) + # TODO: GDAL exports features to a temporary SQLite database even though the source # is PostGIS hence is able to generate MVT with ST_AsMVT(). Letting PostGIS generate # tiles is likely to speed up things. - feature_count += exportSourceLayer(ds, lyr_src, lyr_dst, layerdef, extent=extent) + feature_count += exportSourceLayer(ds, lyr_src, lyr_dst, layerdef, + fieldMap=fieldMap, + extent=extent) layer_count += 1 lyr_dst = None lyr_src = None |