diff options
Diffstat (limited to 'webmap-import')
| -rwxr-xr-x | webmap-import | 180 |
1 files changed, 55 insertions, 125 deletions
diff --git a/webmap-import b/webmap-import index c1d07a3..9f9fdca 100755 --- a/webmap-import +++ b/webmap-import @@ -18,7 +18,9 @@ # along with this program. If not, see <https://www.gnu.org/licenses/>. #---------------------------------------------------------------------- +from os import O_WRONLY, O_CREAT, O_TRUNC, O_CLOEXEC import os +from fcntl import flock, LOCK_EX import logging import argparse import tempfile @@ -36,7 +38,6 @@ from osgeo.gdalconst import ( OF_VERBOSE_ERROR as GDAL_OF_VERBOSE_ERROR, CE_None as GDAL_CE_None, DCAP_CREATE as GDAL_DCAP_CREATE, - DCAP_VECTOR as GDAL_DCAP_VECTOR, DCAP_DEFAULT_FIELDS as GDAL_DCAP_DEFAULT_FIELDS, DCAP_NOTNULL_FIELDS as GDAL_DCAP_NOTNULL_FIELDS, DCAP_UNIQUE_FIELDS as GDAL_DCAP_UNIQUE_FIELDS, @@ -45,40 +46,13 @@ import osgeo.gdalconst as gdalconst gdal.UseExceptions() import common - -# Wrapper around gdal.MajorObject.GetMetadataItem(name) -def getMetadataItem(o, k): - v = o.GetMetadataItem(k) - if v is not None and isinstance(v, str): - return v.upper() == 'YES' - else: - return False - -# Return kwargs and driver for OpenEx() -def setOpenExArgs(option_dict, flags=0): - kwargs = { 'nOpenFlags': GDAL_OF_VECTOR | flags } - - fmt = option_dict.get('format', None) - if fmt is None: - drv = None - else: - drv = gdal.GetDriverByName(fmt) - if drv is None: - raise Exception(f'Unknown driver name "{fmt}"') - elif not getMetadataItem(drv, GDAL_DCAP_VECTOR): - raise Exception(f'Driver "{drv.ShortName}" has no vector capabilities') - kwargs['allowed_drivers'] = [ drv.ShortName ] - - oo = option_dict.get('open-options', None) - if oo is not None: - kwargs['open_options'] = [ k + '=' + str(v) for k, v in oo.items() ] - return kwargs, drv +from common import gdalSetOpenExArgs, gdalGetMetadataItem, gdalVersionMin, escapeIdentifier # Open and return the output DS. It is created if create=False or # create-options is a non-empty dictionary. def openOutputDS(def_dict): path = def_dict['path'] - kwargs, drv = setOpenExArgs(def_dict, flags=GDAL_OF_UPDATE|GDAL_OF_VERBOSE_ERROR) + kwargs, drv = gdalSetOpenExArgs(def_dict, flags=GDAL_OF_UPDATE|GDAL_OF_VERBOSE_ERROR) try: logging.debug('OpenEx(%s, %s)', path, str(kwargs)) return gdal.OpenEx(path, **kwargs) @@ -109,7 +83,7 @@ def openOutputDS(def_dict): if not def_dict.get('create', False) and dsco is None: # not configured for creation raise e - if drv is None or not getMetadataItem(drv, GDAL_DCAP_CREATE): + if drv is None or not gdalGetMetadataItem(drv, GDAL_DCAP_CREATE): # not capable of creation raise e @@ -347,13 +321,13 @@ def setFieldIf(cond, attrName, val, data, fldName, drvName, log=logging.warning) # constraints.) def validateSchema(layers, drvo=None, lco_defaults=None): # Cf. https://github.com/OSGeo/gdal/blob/master/NEWS.md - if common.gdal_version_min(maj=3, min=7): + if gdalVersionMin(maj=3, min=7): # list of capability flags supported by the CreateField() API drvoFieldDefnFlags = drvo.GetMetadataItem(gdalconst.DMD_CREATION_FIELD_DEFN_FLAGS) drvoFieldDefnFlags = drvoFieldDefnFlags.split(' ') if drvoFieldDefnFlags is not None else [] drvoSupportsFieldComment = 'Comment' in drvoFieldDefnFlags # GetTZFlag()/SetTZFlag() and OGR_TZFLAG_* constants added in 3.8.0 - hasTZFlagSupport = common.gdal_version_min(maj=3, min=8) + hasTZFlagSupport = gdalVersionMin(maj=3, min=8) else: # list of flags supported by the OGRLayer::AlterFieldDefn() API drvoFieldDefnFlags = drvo.GetMetadataItem(gdalconst.DMD_ALTER_FIELD_DEFN_FLAGS) @@ -364,9 +338,9 @@ def validateSchema(layers, drvo=None, lco_defaults=None): # cache driver capabilities drvoSupportsFieldWidthPrecision = 'WidthPrecision' in drvoFieldDefnFlags - drvoSupportsFieldNullable = 'Nullable' in drvoFieldDefnFlags and getMetadataItem(drvo, GDAL_DCAP_NOTNULL_FIELDS) - drvoSupportsFieldUnique = 'Unique' in drvoFieldDefnFlags and getMetadataItem(drvo, GDAL_DCAP_UNIQUE_FIELDS) - drvoSupportsFieldDefault = 'Default' in drvoFieldDefnFlags and getMetadataItem(drvo, GDAL_DCAP_DEFAULT_FIELDS) + drvoSupportsFieldNullable = 'Nullable' in drvoFieldDefnFlags and gdalGetMetadataItem(drvo, GDAL_DCAP_NOTNULL_FIELDS) + drvoSupportsFieldUnique = 'Unique' in drvoFieldDefnFlags and gdalGetMetadataItem(drvo, GDAL_DCAP_UNIQUE_FIELDS) + drvoSupportsFieldDefault = 'Default' in drvoFieldDefnFlags and gdalGetMetadataItem(drvo, GDAL_DCAP_DEFAULT_FIELDS) drvoSupportsFieldAlternativeName = 'AlternativeName' in drvoFieldDefnFlags for layername, layerdef in layers.items(): @@ -448,62 +422,6 @@ def validateSchema(layers, drvo=None, lco_defaults=None): fields[idx] = fld_def2 -# Return the decoded Spatial Reference System -def getSRS(srs_str): - if srs_str is None: - return - srs = osr.SpatialReference() - if srs_str.startswith('EPSG:'): - code = int(srs_str.removeprefix('EPSG:')) - srs.ImportFromEPSG(code) - else: - raise Exception(f'Unknown SRS {srs_str}') - logging.debug('Default SRS: "%s" (%s)', srs.ExportToProj4(), srs.GetName()) - return srs - -# Convert extent [minX, minY, maxX, maxY] into a polygon and assign the -# given SRS. Like apps/ogr2ogr_lib.cpp, we segmentize the polygon to -# make sure it is sufficiently densified when transforming to source -# layer SRS for spatial filtering. -def getExtent(extent, srs=None): - if extent is None: - return - - if not (isinstance(extent, list) or isinstance(extent, tuple)) or len(extent) != 4: - raise Exception(f'Invalid extent {extent}') - elif srs is None: - raise Exception('Configured extent but no SRS') - - logging.debug('Configured extent in %s: %s', - srs.GetName(), ', '.join(map(str, extent))) - - ring = ogr.Geometry(ogr.wkbLinearRing) - ring.AddPoint_2D(extent[0], extent[1]) - ring.AddPoint_2D(extent[2], extent[1]) - ring.AddPoint_2D(extent[2], extent[3]) - ring.AddPoint_2D(extent[0], extent[3]) - ring.AddPoint_2D(extent[0], extent[1]) - - polygon = ogr.Geometry(ogr.wkbPolygon) - polygon.AddGeometry(ring) - - # we expressed extent as minX, minY, maxX, maxY (easting/northing - # ordered, i.e., in traditional GIS order) - srs2 = srs.Clone() - srs2.SetAxisMappingStrategy(osr.OAMS_TRADITIONAL_GIS_ORDER) - polygon.AssignSpatialReference(srs2) - polygon.TransformTo(srs) - - segment_distance_metre = 10 * 1000 - if srs.IsGeographic(): - dfMaxLength = segment_distance_metre / math.radians(srs.GetSemiMajor()) - polygon.Segmentize(dfMaxLength) - elif srs.IsProjected(): - dfMaxLength = segment_distance_metre / srs.GetLinearUnits() - polygon.Segmentize(dfMaxLength) - - return polygon - # Validate the output layer against the provided SRS and creation options def validateOutputLayer(lyr, srs=None, options=None): ok = True @@ -675,6 +593,9 @@ def createOutputLayer(ds, layername, srs=None, options=None): lyr = dso.CreateLayer(layername, **kwargs) if lyr is None: raise Exception(f'Could not create destination layer "{layername}"') + # TODO use CreateLayerFromGeomFieldDefn() from ≥v3.9 as it's not + # possible to toggle the geomfield's nullable property after fact + # otherwise fields = options['fields'] if len(fields) > 0 and not lyr.TestCapability(ogr.OLCCreateField): @@ -797,14 +718,6 @@ def setOutputFieldMap(defn, sources): rule['replace'] = re.compile(rule['replace']) rules[idx] = ( rule['replace'], rule['with'] ) -# Escape the given identifier, cf. -# swig/python/gdal-utils/osgeo_utils/samples/validate_gpkg.py:_esc_id() -def escapeIdentifier(identifier): - if '\x00' in identifier: - raise Exception(f'Invalid identifier "{identifier}"') - # SQL:1999 delimited identifier - return '"' + identifier.replace('"', '""') + '"' - # Clear the given layer (wipe all its features) def clearLayer(ds, lyr): n = -1 @@ -936,7 +849,7 @@ def setFieldMapValue(fld, idx, val): # while we want a single transaction for the entire desination layer, # including truncation, source imports, and metadata changes. def importSource2(lyr_dst, path, args={}, basedir=None, extent=None): - kwargs, _ = setOpenExArgs(args, flags=GDAL_OF_READONLY|GDAL_OF_VERBOSE_ERROR) + kwargs, _ = gdalSetOpenExArgs(args, flags=GDAL_OF_READONLY|GDAL_OF_VERBOSE_ERROR) path2 = path if basedir is None else str(basedir.joinpath(path)) logging.debug('OpenEx(%s, %s)', path2, str(kwargs)) @@ -993,6 +906,7 @@ def importSource2(lyr_dst, path, args={}, basedir=None, extent=None): fields = args['field-map'] fieldSet = set() + ignoredFieldNames = [] for i in range(fieldCount): fld = defn.GetFieldDefn(i) fldName = fld.GetName() @@ -1003,6 +917,7 @@ def importSource2(lyr_dst, path, args={}, basedir=None, extent=None): # call SetIgnored() on unwanted source fields logging.debug('Set Ignored=True on output field "%s"', fldName) fld.SetIgnored(True) + ignoredFieldNames.append(fldName) count0 = -1 if lyr.TestCapability(ogr.OLCFastFeatureCount): @@ -1035,11 +950,14 @@ def importSource2(lyr_dst, path, args={}, basedir=None, extent=None): if count0 >= 0: if count1 >= 0: - logging.info('Source layer "%s" has %d features (of which %d within extent)', + logging.info('Source layer "%s" has %d features (%d of which intersecting extent)', layername, count0, count1) else: logging.info('Source layer "%s" has %d features', layername, count0) + logging.info('Ignored fields from source layer: %s', + '-' if len(ignoredFieldNames) == 0 else ', '.join(ignoredFieldNames)) + # build a list of triplets (field index, replacement_for_null, [(from_value, to_value), …]) valueMap = [] for fldName, rules in args.get('value-map', {}).items(): @@ -1133,27 +1051,31 @@ def importSource2(lyr_dst, path, args={}, basedir=None, extent=None): feature2.SetFromWithMap(feature, False, fieldMap) geom = feature2.GetGeometryRef() - if ct is not None and geom.Transform(ct) != ogr.OGRERR_NONE: - raise Exception('Could not apply coordinate transformation') - - eGType = geom.GetGeometryType() - if eGType != eGType_dst and not dGeomIsUnknown: - # Promote to multi, cf. apps/ogr2ogr_lib.cpp:ConvertType() - eGType2 = eGType - if eGType == ogr.wkbTriangle or eGType == ogr.wkbTIN or eGType == ogr.wkbPolyhedralSurface: - eGType2 = ogr.wkbMultiPolygon - elif not ogr.GT_IsSubClassOf(eGType, ogr.wkbGeometryCollection): - eGType2 = ogr.GT_GetCollection(eGType) - - eGType2 = ogr.GT_SetModifier(eGType2, eGType_dst_HasZ, eGType_dst_HasM) - if eGType2 == eGType_dst: - mismatch[eGType] = mismatch.get(eGType, 0) + 1 - geom = ogr.ForceTo(geom, eGType_dst) - # TODO call MakeValid()? - else: - raise Exception(f'Conversion from {ogr.GeometryTypeToName(eGType)} ' - f'to {ogr.GeometryTypeToName(eGType_dst)} not implemented') - feature2.SetGeometryDirectly(geom) + if geom is None: + if eGType_dst != ogr.wkbNone: + logging.warning('Source feature #%d has no geometry, trying to transfer anyway', feature.GetFID()) + else: + if ct is not None and geom.Transform(ct) != ogr.OGRERR_NONE: + raise Exception('Could not apply coordinate transformation') + + eGType = geom.GetGeometryType() + if eGType != eGType_dst and not dGeomIsUnknown: + # Promote to multi, cf. apps/ogr2ogr_lib.cpp:ConvertType() + eGType2 = eGType + if eGType == ogr.wkbTriangle or eGType == ogr.wkbTIN or eGType == ogr.wkbPolyhedralSurface: + eGType2 = ogr.wkbMultiPolygon + elif not ogr.GT_IsSubClassOf(eGType, ogr.wkbGeometryCollection): + eGType2 = ogr.GT_GetCollection(eGType) + + eGType2 = ogr.GT_SetModifier(eGType2, eGType_dst_HasZ, eGType_dst_HasM) + if eGType2 == eGType_dst: + mismatch[eGType] = mismatch.get(eGType, 0) + 1 + geom = ogr.ForceTo(geom, eGType_dst) + # TODO call MakeValid()? + else: + raise Exception(f'Conversion from {ogr.GeometryTypeToName(eGType)} ' + f'to {ogr.GeometryTypeToName(eGType_dst)} not implemented') + feature2.SetGeometryDirectly(geom) if lyr_dst.CreateFeature(feature2) != ogr.OGRERR_NONE: raise Exception(f'Could not transfer source feature #{feature.GetFID()}') @@ -1189,6 +1111,8 @@ if __name__ == '__main__': help=f'cache directory (default: {os.curdir})') parser.add_argument('--debug', action='count', default=0, help=argparse.SUPPRESS) + parser.add_argument('--lockfile', default=None, + help=f'obtain an exclusive lock before starting unpacking and importing') parser.add_argument('groupname', nargs='*', help='group layer name(s) to process') args = parser.parse_args() @@ -1237,8 +1161,14 @@ if __name__ == '__main__': lco_defaults=common.config['dataset'].get('create-layer-options', None)) # get configured Spatial Reference System and extent - srs = getSRS(common.config.get('SRS', None)) - extent = getExtent(common.config.get('extent', None), srs=srs) + srs = common.getSRS(common.config.get('SRS', None)) + extent = common.getExtent(common.config.get('extent', None), srs=srs)[0] + + if args.lockfile is not None: + # obtain an exclusive lock and don't release it until exiting the program + lock_fd = os.open(args.lockfile, O_WRONLY|O_CREAT|O_TRUNC|O_CLOEXEC, mode=0o644) + logging.debug('flock("%s", LOCK_EX)', args.lockfile) + flock(lock_fd, LOCK_EX) cachedir = Path(args.cachedir) if args.cachedir is not None else None rv = 0 |