diff options
| -rw-r--r-- | config.yml | 80 | ||||
| -rw-r--r-- | schema.sql | 50 | ||||
| -rwxr-xr-x | webmap-import | 99 |
3 files changed, 205 insertions, 24 deletions
@@ -228,8 +228,21 @@ layers: # - field_name1 # - field_name2 # field-map: +# source_field_name1: field_name1 # source_field_name2: field_name2 -# source_field_name2: field_name2 +# +# # Mapping of (source) field values, to replace particular +# # values with something else. Useful for instance to replace +# # the empty string with NULL. +# value-map: +# source_field_name1: +# - from: '' +# to: null +# - from: 'foo' +# to: 'bar' +# source_field_name2: +# - from: 0 +# to: 1 'lansyta': description: Sveriges län (Lantmäteriet) @@ -521,9 +534,9 @@ layers: - name: Beteckn type: String width: 12 - #unique: true - # TODO: map "Visas ej" to NULL - nullable: false + unique: true + # obfuscated codes are mapped to NULL + nullable: true #comment: ärendebeteckning - name: ArendeAr type: Integer @@ -611,6 +624,10 @@ layers: Forebild: Forebild Efterbild: Efterbild Arealha: ArealHa + value-map: + Beteckn: + - from: 'Visas ej' + to: null - source: download: @@ -642,6 +659,10 @@ layers: Forebild: Forebild Efterbild: Efterbild Arealha: ArealHa + value-map: + Beteckn: + - from: 'Visas ej' + to: null - source: download: @@ -673,6 +694,10 @@ layers: Forebild: Forebild Efterbild: Efterbild Arealha: ArealHa + value-map: + Beteckn: + - from: 'Visas ej' + to: null - source: download: @@ -702,6 +727,10 @@ layers: Forebild: Forebild Efterbild: Efterbild Arealha: ArealHa + value-map: + Beteckn: + - from: 'Visas ej' + to: null 'st:betesomraden': source: @@ -807,8 +836,7 @@ layers: #comment: när tillståndets giltighet slutar - name: DiaryNr type: String - # TODO map empty string to NULL - #unique: true + unique: true width: 16 - name: LastUpdated # TODO convert to Date @@ -832,6 +860,10 @@ layers: 'Valid to': ValidTo 'Diary nr': DiaryNr 'Last updated': LastUpdated + value-map: + 'Diary nr': + - from: '' + to: null 'mrr:markanvisningar': description: Markanvisning till koncession (SGU) @@ -845,12 +877,10 @@ layers: #comment: benämning på området - name: DecisionDate type: Date - # TODO map empty string to NULL #comment: beslutsdatum - name: DiaryNr type: String - # TODO map empty string to NULL - #unique: true + unique: true width: 16 - name: LastUpdated # TODO convert to Date @@ -871,6 +901,13 @@ layers: 'Decision date': DecisionDate 'Diary nr': DiaryNr 'Last updated': LastUpdated + value-map: + 'Decision date': + - from: '' + to: null + 'Diary nr': + - from: '' + to: null 'mrr:mineral_applied': description: Undersökningstillstånd, metallar och mineral, ansökta (SGU) @@ -959,8 +996,7 @@ layers: #comment: när tillståndets giltighet slutar - name: DiaryNr type: String - # TODO map empty string to NULL - #unique: true + unique: true width: 16 - name: LastUpdated # TODO convert to Date @@ -985,6 +1021,10 @@ layers: 'Valid to': ValidTo 'Diary nr': DiaryNr 'Last updated': LastUpdated + value-map: + 'Diary nr': + - from: '' + to: null # 'mrr:mineral_expired': # source: @@ -1100,8 +1140,7 @@ layers: #comment: när tillståndets giltighet slutar - name: DiaryNr type: String - # TODO map empty string to NULL - #unique: true + unique: true width: 16 - name: LastUpdated # TODO convert to Date @@ -1126,6 +1165,10 @@ layers: 'Valid to': ValidTo 'Diary nr': DiaryNr 'Last updated': LastUpdated + value-map: + 'Diary nr': + - from: '' + to: null 'mrr:torvkoncessioner': description: Torvkoncessioner (SGU) @@ -1140,7 +1183,6 @@ layers: #comment: benämning på området - name: Mineral type: String - # TODO map empty string to NULL width: 254 #comment: koncessionsmineral - name: Owner @@ -1158,8 +1200,7 @@ layers: #comment: när tillståndets giltighet slutar - name: DiaryNr type: String - # TODO map empty string to NULL - #unique: true + unique: true width: 16 - name: LastUpdated # TODO convert to Date @@ -1183,3 +1224,10 @@ layers: 'Valid to': ValidTo 'Diary nr': DiaryNr 'Last updated': LastUpdated + value-map: + 'Mineral': + - from: '' + to: null + 'Diary nr': + - from: '' + to: null @@ -591,7 +591,7 @@ ALTER SEQUENCE postgis."sks:AvverkAnm_ogc_fid_seq" OWNED BY postgis."sks:AvverkA CREATE TABLE postgis."sks:UtfordAvverk" ( ogc_fid bigint NOT NULL, "OBJECTID" integer NOT NULL, - "Beteckn" character(12) NOT NULL, + "Beteckn" character(12), "ArendeAr" smallint NOT NULL, "Avverktyp" character varying(254) NOT NULL, "Skogstyp" character varying(254) NOT NULL, @@ -795,6 +795,14 @@ ALTER TABLE ONLY postgis."mrr:bearbetningskoncessioner_applied" -- +-- Name: mrr:bearbetningskoncessioner_approved mrr:bearbetningskoncessioner_approved_DiaryNr_key; Type: CONSTRAINT; Schema: postgis; Owner: webmap_import +-- + +ALTER TABLE ONLY postgis."mrr:bearbetningskoncessioner_approved" + ADD CONSTRAINT "mrr:bearbetningskoncessioner_approved_DiaryNr_key" UNIQUE ("DiaryNr"); + + +-- -- Name: mrr:bearbetningskoncessioner_approved mrr:bearbetningskoncessioner_approved_Name_key; Type: CONSTRAINT; Schema: postgis; Owner: webmap_import -- @@ -811,6 +819,14 @@ ALTER TABLE ONLY postgis."mrr:bearbetningskoncessioner_approved" -- +-- Name: mrr:markanvisningar mrr:markanvisningar_DiaryNr_key; Type: CONSTRAINT; Schema: postgis; Owner: webmap_import +-- + +ALTER TABLE ONLY postgis."mrr:markanvisningar" + ADD CONSTRAINT "mrr:markanvisningar_DiaryNr_key" UNIQUE ("DiaryNr"); + + +-- -- Name: mrr:markanvisningar mrr:markanvisningar_pkey; Type: CONSTRAINT; Schema: postgis; Owner: webmap_import -- @@ -843,6 +859,14 @@ ALTER TABLE ONLY postgis."mrr:mineral_applied" -- +-- Name: mrr:mineral_approved mrr:mineral_approved_DiaryNr_key; Type: CONSTRAINT; Schema: postgis; Owner: webmap_import +-- + +ALTER TABLE ONLY postgis."mrr:mineral_approved" + ADD CONSTRAINT "mrr:mineral_approved_DiaryNr_key" UNIQUE ("DiaryNr"); + + +-- -- Name: mrr:mineral_approved mrr:mineral_approved_LicenceID_key; Type: CONSTRAINT; Schema: postgis; Owner: webmap_import -- @@ -891,6 +915,14 @@ ALTER TABLE ONLY postgis."mrr:olja_gas_diamant_applied" -- +-- Name: mrr:olja_gas_diamant_approved mrr:olja_gas_diamant_approved_DiaryNr_key; Type: CONSTRAINT; Schema: postgis; Owner: webmap_import +-- + +ALTER TABLE ONLY postgis."mrr:olja_gas_diamant_approved" + ADD CONSTRAINT "mrr:olja_gas_diamant_approved_DiaryNr_key" UNIQUE ("DiaryNr"); + + +-- -- Name: mrr:olja_gas_diamant_approved mrr:olja_gas_diamant_approved_LicenceID_key; Type: CONSTRAINT; Schema: postgis; Owner: webmap_import -- @@ -915,6 +947,14 @@ ALTER TABLE ONLY postgis."mrr:olja_gas_diamant_approved" -- +-- Name: mrr:torvkoncessioner mrr:torvkoncessioner_DiaryNr_key; Type: CONSTRAINT; Schema: postgis; Owner: webmap_import +-- + +ALTER TABLE ONLY postgis."mrr:torvkoncessioner" + ADD CONSTRAINT "mrr:torvkoncessioner_DiaryNr_key" UNIQUE ("DiaryNr"); + + +-- -- Name: mrr:torvkoncessioner mrr:torvkoncessioner_Name_key; Type: CONSTRAINT; Schema: postgis; Owner: webmap_import -- @@ -955,6 +995,14 @@ ALTER TABLE ONLY postgis."sks:AvverkAnm" -- +-- Name: sks:UtfordAvverk sks:UtfordAvverk_Beteckn_key; Type: CONSTRAINT; Schema: postgis; Owner: webmap_import +-- + +ALTER TABLE ONLY postgis."sks:UtfordAvverk" + ADD CONSTRAINT "sks:UtfordAvverk_Beteckn_key" UNIQUE ("Beteckn"); + + +-- -- Name: sks:UtfordAvverk sks:UtfordAvverk_OBJECTID_key; Type: CONSTRAINT; Schema: postgis; Owner: webmap_import -- diff --git a/webmap-import b/webmap-import index e9ae302..e978f34 100755 --- a/webmap-import +++ b/webmap-import @@ -730,8 +730,9 @@ def setOutputFieldMap(defn, sources): fieldMap[fldName] = i for source in sources: - src = source['source']['path'] - fieldMap2 = source['import'].get('field-map', None) + source_import = source['import'] + + fieldMap2 = source_import.get('field-map', None) if fieldMap2 is None: fieldMap2 = fieldMap else: @@ -744,10 +745,22 @@ def setOutputFieldMap(defn, sources): if i is None: raise Exception(f'Ouput layer has no field named "{ofld}"') fieldMap2[ifld] = i - - source['import']['field-map'] = fieldMap2 - - return fieldMap + source_import['field-map'] = fieldMap2 + + # validate field value mapping + valueMap = source_import.get('value-map', None) + if valueMap is not None: + for fldName, rules in valueMap.items(): + if rules is None: + continue + if not isinstance(rules, list): + rules = [rules] + for idx, rule in enumerate(rules): + if rule is None or not isinstance(rule, dict): + raise Exception(f'Field "{fldName}" has invalid rule #{idx}: {rule}') + if 'from' not in rule or 'to' not in rule or len(rule) != 2: + raise Exception(f'Field "{fldName}" has invalid rule #{idx}: {rule}') + rules[idx] = ( rule['from'], rule['to'] ) # Escape the given identifier, cf. # swig/python/gdal-utils/osgeo_utils/samples/validate_gpkg.py:_esc_id() @@ -856,6 +869,31 @@ def importSource(lyr, path=None, unar=None, args={}, cachedir=None, extent=None) return importSource2(lyr, ds_srcpath, args=args, basedir=Path(tmpdir), extent=extent) +# Validate field value mapping +def setFieldMapValue(fld, idx, val): + if val is None: + if not fld.IsNullable(): + logging.warning('Field "%s" is not NULLable but remaps NULL', fld.GetName()) + return None + + fldType = fld.GetType() + if fldType == ogr.OFTInteger or fldType == ogr.OFTInteger64: + if isinstance(val, int): + return val + elif fldType == ogr.OFTString: + if isinstance(val, str): + return val + elif fldType == ogr.OFTBinary: + if isinstance(val, bytes): + return val + elif fldType == ogr.OFTReal: + if isinstance(val, int): + return float(val) + elif isinstance(val, float): + return val + + raise Exception(f'Field "{fld.GetName()}" mapping #{idx} has incompatible type for {ogr.GetFieldTypeName(fldType)}') + # Import a source layer (already extracted) # This is more or less like ogr2ogr/GDALVectorTranslate() but we roll # out our own (slower) version because GDALVectorTranslate() insists in @@ -927,7 +965,6 @@ def importSource2(lyr_dst, path, args={}, basedir=None, extent=None): # call SetIgnored() on unwanted source fields logging.debug('Set Ignored=True on output field "%s"', fldName) fld.SetIgnored(True) - defn = None count0 = -1 if lyr.TestCapability(ogr.OLCFastFeatureCount): @@ -965,6 +1002,32 @@ def importSource2(lyr_dst, path, args={}, basedir=None, extent=None): else: logging.info('Source layer "%s" has %d features', layername, count0) + # build a list of pairs (field index, mapping_dict) + valueMapLiteral = [] + for fldName, rules in args.get('value-map', {}).items(): + i = defn.GetFieldIndex(fldName) + if i < 0: + raise Exception(f'Source layer "{layername}" has no field named "{fldName}"') + if fieldMap[i] < 0: + logging.warning('Ignored source field "%s" has value map', fldName) + continue + + h = {} + fld = defn.GetFieldDefn(i) + for idx, (rFrom, rTo) in enumerate(rules): + # use fld for both from and to (the types must match, + # casting is not allowed in the mapping) + rFrom = setFieldMapValue(fld, idx, rFrom) + rTo = setFieldMapValue(fld, idx, rTo) + h[rFrom] = rTo + + if len(h) > 0: + valueMapLiteral.append((i, h)) + + bValueMapLiteral = len(valueMapLiteral) > 0 + + defn = None + defn_dst = lyr_dst.GetLayerDefn() eGType_dst = defn_dst.GetGeomType() eGType_dst_HasZ = ogr.GT_HasZ(eGType_dst) @@ -975,6 +1038,28 @@ def importSource2(lyr_dst, path, args={}, basedir=None, extent=None): mismatch = {} feature = lyr.GetNextFeature() while feature is not None: + if bValueMapLiteral: + for i, h in valueMapLiteral: + if not feature.IsFieldSet(i): + continue + elif feature.IsFieldNull(i): + if None in h: + v = h[None] + if v is not None: + # replace NULL with non-NULL value + feature.SetField(i, v) + continue + + v = feature.GetField(i) + if v in h: + v2 = h[v] + if v2 is None: + # replace non-NULL value with NULL + feature.SetFieldNull(i) + else: + # replace non-NULL value with non-NULL value + feature.SetField(i, v2) + feature2 = ogr.Feature(defn_dst) feature2.SetFromWithMap(feature, False, fieldMap) |