Add logic to replace field value literals.

And set them to NULL.

3 files changed, 205 insertions, 24 deletions

diff --git a/config.yml b/config.yml
index 6587643..450cc5f 100644
--- a/config.yml
+++ b/config.yml
@@ -228,8 +228,21 @@ layers:
 #            - field_name1
 #            - field_name2
 #          field-map:
+#            source_field_name1: field_name1
 #            source_field_name2: field_name2
-#            source_field_name2: field_name2
+#
+#          # Mapping of (source) field values, to replace particular
+#          # values with something else.  Useful for instance to replace
+#          # the empty string with NULL.
+#          value-map:
+#            source_field_name1:
+#              - from: ''
+#                to: null
+#              - from: 'foo'
+#                to: 'bar'
+#            source_field_name2:
+#              - from: 0
+#                to: 1
 
   'lansyta':
     description: Sveriges län (Lantmäteriet)
@@ -521,9 +534,9 @@ layers:
         - name: Beteckn
           type: String
           width: 12
-          #unique: true
-          # TODO: map "Visas ej" to NULL
-          nullable: false
+          unique: true
+          # obfuscated codes are mapped to NULL
+          nullable: true
           #comment: ärendebeteckning
         - name: ArendeAr
           type: Integer
@@ -611,6 +624,10 @@ layers:
             Forebild: Forebild
             Efterbild: Efterbild
             Arealha: ArealHa
+          value-map:
+            Beteckn:
+              - from: 'Visas ej'
+                to: null
 
       - source:
           download:
@@ -642,6 +659,10 @@ layers:
             Forebild: Forebild
             Efterbild: Efterbild
             Arealha: ArealHa
+          value-map:
+            Beteckn:
+              - from: 'Visas ej'
+                to: null
 
       - source:
           download:
@@ -673,6 +694,10 @@ layers:
             Forebild: Forebild
             Efterbild: Efterbild
             Arealha: ArealHa
+          value-map:
+            Beteckn:
+              - from: 'Visas ej'
+                to: null
 
       - source:
           download:
@@ -702,6 +727,10 @@ layers:
             Forebild: Forebild
             Efterbild: Efterbild
             Arealha: ArealHa
+          value-map:
+            Beteckn:
+              - from: 'Visas ej'
+                to: null
 
   'st:betesomraden':
     source:
@@ -807,8 +836,7 @@ layers:
           #comment: när tillståndets giltighet slutar
         - name: DiaryNr
           type: String
-          # TODO map empty string to NULL
-          #unique: true
+          unique: true
           width: 16
         - name: LastUpdated
           # TODO convert to Date
@@ -832,6 +860,10 @@ layers:
         'Valid to': ValidTo
         'Diary nr': DiaryNr
         'Last updated': LastUpdated
+      value-map:
+        'Diary nr':
+          - from: ''
+            to: null
 
   'mrr:markanvisningar':
     description: Markanvisning till koncession (SGU)
@@ -845,12 +877,10 @@ layers:
           #comment: benämning på området
         - name: DecisionDate
           type: Date
-          # TODO map empty string to NULL
           #comment: beslutsdatum
         - name: DiaryNr
           type: String
-          # TODO map empty string to NULL
-          #unique: true
+          unique: true
           width: 16
         - name: LastUpdated
           # TODO convert to Date
@@ -871,6 +901,13 @@ layers:
         'Decision date': DecisionDate
         'Diary nr': DiaryNr
         'Last updated': LastUpdated
+      value-map:
+        'Decision date':
+          - from: ''
+            to: null
+        'Diary nr':
+          - from: ''
+            to: null
 
   'mrr:mineral_applied':
     description: Undersökningstillstånd, metallar och mineral, ansökta (SGU)
@@ -959,8 +996,7 @@ layers:
           #comment: när tillståndets giltighet slutar
         - name: DiaryNr
           type: String
-          # TODO map empty string to NULL
-          #unique: true
+          unique: true
           width: 16
         - name: LastUpdated
           # TODO convert to Date
@@ -985,6 +1021,10 @@ layers:
         'Valid to': ValidTo
         'Diary nr': DiaryNr
         'Last updated': LastUpdated
+      value-map:
+        'Diary nr':
+          - from: ''
+            to: null
 
 #  'mrr:mineral_expired':
 #    source:
@@ -1100,8 +1140,7 @@ layers:
           #comment: när tillståndets giltighet slutar
         - name: DiaryNr
           type: String
-          # TODO map empty string to NULL
-          #unique: true
+          unique: true
           width: 16
         - name: LastUpdated
           # TODO convert to Date
@@ -1126,6 +1165,10 @@ layers:
         'Valid to': ValidTo
         'Diary nr': DiaryNr
         'Last updated': LastUpdated
+      value-map:
+        'Diary nr':
+          - from: ''
+            to: null
 
   'mrr:torvkoncessioner':
     description: Torvkoncessioner (SGU)
@@ -1140,7 +1183,6 @@ layers:
           #comment: benämning på området
         - name: Mineral
           type: String
-          # TODO map empty string to NULL
           width: 254
           #comment: koncessionsmineral
         - name: Owner
@@ -1158,8 +1200,7 @@ layers:
           #comment: när tillståndets giltighet slutar
         - name: DiaryNr
           type: String
-          # TODO map empty string to NULL
-          #unique: true
+          unique: true
           width: 16
         - name: LastUpdated
           # TODO convert to Date
@@ -1183,3 +1224,10 @@ layers:
         'Valid to': ValidTo
         'Diary nr': DiaryNr
         'Last updated': LastUpdated
+      value-map:
+        'Mineral':
+          - from: ''
+            to: null
+        'Diary nr':
+          - from: ''
+            to: null
diff --git a/schema.sql b/schema.sql
index b1ab6ad..07bda74 100644
--- a/schema.sql
+++ b/schema.sql
@@ -591,7 +591,7 @@ ALTER SEQUENCE postgis."sks:AvverkAnm_ogc_fid_seq" OWNED BY postgis."sks:AvverkA
 CREATE TABLE postgis."sks:UtfordAvverk" (
     ogc_fid bigint NOT NULL,
     "OBJECTID" integer NOT NULL,
-    "Beteckn" character(12) NOT NULL,
+    "Beteckn" character(12),
     "ArendeAr" smallint NOT NULL,
     "Avverktyp" character varying(254) NOT NULL,
     "Skogstyp" character varying(254) NOT NULL,
@@ -795,6 +795,14 @@ ALTER TABLE ONLY postgis."mrr:bearbetningskoncessioner_applied"
 
 
 --
+-- Name: mrr:bearbetningskoncessioner_approved mrr:bearbetningskoncessioner_approved_DiaryNr_key; Type: CONSTRAINT; Schema: postgis; Owner: webmap_import
+--
+
+ALTER TABLE ONLY postgis."mrr:bearbetningskoncessioner_approved"
+    ADD CONSTRAINT "mrr:bearbetningskoncessioner_approved_DiaryNr_key" UNIQUE ("DiaryNr");
+
+
+--
 -- Name: mrr:bearbetningskoncessioner_approved mrr:bearbetningskoncessioner_approved_Name_key; Type: CONSTRAINT; Schema: postgis; Owner: webmap_import
 --
 
@@ -811,6 +819,14 @@ ALTER TABLE ONLY postgis."mrr:bearbetningskoncessioner_approved"
 
 
 --
+-- Name: mrr:markanvisningar mrr:markanvisningar_DiaryNr_key; Type: CONSTRAINT; Schema: postgis; Owner: webmap_import
+--
+
+ALTER TABLE ONLY postgis."mrr:markanvisningar"
+    ADD CONSTRAINT "mrr:markanvisningar_DiaryNr_key" UNIQUE ("DiaryNr");
+
+
+--
 -- Name: mrr:markanvisningar mrr:markanvisningar_pkey; Type: CONSTRAINT; Schema: postgis; Owner: webmap_import
 --
 
@@ -843,6 +859,14 @@ ALTER TABLE ONLY postgis."mrr:mineral_applied"
 
 
 --
+-- Name: mrr:mineral_approved mrr:mineral_approved_DiaryNr_key; Type: CONSTRAINT; Schema: postgis; Owner: webmap_import
+--
+
+ALTER TABLE ONLY postgis."mrr:mineral_approved"
+    ADD CONSTRAINT "mrr:mineral_approved_DiaryNr_key" UNIQUE ("DiaryNr");
+
+
+--
 -- Name: mrr:mineral_approved mrr:mineral_approved_LicenceID_key; Type: CONSTRAINT; Schema: postgis; Owner: webmap_import
 --
 
@@ -891,6 +915,14 @@ ALTER TABLE ONLY postgis."mrr:olja_gas_diamant_applied"
 
 
 --
+-- Name: mrr:olja_gas_diamant_approved mrr:olja_gas_diamant_approved_DiaryNr_key; Type: CONSTRAINT; Schema: postgis; Owner: webmap_import
+--
+
+ALTER TABLE ONLY postgis."mrr:olja_gas_diamant_approved"
+    ADD CONSTRAINT "mrr:olja_gas_diamant_approved_DiaryNr_key" UNIQUE ("DiaryNr");
+
+
+--
 -- Name: mrr:olja_gas_diamant_approved mrr:olja_gas_diamant_approved_LicenceID_key; Type: CONSTRAINT; Schema: postgis; Owner: webmap_import
 --
 
@@ -915,6 +947,14 @@ ALTER TABLE ONLY postgis."mrr:olja_gas_diamant_approved"
 
 
 --
+-- Name: mrr:torvkoncessioner mrr:torvkoncessioner_DiaryNr_key; Type: CONSTRAINT; Schema: postgis; Owner: webmap_import
+--
+
+ALTER TABLE ONLY postgis."mrr:torvkoncessioner"
+    ADD CONSTRAINT "mrr:torvkoncessioner_DiaryNr_key" UNIQUE ("DiaryNr");
+
+
+--
 -- Name: mrr:torvkoncessioner mrr:torvkoncessioner_Name_key; Type: CONSTRAINT; Schema: postgis; Owner: webmap_import
 --
 
@@ -955,6 +995,14 @@ ALTER TABLE ONLY postgis."sks:AvverkAnm"
 
 
 --
+-- Name: sks:UtfordAvverk sks:UtfordAvverk_Beteckn_key; Type: CONSTRAINT; Schema: postgis; Owner: webmap_import
+--
+
+ALTER TABLE ONLY postgis."sks:UtfordAvverk"
+    ADD CONSTRAINT "sks:UtfordAvverk_Beteckn_key" UNIQUE ("Beteckn");
+
+
+--
 -- Name: sks:UtfordAvverk sks:UtfordAvverk_OBJECTID_key; Type: CONSTRAINT; Schema: postgis; Owner: webmap_import
 --
 
diff --git a/webmap-import b/webmap-import
index e9ae302..e978f34 100755
--- a/webmap-import
+++ b/webmap-import
@@ -730,8 +730,9 @@ def setOutputFieldMap(defn, sources):
         fieldMap[fldName] = i
 
     for source in sources:
-        src = source['source']['path']
-        fieldMap2 = source['import'].get('field-map', None)
+        source_import = source['import']
+
+        fieldMap2 = source_import.get('field-map', None)
         if fieldMap2 is None:
             fieldMap2 = fieldMap
         else:
@@ -744,10 +745,22 @@ def setOutputFieldMap(defn, sources):
                 if i is None:
                     raise Exception(f'Ouput layer has no field named "{ofld}"')
                 fieldMap2[ifld] = i
-
-        source['import']['field-map'] = fieldMap2
-
-    return fieldMap
+        source_import['field-map'] = fieldMap2
+
+        # validate field value mapping
+        valueMap = source_import.get('value-map', None)
+        if valueMap is not None:
+            for fldName, rules in valueMap.items():
+                if rules is None:
+                    continue
+                if not isinstance(rules, list):
+                    rules = [rules]
+                for idx, rule in enumerate(rules):
+                    if rule is None or not isinstance(rule, dict):
+                        raise Exception(f'Field "{fldName}" has invalid rule #{idx}: {rule}')
+                    if 'from' not in rule or 'to' not in rule or len(rule) != 2:
+                        raise Exception(f'Field "{fldName}" has invalid rule #{idx}: {rule}')
+                    rules[idx] = ( rule['from'], rule['to'] )
 
 # Escape the given identifier, cf.
 # swig/python/gdal-utils/osgeo_utils/samples/validate_gpkg.py:_esc_id()
@@ -856,6 +869,31 @@ def importSource(lyr, path=None, unar=None, args={}, cachedir=None, extent=None)
         return importSource2(lyr, ds_srcpath, args=args,
                     basedir=Path(tmpdir), extent=extent)
 
+# Validate field value mapping
+def setFieldMapValue(fld, idx, val):
+    if val is None:
+        if not fld.IsNullable():
+            logging.warning('Field "%s" is not NULLable but remaps NULL', fld.GetName())
+        return None
+
+    fldType = fld.GetType()
+    if fldType == ogr.OFTInteger or fldType == ogr.OFTInteger64:
+        if isinstance(val, int):
+            return val
+    elif fldType == ogr.OFTString:
+        if isinstance(val, str):
+            return val
+    elif fldType == ogr.OFTBinary:
+        if isinstance(val, bytes):
+            return val
+    elif fldType == ogr.OFTReal:
+        if isinstance(val, int):
+            return float(val)
+        elif isinstance(val, float):
+            return val
+
+    raise Exception(f'Field "{fld.GetName()}" mapping #{idx} has incompatible type for {ogr.GetFieldTypeName(fldType)}')
+
 # Import a source layer (already extracted)
 # This is more or less like ogr2ogr/GDALVectorTranslate() but we roll
 # out our own (slower) version because GDALVectorTranslate() insists in
@@ -927,7 +965,6 @@ def importSource2(lyr_dst, path, args={}, basedir=None, extent=None):
             # call SetIgnored() on unwanted source fields
             logging.debug('Set Ignored=True on output field "%s"', fldName)
             fld.SetIgnored(True)
-    defn = None
 
     count0 = -1
     if lyr.TestCapability(ogr.OLCFastFeatureCount):
@@ -965,6 +1002,32 @@ def importSource2(lyr_dst, path, args={}, basedir=None, extent=None):
         else:
             logging.info('Source layer "%s" has %d features', layername, count0)
 
+    # build a list of pairs (field index, mapping_dict)
+    valueMapLiteral = []
+    for fldName, rules in args.get('value-map', {}).items():
+        i = defn.GetFieldIndex(fldName)
+        if i < 0:
+            raise Exception(f'Source layer "{layername}" has no field named "{fldName}"')
+        if fieldMap[i] < 0:
+            logging.warning('Ignored source field "%s" has value map', fldName)
+            continue
+
+        h = {}
+        fld = defn.GetFieldDefn(i)
+        for idx, (rFrom, rTo) in enumerate(rules):
+            # use fld for both from and to (the types must match,
+            # casting is not allowed in the mapping)
+            rFrom = setFieldMapValue(fld, idx, rFrom)
+            rTo   = setFieldMapValue(fld, idx, rTo)
+            h[rFrom] = rTo
+
+        if len(h) > 0:
+            valueMapLiteral.append((i, h))
+
+    bValueMapLiteral = len(valueMapLiteral) > 0
+
+    defn = None
+
     defn_dst = lyr_dst.GetLayerDefn()
     eGType_dst = defn_dst.GetGeomType()
     eGType_dst_HasZ = ogr.GT_HasZ(eGType_dst)
@@ -975,6 +1038,28 @@ def importSource2(lyr_dst, path, args={}, basedir=None, extent=None):
     mismatch = {}
     feature = lyr.GetNextFeature()
     while feature is not None:
+        if bValueMapLiteral:
+            for i, h in valueMapLiteral:
+                if not feature.IsFieldSet(i):
+                    continue
+                elif feature.IsFieldNull(i):
+                    if None in h:
+                        v = h[None]
+                        if v is not None:
+                            # replace NULL with non-NULL value
+                            feature.SetField(i, v)
+                    continue
+
+                v = feature.GetField(i)
+                if v in h:
+                    v2 = h[v]
+                    if v2 is None:
+                        # replace non-NULL value with NULL
+                        feature.SetFieldNull(i)
+                    else:
+                        # replace non-NULL value with non-NULL value
+                        feature.SetField(i, v2)
+
         feature2 = ogr.Feature(defn_dst)
         feature2.SetFromWithMap(feature, False, fieldMap)