Add logic for field regex substitution.

This is useful to replace a YYYYMMDD formatted date with YYYY-MM-DD.
The target field can then be set to not-nullable and its type set to
Date, as the OGR_F_SetField*() with take care of the conversion.

We could also do that via an SQL query, but in our case the sources are
not proper RDBMS so SQL is emulated anyway.

1 files changed, 79 insertions, 55 deletions

diff --git a/config.yml b/config.yml
index 183599c..60b9a20 100644
--- a/config.yml
+++ b/config.yml
@@ -231,18 +231,23 @@ layers:
 #            source_field_name1: field_name1
 #            source_field_name2: field_name2
 #
-#          # Mapping of (source) field values, to replace particular
-#          # values with something else.  Useful for instance to replace
-#          # the empty string with NULL.
+#          # Mapping of (source) field values, to replace literal values or values (fully)
+#          # matching a regular expression with something else.  Useful for instance to
+#          # replace the empty string with NULL.  Rules are tested in the order given and
+#          # stop at the first match.
 #          value-map:
 #            source_field_name1:
-#              - from: ''
-#                to: null
-#              - from: 'foo'
-#                to: 'bar'
+#              - replace: ''
+#                with: null
+#                #type: literal
+#              - replace: 'foo'
+#                with: 'bar'
+#              - replace: 'fo[a-z]'
+#                with: 'baz'
+#                type: regex
 #            source_field_name2:
-#              - from: 0
-#                to: 1
+#              - replace: 0
+#                with: 1
 
   'lansyta':
     description: Sveriges län (Lantmäteriet)
@@ -626,8 +631,8 @@ layers:
             Arealha: ArealHa
           value-map:
             Beteckn:
-              - from: 'Visas ej'
-                to: null
+              - replace: 'Visas ej'
+                with: null
 
       - source:
           download:
@@ -661,8 +666,8 @@ layers:
             Arealha: ArealHa
           value-map:
             Beteckn:
-              - from: 'Visas ej'
-                to: null
+              - replace: 'Visas ej'
+                with: null
 
       - source:
           download:
@@ -696,8 +701,8 @@ layers:
             Arealha: ArealHa
           value-map:
             Beteckn:
-              - from: 'Visas ej'
-                to: null
+              - replace: 'Visas ej'
+                with: null
 
       - source:
           download:
@@ -729,8 +734,8 @@ layers:
             Arealha: ArealHa
           value-map:
             Beteckn:
-              - from: 'Visas ej'
-                to: null
+              - replace: 'Visas ej'
+                with: null
 
   'st:betesomraden':
     source:
@@ -784,9 +789,7 @@ layers:
           nullable: false
           width: 16
         - name: LastUpdated
-          # TODO convert to Date
-          type: String
-          width: 10
+          type: Date
           nullable: false
           #comment: datum för senaste uppdatering
     source:
@@ -804,6 +807,11 @@ layers:
         'Application date': ApplicationDate
         'Diary nr': DiaryNr
         'Last updated': LastUpdated
+      value-map:
+        'Last updated':
+          - replace: '([0-9]{4})([0-9]{2})([0-9]{2})'
+            with: '{0}-{1}-{2}'
+            type: regex
 
   'mrr:bearbetningskoncessioner_approved':
     description: Bearbetningskoncessioner, beviljade (SGU)
@@ -839,9 +847,7 @@ layers:
           unique: true
           width: 16
         - name: LastUpdated
-          # TODO convert to Date
-          type: String
-          width: 10
+          type: Date
           nullable: false
           #comment: datum för senaste uppdatering
     source:
@@ -862,8 +868,12 @@ layers:
         'Last updated': LastUpdated
       value-map:
         'Diary nr':
-          - from: ''
-            to: null
+          - replace: ''
+            with: null
+        'Last updated':
+          - replace: '([0-9]{4})([0-9]{2})([0-9]{2})'
+            with: '{0}-{1}-{2}'
+            type: regex
 
   'mrr:markanvisningar':
     description: Markanvisning till koncession (SGU)
@@ -883,9 +893,7 @@ layers:
           unique: true
           width: 16
         - name: LastUpdated
-          # TODO convert to Date
-          type: String
-          width: 10
+          type: Date
           nullable: false
           #comment: datum för senaste uppdatering
     source:
@@ -903,11 +911,15 @@ layers:
         'Last updated': LastUpdated
       value-map:
         'Decision date':
-          - from: ''
-            to: null
+          - replace: ''
+            with: null
         'Diary nr':
-          - from: ''
-            to: null
+          - replace: ''
+            with: null
+        'Last updated':
+          - replace: '([0-9]{4})([0-9]{2})([0-9]{2})'
+            with: '{0}-{1}-{2}'
+            type: regex
 
   'mrr:mineral_applied':
     description: Undersökningstillstånd, metallar och mineral, ansökta (SGU)
@@ -939,9 +951,7 @@ layers:
           nullable: false
           width: 16
         - name: LastUpdated
-          # TODO convert to Date
-          type: String
-          width: 10
+          type: Date
           nullable: false
           #comment: datum för senaste uppdatering
     source:
@@ -959,6 +969,11 @@ layers:
         'Application date': ApplicationDate
         'Diary nr': DiaryNr
         'Last updated': LastUpdated
+      value-map:
+        'Last updated':
+          - replace: '([0-9]{4})([0-9]{2})([0-9]{2})'
+            with: '{0}-{1}-{2}'
+            type: regex
 
   'mrr:mineral_approved':
     description: Undersökningstillstånd, metallar och mineral, beviljade (SGU)
@@ -999,9 +1014,7 @@ layers:
           unique: true
           width: 16
         - name: LastUpdated
-          # TODO convert to Date
-          type: String
-          width: 10
+          type: Date
           nullable: false
           #comment: datum för senaste uppdatering
     source:
@@ -1023,8 +1036,12 @@ layers:
         'Last updated': LastUpdated
       value-map:
         'Diary nr':
-          - from: ''
-            to: null
+          - replace: ''
+            with: null
+        'Last updated':
+          - replace: '([0-9]{4})([0-9]{2})([0-9]{2})'
+            with: '{0}-{1}-{2}'
+            type: regex
 
 #  'mrr:mineral_expired':
 #    source:
@@ -1083,9 +1100,7 @@ layers:
           nullable: false
           width: 16
         - name: LastUpdated
-          # TODO convert to Date
-          type: String
-          width: 10
+          type: Date
           nullable: false
           #comment: datum för senaste uppdatering
     source:
@@ -1103,6 +1118,11 @@ layers:
         'Application date': ApplicationDate
         'Diary nr': DiaryNr
         'Last updated': LastUpdated
+      value-map:
+        'Last updated':
+          - replace: '([0-9]{4})([0-9]{2})([0-9]{2})'
+            with: '{0}-{1}-{2}'
+            type: regex
 
   'mrr:olja_gas_diamant_approved':
     description: Undersökningstillstånd, olja, gas och diamant, beviljade (SGU)
@@ -1143,9 +1163,7 @@ layers:
           unique: true
           width: 16
         - name: LastUpdated
-          # TODO convert to Date
-          type: String
-          width: 10
+          type: Date
           nullable: false
           #comment: datum för senaste uppdatering
     source:
@@ -1167,8 +1185,12 @@ layers:
         'Last updated': LastUpdated
       value-map:
         'Diary nr':
-          - from: ''
-            to: null
+          - replace: ''
+            with: null
+        'Last updated':
+          - replace: '([0-9]{4})([0-9]{2})([0-9]{2})'
+            with: '{0}-{1}-{2}'
+            type: regex
 
   'mrr:torvkoncessioner':
     description: Torvkoncessioner (SGU)
@@ -1203,9 +1225,7 @@ layers:
           unique: true
           width: 16
         - name: LastUpdated
-          # TODO convert to Date
-          type: String
-          width: 10
+          type: Date
           nullable: false
           #comment: datum för senaste uppdatering
     source:
@@ -1226,8 +1246,12 @@ layers:
         'Last updated': LastUpdated
       value-map:
         'Mineral':
-          - from: ''
-            to: null
+          - replace: ''
+            with: null
         'Diary nr':
-          - from: ''
-            to: null
+          - replace: ''
+            with: null
+        'Last updated':
+          - replace: '([0-9]{4})([0-9]{2})([0-9]{2})'
+            with: '{0}-{1}-{2}'
+            type: regex