From ceb76b0893b5a0cbfeab269d373b6bb656222b69 Mon Sep 17 00:00:00 2001 From: Guilhem Moulin Date: Wed, 19 Jun 2024 04:29:26 +0200 Subject: Add logic for field regex substitution. This is useful to replace a YYYYMMDD formatted date with YYYY-MM-DD. The target field can then be set to not-nullable and its type set to Date, as the OGR_F_SetField*() with take care of the conversion. We could also do that via an SQL query, but in our case the sources are not proper RDBMS so SQL is emulated anyway. --- config.yml | 134 ++++++++++++++++++++++++++++++++++++------------------------- 1 file changed, 79 insertions(+), 55 deletions(-) (limited to 'config.yml') diff --git a/config.yml b/config.yml index 183599c..60b9a20 100644 --- a/config.yml +++ b/config.yml @@ -231,18 +231,23 @@ layers: # source_field_name1: field_name1 # source_field_name2: field_name2 # -# # Mapping of (source) field values, to replace particular -# # values with something else. Useful for instance to replace -# # the empty string with NULL. +# # Mapping of (source) field values, to replace literal values or values (fully) +# # matching a regular expression with something else. Useful for instance to +# # replace the empty string with NULL. Rules are tested in the order given and +# # stop at the first match. # value-map: # source_field_name1: -# - from: '' -# to: null -# - from: 'foo' -# to: 'bar' +# - replace: '' +# with: null +# #type: literal +# - replace: 'foo' +# with: 'bar' +# - replace: 'fo[a-z]' +# with: 'baz' +# type: regex # source_field_name2: -# - from: 0 -# to: 1 +# - replace: 0 +# with: 1 'lansyta': description: Sveriges län (Lantmäteriet) @@ -626,8 +631,8 @@ layers: Arealha: ArealHa value-map: Beteckn: - - from: 'Visas ej' - to: null + - replace: 'Visas ej' + with: null - source: download: @@ -661,8 +666,8 @@ layers: Arealha: ArealHa value-map: Beteckn: - - from: 'Visas ej' - to: null + - replace: 'Visas ej' + with: null - source: download: @@ -696,8 +701,8 @@ layers: Arealha: ArealHa value-map: Beteckn: - - from: 'Visas ej' - to: null + - replace: 'Visas ej' + with: null - source: download: @@ -729,8 +734,8 @@ layers: Arealha: ArealHa value-map: Beteckn: - - from: 'Visas ej' - to: null + - replace: 'Visas ej' + with: null 'st:betesomraden': source: @@ -784,9 +789,7 @@ layers: nullable: false width: 16 - name: LastUpdated - # TODO convert to Date - type: String - width: 10 + type: Date nullable: false #comment: datum för senaste uppdatering source: @@ -804,6 +807,11 @@ layers: 'Application date': ApplicationDate 'Diary nr': DiaryNr 'Last updated': LastUpdated + value-map: + 'Last updated': + - replace: '([0-9]{4})([0-9]{2})([0-9]{2})' + with: '{0}-{1}-{2}' + type: regex 'mrr:bearbetningskoncessioner_approved': description: Bearbetningskoncessioner, beviljade (SGU) @@ -839,9 +847,7 @@ layers: unique: true width: 16 - name: LastUpdated - # TODO convert to Date - type: String - width: 10 + type: Date nullable: false #comment: datum för senaste uppdatering source: @@ -862,8 +868,12 @@ layers: 'Last updated': LastUpdated value-map: 'Diary nr': - - from: '' - to: null + - replace: '' + with: null + 'Last updated': + - replace: '([0-9]{4})([0-9]{2})([0-9]{2})' + with: '{0}-{1}-{2}' + type: regex 'mrr:markanvisningar': description: Markanvisning till koncession (SGU) @@ -883,9 +893,7 @@ layers: unique: true width: 16 - name: LastUpdated - # TODO convert to Date - type: String - width: 10 + type: Date nullable: false #comment: datum för senaste uppdatering source: @@ -903,11 +911,15 @@ layers: 'Last updated': LastUpdated value-map: 'Decision date': - - from: '' - to: null + - replace: '' + with: null 'Diary nr': - - from: '' - to: null + - replace: '' + with: null + 'Last updated': + - replace: '([0-9]{4})([0-9]{2})([0-9]{2})' + with: '{0}-{1}-{2}' + type: regex 'mrr:mineral_applied': description: Undersökningstillstånd, metallar och mineral, ansökta (SGU) @@ -939,9 +951,7 @@ layers: nullable: false width: 16 - name: LastUpdated - # TODO convert to Date - type: String - width: 10 + type: Date nullable: false #comment: datum för senaste uppdatering source: @@ -959,6 +969,11 @@ layers: 'Application date': ApplicationDate 'Diary nr': DiaryNr 'Last updated': LastUpdated + value-map: + 'Last updated': + - replace: '([0-9]{4})([0-9]{2})([0-9]{2})' + with: '{0}-{1}-{2}' + type: regex 'mrr:mineral_approved': description: Undersökningstillstånd, metallar och mineral, beviljade (SGU) @@ -999,9 +1014,7 @@ layers: unique: true width: 16 - name: LastUpdated - # TODO convert to Date - type: String - width: 10 + type: Date nullable: false #comment: datum för senaste uppdatering source: @@ -1023,8 +1036,12 @@ layers: 'Last updated': LastUpdated value-map: 'Diary nr': - - from: '' - to: null + - replace: '' + with: null + 'Last updated': + - replace: '([0-9]{4})([0-9]{2})([0-9]{2})' + with: '{0}-{1}-{2}' + type: regex # 'mrr:mineral_expired': # source: @@ -1083,9 +1100,7 @@ layers: nullable: false width: 16 - name: LastUpdated - # TODO convert to Date - type: String - width: 10 + type: Date nullable: false #comment: datum för senaste uppdatering source: @@ -1103,6 +1118,11 @@ layers: 'Application date': ApplicationDate 'Diary nr': DiaryNr 'Last updated': LastUpdated + value-map: + 'Last updated': + - replace: '([0-9]{4})([0-9]{2})([0-9]{2})' + with: '{0}-{1}-{2}' + type: regex 'mrr:olja_gas_diamant_approved': description: Undersökningstillstånd, olja, gas och diamant, beviljade (SGU) @@ -1143,9 +1163,7 @@ layers: unique: true width: 16 - name: LastUpdated - # TODO convert to Date - type: String - width: 10 + type: Date nullable: false #comment: datum för senaste uppdatering source: @@ -1167,8 +1185,12 @@ layers: 'Last updated': LastUpdated value-map: 'Diary nr': - - from: '' - to: null + - replace: '' + with: null + 'Last updated': + - replace: '([0-9]{4})([0-9]{2})([0-9]{2})' + with: '{0}-{1}-{2}' + type: regex 'mrr:torvkoncessioner': description: Torvkoncessioner (SGU) @@ -1203,9 +1225,7 @@ layers: unique: true width: 16 - name: LastUpdated - # TODO convert to Date - type: String - width: 10 + type: Date nullable: false #comment: datum för senaste uppdatering source: @@ -1226,8 +1246,12 @@ layers: 'Last updated': LastUpdated value-map: 'Mineral': - - from: '' - to: null + - replace: '' + with: null 'Diary nr': - - from: '' - to: null + - replace: '' + with: null + 'Last updated': + - replace: '([0-9]{4})([0-9]{2})([0-9]{2})' + with: '{0}-{1}-{2}' + type: regex -- cgit v1.2.3