diff options
| -rw-r--r-- | common.py | 60 | ||||
| -rw-r--r-- | config.yml | 312 | ||||
| -rw-r--r-- | export_mvt.py | 98 | ||||
| -rwxr-xr-x | webmap-download | 62 | ||||
| -rwxr-xr-x | webmap-import | 30 |
5 files changed, 470 insertions, 92 deletions
@@ -19,7 +19,7 @@ # pylint: disable=missing-module-docstring import os -from os import path as os_path, curdir as os_curdir +from os import path as os_path, curdir as os_curdir, pardir as os_pardir, sep as os_sep import sys from fnmatch import fnmatchcase from pathlib import Path, PosixPath @@ -151,6 +151,64 @@ def parse_config(path : Optional[Path] = None, return config +def _check_key_type(k : str, v : str, known_keys : list[type, tuple[set[str]]]) -> bool: + for t, ks in known_keys: + if k in ks and isinstance(v, t): + return True + return False + +def parse_config_dl(downloads) -> dict[str, dict[str, str|int]]: + """Parse and validate the "downloads" section from the configuration dictionary""" + + if not isinstance(downloads, list): + raise BadConfiguration(f'Invalid download recipe: {downloads}') + + known_keys = [ + (str, {'path', 'url'}), + (int, {'max-age', 'max-size'}), + ] + + destinations = {} + known_keys_set = {k for _,ks in known_keys for k in ks} + for dl in downloads: + if 'url' in dl: + dls = [dl] + elif 'basedir' in dl and 'baseurl' in dl and 'files' in dl and 'path' not in dl: + dls = [] + for filename in dl['files']: + dl2 = { + 'path' : os_path.join(dl['basedir'], filename), + 'url' : dl['baseurl'] + filename + } + for k, v in dl.items(): + if k not in ('basedir', 'baseurl', 'files'): + dl2[k] = v + dls.append(dl2) + else: + raise BadConfiguration(f'Invalid download recipe: {dl}') + + for dl in dls: + path = dl.get('path', None) + if path is None or path in ('', os_curdir, os_pardir) or path.endswith(os_sep): + raise BadConfiguration(f'Invalid destination path "{path}"') + if path in destinations: + raise BadConfiguration(f'Duplicate download recipe for "{path}"') + dl2 = {} + for k, v in dl.items(): + if k == 'path': + continue + if k not in known_keys_set: + logging.warning('Ignoring unknown setting "%s" in download recipe for "%s"', + k, path) + elif not _check_key_type(k, v, known_keys): + logging.warning('Ignoring setting "%s" in download recipe for "%s"' + ' (invalid type)', k, path) + else: + dl2[k] = v + destinations[path] = dl2 + + return destinations + # pylint: disable-next=invalid-name def getSourcePathLockFileName(path : str) -> str: """Return the name of the lockfile associated with a source path.""" @@ -231,6 +231,292 @@ downloads: - path: svk/SVK_STAMNAT.zip url: https://gis-services.metria.se/svkfeed/filer/SVK_STAMNAT.zip +license-info: + # Map source paths to their metada (description, copyright, license + # information and link). + nvk/nvr/TILLTRADESFORBUD.zip: + description: "Skyddade områden: tillträdesförbud (föreskriftsområden)" + copyright: © Naturvårdsverket + product_url: https://geodatakatalogen.naturvardsverket.se/geonetwork/srv/swe/catalog.search#/metadata/F2554ED6-3A9B-4955-B4AC-D61B35026C88 + license: + name: CC0 1.0 Universiell + url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv + nvk/nvr/NP.zip: + description: "Skyddade områden: nationalparker" + copyright: © Naturvårdsverket + product_url: https://geodatakatalogen.naturvardsverket.se/geonetwork/srv/swe/catalog.search#/metadata/bfc33845-ffb9-4835-8355-76af3773d4e0 + license: + name: CC0 1.0 Universiell + url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv + nvk/nvr/NR.zip: + description: "Skyddade områden: naturreservat" + copyright: © Naturvårdsverket + product_url: https://geodatakatalogen.naturvardsverket.se/geonetwork/srv/swe/catalog.search#/metadata/2921b01a-0baf-4702-a89f-9c5626c97844 + license: + name: CC0 1.0 Universiell + url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv + nvk/nvr/NVO.zip: + description: "Skyddade områden: naturvårdsområden" + copyright: © Naturvårdsverket + product_url: https://geodatakatalogen.naturvardsverket.se/geonetwork/srv/swe/catalog.search#/metadata/dd8371a0-f692-44e3-bd0b-25de8dee8906 + license: + name: CC0 1.0 Universiell + url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv + nvk/nvr/DVO.zip: + description: "Skyddade områden: djur- och växtskyddsområden" + copyright: © Naturvårdsverket + product_url: https://geodatakatalogen.naturvardsverket.se/geonetwork/srv/swe/catalog.search#/metadata/b4bb8837-8980-4093-be7e-c09f650df996 + license: + name: CC0 1.0 Universiell + url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv + nvk/nvr/KR.zip: + description: "Skyddade områden: kulturreservat" + copyright: © Naturvårdsverket + product_url: https://geodatakatalogen.naturvardsverket.se/geonetwork/srv/swe/catalog.search#/metadata/55d17118-f977-46c9-8691-20baf657796e + license: + name: CC0 1.0 Universiell + url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv + nvk/nvr/VSO.zip: + description: "Skyddade områden: vattenskyddsområden" + copyright: © Naturvårdsverket + product_url: https://geodatakatalogen.naturvardsverket.se/geonetwork/srv/swe/catalog.search#/metadata/ae8d79d2-a799-4e1b-b500-05747a428816 + license: + name: CC0 1.0 Universiell + url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv + nvk/nvr/LBSO.zip: + description: "Landskapsbildsskyddsområde" + copyright: © Naturvårdsverket + product_url: https://geodatakatalogen.naturvardsverket.se/geonetwork/srv/swe/catalog.search#/metadata/bf435698-15a4-4b0b-85ec-727605a0a6ba + license: + name: Inga begränsningar + url: https://inspire.ec.europa.eu/metadata-codelist/LimitationsOnPublicAccess/noLimitations + sks/sksBiotopskydd_gpkg.zip: + description: "Biotopskydd beslutade av Skogsstyrelsen" + copyright: © Skogsstyrelsen + product_url: https://www.geodata.se/geodataportalen/GetMetaDataById?ID=772d46b8-25a2-42f7-b3da-4b17f610bc53 + license: + name: CC0 1.0 Universiell + url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv + nvk/nvr/OBO.zip: + description: "Skyddade områden: biotopskyddsområden" + copyright: © Naturvårdsverket + product_url: https://geodatakatalogen.naturvardsverket.se/geonetwork/srv/swe/catalog.search#/metadata/c3dd73b1-1c82-4db5-aac3-c8c6f240fa25 + license: + name: CC0 1.0 Universiell + url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv + nvk/nvr/NM.zip: + description: "Skyddade områden: naturminnen" + copyright: © Naturvårdsverket + product_url: https://geodatakatalogen.naturvardsverket.se/geonetwork/srv/swe/catalog.search#/metadata/c6b02e88-8084-4b3f-8a7d-33e5d45349c4 + license: + name: CC0 1.0 Universiell + url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv + nvk/nvr/IF.zip: + description: "Skyddade områden: interimistiska förbud" + copyright: © Naturvårdsverket + product_url: https://geodatakatalogen.naturvardsverket.se/geonetwork/srv/swe/catalog.search#/metadata/03e6e0d2-9ff8-4234-8dba-1a1ef88cb1ad + license: + name: CC0 1.0 Universiell + url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv + nvk/nvr/SPA_Rikstackande.zip: + description: "Skyddade områden: fågeldirektivet (Natura 2000, SPA)" + copyright: © Naturvårdsverket + product_url: https://geodatakatalogen.naturvardsverket.se/geonetwork/srv/swe/catalog.search#/metadata/a80bf3d7-e70c-42d1-9b8d-8148e53e011d + license: + name: CC0 1.0 Universiell + url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv + nvk/nvr/SCI_Rikstackande.zip: + description: "Skyddade områden: Art- och habitatdirektivet (Natura2000, SCI, SAC)" + copyright: © Naturvårdsverket + product_url: https://geodatakatalogen.naturvardsverket.se/geonetwork/srv/swe/catalog.search#/metadata/945e918f-8426-4155-8fd6-3f780a85dd8f + license: + name: CC0 1.0 Universiell + url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv + nvk/nvr/HELCOM.zip: + description: "Skyddade områden: marina områden i Sverige enligt Helcom (MPA)" + copyright: © Naturvårdsverket + product_url: https://geodatakatalogen.naturvardsverket.se/geonetwork/srv/swe/catalog.search#/metadata/834a442f-310f-4d2d-bd12-d8978d9683c5 + license: + name: CC0 1.0 Universiell + url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv + nvk/nvr/Ramsar_2018.zip: + description: "Skyddade områden: Ramsarområden" + copyright: © Naturvårdsverket + product_url: https://geodatakatalogen.naturvardsverket.se/geonetwork/srv/swe/catalog.search#/metadata/f2d8691f-8b75-4a62-8d94-7cb1982cceea + license: + name: CC0 1.0 Universiell + url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv + nvk/nvr/OSPAR.zip: + description: "Skyddade områden: marina områden i Sverige enligt Ospar" + copyright: © Naturvårdsverket + product_url: https://geodatakatalogen.naturvardsverket.se/geonetwork/srv/swe/catalog.search#/metadata/39948786-a278-4cdb-8b95-2ce99f941f65 + license: + name: CC0 1.0 Universiell + url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv + nvk/nvr/Varldsarv.zip: + description: "Skyddade områden: världsarv med höga naturvärden" + copyright: © Naturvårdsverket + product_url: https://geodatakatalogen.naturvardsverket.se/geonetwork/srv/swe/catalog.search#/metadata/f57de73f-0ce0-4be0-a638-5778bec38cde + license: + name: CC0 1.0 Universiell + url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv + nvk/nvr/biosfarsomraden.zip: + description: "Skyddade områden: biosfärsområden" + copyright: © Naturvårdsverket + product_url: https://geodatakatalogen.naturvardsverket.se/geonetwork/srv/swe/catalog.search#/metadata/bc2ce857-fa87-42f6-8870-fbdc3a9b113e + license: + name: CC0 1.0 Universiell + url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv + nvk/nvr/NVA.zip: + description: "Områden som omfattas av naturvårdsavtal (Naturvårdsverket, Länsstyrelsen)" + copyright: © Naturvårdsverket + product_url: https://geodatakatalogen.naturvardsverket.se/geonetwork/srv/swe/catalog.search#/metadata/3a5790ff-8cd3-45ea-bbee-28cf2c1b6b06 + license: + name: CC0 1.0 Universiell + url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv + sks/sksNaturvardsavtal_gpkg.zip: + description: "Naturvårdsavtal upprättade av Skogsstyrelsen" + copyright: © Skogsstyrelsen + product_url: https://www.geodata.se/geodataportalen/GetMetaDataById?ID=f56d281c-8246-40aa-83cd-9db0d4389d5a + license: + name: CC0 1.0 Universiell + url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv + sks/sksAvverkAnm_gpkg.zip: + description: "Avverkningsanmälningar" + copyright: © Skogsstyrelsen + product_url: https://www.skogsstyrelsen.se/e-tjanster-och-kartor/karttjanster/skogsstyrelsens-geodata/ + license: + # https://www.skogsstyrelsen.se/e-tjanster-och-kartor/karttjanster/geodatatjanster/villkor/ + name: CC0 1.0 Universiell + url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv + sks/sksUtfordAvverk_gpkg.zip: + description: "Utförda avverkningar" + copyright: © Skogsstyrelsen + product_url: https://www.skogsstyrelsen.se/e-tjanster-och-kartor/karttjanster/skogsstyrelsens-geodata/ + license: + # https://www.skogsstyrelsen.se/e-tjanster-och-kartor/karttjanster/geodatatjanster/villkor/ + name: CC0 1.0 Universiell + url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv + mrr/mineralrattigheter.zip: + description: "Mineralrättigheter och prospektering" + copyright: © Sveriges geologiska undersökning (utdrag ur Bergsstatens mineralrättsregister) + product_url: https://www.sgu.se/produkter-och-tjanster/geologiska-data/malmer-och-mineral--geologiska-data/mineralrattigheter-och-prospektering/ + license: + name: CC0 1.0 Universiell + url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv + ren/ren.riks_ren.zip: + description: "Riksintresse Rennäringen" + copyright: © Sametinget + product_url: https://ext-geodatakatalog-forv.lansstyrelsen.se/PlaneringsKatalogen/GetMetaDataById?id=103cf137-9d56-452b-97d6-9b12cba6c864_C + license: + name: CC BY 4.0 + url: https://creativecommons.org/licenses/by/4.0/deed.sv + ren/ren.omr_riks.zip: + description: "Riksintresse Rennäringen - Kärnområde" + copyright: © Sametinget + product_url: https://ext-geodatakatalog-forv.lansstyrelsen.se/PlaneringsKatalogen/GetMetaDataById?id=b665a528-cd25-4612-8ab3-fa3e692b46c3_C + license: + name: CC BY 4.0 + url: https://creativecommons.org/licenses/by/4.0/deed.sv + sametinget/Samebyarnas_markanvandningsredovisning.zip: + description: "Samebyarnas markanvändningsområden" + copyright: © Sametinget (Rennäringens markanvändningsdatabas IRENMARK) + product_url: https://ext-geodatakatalog-forv.lansstyrelsen.se/PlaneringsKatalogen/GetMetaDataById?id=19b7addd-a790-4829-991f-f2266009e863_C + # Licens behövs, cf. 'Samebyarnas markanvändningsredovisning/avtal.pdf' + license: Se avtal.pdf i zip-filen + sametinget/Samebyarnas_betesomraden.zip: + description: "Samebyarnas betesområden" + copyright: © Sametinget (Rennäringens markanvändningsdatabas, IRENMARK) + product_url: https://ext-geodatakatalog-forv.lansstyrelsen.se/PlaneringsKatalogen/GetMetaDataById?id=a216dea8-bfcb-4984-a18b-3a421cde2d57_C + # Licens behövs, cf. 'Samebyarnas_betesomraden/Samebyarnas betesområden/avtal.pdf' + license: Se avtal.pdf i zip-filen + nvk/RI_Naturvard.zip: + description: "Riksintresse naturvård" + copyright: © Naturvårdsverket + product_url: https://geodatakatalogen.naturvardsverket.se/geonetwork/srv/swe/catalog.search#/metadata/fb9ff32f-b6f8-4d8e-ac5c-20ebb0986908 + license: + name: CC0 1.0 Universiell + url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv + nvk/RI_Friluftsliv.zip: + description: "Riksintresse friluftsliv" + copyright: © Naturvårdsverket + product_url: https://geodatakatalogen.naturvardsverket.se/geonetwork/srv/swe/catalog.search#/metadata/22afb5cb-cdb0-4f3a-8b0f-a34344285864 + license: + name: CC0 1.0 Universiell + url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv + lst/lst.LST_RI_Rorligt_friluftsliv_MB4kap2.zip: + description: "Riksintresse rörligt friluftsliv (MB 4 kap 1 och 2 §§)" + copyright: © Länsstyrelsen + product_url: https://ext-geodatakatalog.lansstyrelsen.se/GeodataKatalogen/srv/api/records/GetMetaDataById?id=072b6b36-2cf6-4717-a616-bbf3fddea83d + license: + name: CC BY 4.0 + url: https://creativecommons.org/licenses/by/4.0/deed.sv + lst/lst.LST_RI_Obruten_kust_MB4kap3.zip: + description: "Riksintresse obruten kust (MB 4 kap 3 §)" + copyright: © Länsstyrelsen — Förvaltningsobjekt Samhällsplanerin + product_url: https://ext-geodatakatalog.lansstyrelsen.se/GeodataKatalogen/srv/api/records/GetMetaDataById?id=2b5b141f-a9a4-433a-8dc7-bf983acdb859 + license: + name: CC0 1.0 Universiell + url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv + lst/lst.Lst_RI_Obrutet_fjall_MB4kap5.zip: + description: "Riksintresse obrutet fjäll (MB 4 kap 5 §)" + copyright: © Länsstyrelsen + product_url: https://ext-geodatakatalog.lansstyrelsen.se/GeodataKatalogen/srv/api/records/GetMetaDataById?id=b1d59cb0-2e71-4c08-b99d-e4cc7507cb92 + license: Inga begränsningar + lst/lst.LST_RI_Skyddade_vattendrag_MB4kap6.zip: + description: "Riksintresse skyddade vattendrag (MB 4 kap 6 §)" + copyright: © Länsstyrelsen + product_url: https://ext-geodatakatalog.lansstyrelsen.se/GeodataKatalogen/srv/api/records/GetMetaDataById?id=61e21a50-4320-4db4-8e44-56252dab777e + license: + name: CC BY 4.0 + url: https://creativecommons.org/licenses/by/4.0/deed.sv + vbk/lst.vbk_vindkraftverk.zip: + description: "Vindbrukskollen vindkraftverk" + copyright: © Länsstyrelsen + product_url: https://ext-geodatakatalog.lansstyrelsen.se/GeodataKatalogen/srv/api/records/GetMetaDataById?id=ed5814b2-08bf-493a-a164-7819e1b590d6 + license: + name: CC0 1.0 Universiell + url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv + vbk/lst.vbk_projekteringsomraden.zip: + description: "Vindbrukskollen projekteringsområden och vägar" + copyright: © Länsstyrelsen + product_url: https://ext-geodatakatalog-forv.lansstyrelsen.se/PlaneringsKatalogen/GetMetaDataById?id=c816bd1e-bc6c-487f-a962-770f05f677b6_C + license: + name: CC0 1.0 Universiell + url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv + vbk/lst.vbk_havsbaserad_vindkraft.zip: + description: "LST Vindbrukskollen havsbaserad vindkraft" + copyright: © Länsstyrelsen + product_url: https://ext-geodatakatalog.lansstyrelsen.se/GeodataKatalogen/srv/api/records/GetMetaDataById?id=c290bc31-1af8-497e-a9a5-87fcec55d0ce + license: + name: CC0 1.0 Universiell + url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv + svk/SVK_STAMNAT.zip: + description: "Transmissionsnät för el i Sverige" + copyright: © Svenska kraftnät + product_url: https://ext-geodatakatalog-forv.lansstyrelsen.se/PlaneringsKatalogen/GetMetaDataById?id=08ec56a0-6b5c-4f83-b29e-375e6f1a34b9_C + license: Okänd + custom/svk/transmissionsnatsprojekt.geojson: + description: "Transmissionsnätsprojekt" + copyright: © Guilhem Moulin (egen ritning baserad på SvK:s tillståndsansökningar och handlingar) + product_url: https://www.svk.se/utveckling-av-kraftsystemet/transmissionsnatet/transmissionsnatsprojekt/ + license: + name: CC0 1.0 Universiell + url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv + custom/gigafactories.geojson: + description: "Stora industrisatsningar" + copyright: © Guilhem Moulin + license: + name: CC0 1.0 Universiell + url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv + custom/HY_PhysicalWaters_ManMadeObject.zip: + description: "Dammregistret" + copyright: © Sveriges meteorologiska och hydrologiska institut (SMHI) + product_url: https://www.smhi.se/data/sok-oppna-data-i-utforskaren/se-hy-dammregistret + license: + name: CC BY 4.0 + url: https://creativecommons.org/licenses/by/4.0/deed.sv + layers: # # Dictionary of layer names and source recipes in the output dataset. If a layer @@ -1210,7 +1496,6 @@ layers: publish: landskapsbildsskyddsomrade 'nvr:Biotopskydd': - # https://www.geodata.se/geodataportalen/GetMetaDataById?ID=772d46b8-25a2-42f7-b3da-4b17f610bc53 # https://www.skogsstyrelsen.se/globalassets/sjalvservice/karttjanster/geodatatjanster/produktbeskrivningar/biotopskydd---produktbeskrivning.pdf description: Biotopskydd i skogsmark (beslutade av Skogsstyrelsen) create: @@ -1992,7 +2277,6 @@ layers: publish: biosfarsomraden 'nva:Naturvardsverket_Lansstyrelse': - # https://geodatakatalogen.naturvardsverket.se/geonetwork/srv/swe/catalog.search#/metadata/3a5790ff-8cd3-45ea-bbee-28cf2c1b6b06 description: Naturvårdsavtal (Naturvårdsverket, Länsstyrelse) create: geometry-type: MULTIPOLYGON @@ -2037,7 +2321,6 @@ layers: publish: naturvardsavtal 'nva:Skogsstyrelsen': - # https://www.geodata.se/geodataportalen/GetMetaDataById?ID=f56d281c-8246-40aa-83cd-9db0d4389d5a description: Naturvårdsavtal (Skogsstyrelsen) create: geometry-type: MULTIPOLYGON @@ -2275,8 +2558,6 @@ layers: publish: clearcut_comp 'sametinget:betesomraden': - # https://ext-geodatakatalog-forv.lansstyrelsen.se/PlaneringsKatalogen/GetMetaDataById?id=a216dea8-bfcb-4984-a18b-3a421cde2d57_C - # Licens behövs, cf. 'Samebyarnas_betesomraden/Samebyarnas betesområden/avtal.pdf' description: 'Samebyarnas betesområden: Renbetesområden' create: # https://ext-dokument.lansstyrelsen.se/Gemensamt/Geodata/Datadistribution/Information,%20Skiktf%C3%B6rteckning%20och%20f%C3%B6rklaringar.pdf @@ -2328,8 +2609,6 @@ layers: publish: betesomraden 'sametinget:flyttled': - # https://ext-geodatakatalog-forv.lansstyrelsen.se/PlaneringsKatalogen/GetMetaDataById?id=19b7addd-a790-4829-991f-f2266009e863_C - # Licens behövs, cf. 'Samebyarnas_betesomraden/Samebyarnas betesområden/avtal.pdf' description: 'Samebyarnas markanvändningsredovisning: Flyttled' create: # https://ext-dokument.lansstyrelsen.se/Gemensamt/Geodata/Datadistribution/Information,%20Skiktf%C3%B6rteckning%20och%20f%C3%B6rklaringar.pdf @@ -2408,7 +2687,6 @@ layers: publish: flyttled 'ren:riks_ren': - # https://ext-geodatakatalog-forv.lansstyrelsen.se/PlaneringsKatalogen/GetMetaDataById?id=103cf137-9d56-452b-97d6-9b12cba6c864_C description: 'Riksintresse Rennäringen' create: geometry-type: MULTIPOLYGON @@ -2451,7 +2729,6 @@ layers: publish: riks_ren 'ren:omr_riks': - # https://ext-geodatakatalog-forv.lansstyrelsen.se/PlaneringsKatalogen/GetMetaDataById?id=b665a528-cd25-4612-8ab3-fa3e692b46c3_C description: 'Riksintresse Rennäringen — Kärnområde' create: geometry-type: MULTIPOLYGON @@ -3857,7 +4134,6 @@ layers: with: null 'ri:naturvard': - # https://geodatakatalogen.naturvardsverket.se/geonetwork/srv/swe/catalog.search#/metadata/fb9ff32f-b6f8-4d8e-ac5c-20ebb0986908 description: Riksintresse naturvård create: geometry-type: MULTIPOLYGON @@ -3909,7 +4185,6 @@ layers: publish: naturvard 'ri:friluftsliv': - # https://geodatakatalogen.naturvardsverket.se/geonetwork/srv/swe/catalog.search#/metadata/22afb5cb-cdb0-4f3a-8b0f-a34344285864 description: Riksintresse friluftsliv create: geometry-type: MULTIPOLYGON @@ -3976,8 +4251,7 @@ layers: publish: friluftsliv 'ri:rorligt_friluftsliv': - # https://ext-geodatakatalog-forv.lansstyrelsen.se/PlaneringsKatalogen/GetMetaDataById?id=22afb5cb-cdb0-4f3a-8b0f-a34344285864_C - description: Rörligt friluftsliv (MB 4 kap 1§ och 2§) + description: Rörligt friluftsliv (MB 4 kap 1 och 2 §§) create: geometry-type: MULTIPOLYGON fields: @@ -4020,8 +4294,7 @@ layers: publish: rorligt_friluftsliv 'ri:obruten_kust': - # https://ext-geodatakatalog.lansstyrelsen.se/GeodataKatalogen/srv/api/records/GetMetaDataById?id=2b5b141f-a9a4-433a-8dc7-bf983acdb859 - description: Obruten kust (MB 4 kap 3§) + description: Obruten kust (MB 4 kap 3 §) create: geometry-type: MULTIPOLYGON fields: @@ -4067,8 +4340,7 @@ layers: publish: obruten_kust 'ri:obrutet_fjall': - # https://ext-geodatakatalog.lansstyrelsen.se/GeodataKatalogen/srv/api/records/GetMetaDataById?id=b1d59cb0-2e71-4c08-b99d-e4cc7507cb92 - description: Obrutet fjäll (MB 4 kap 5§) + description: Obrutet fjäll (MB 4 kap 5 §) create: geometry-type: MULTIPOLYGON fields: @@ -4109,8 +4381,7 @@ layers: publish: obrutet_fjall 'ri:skyddade_vattendrag': - # https://ext-geodatakatalog.lansstyrelsen.se/GeodataKatalogen/srv/api/records/GetMetaDataById?id=61e21a50-4320-4db4-8e44-56252dab777e - description: Skyddade vattendrag (MB 4 kap 6§) + description: Skyddade vattendrag (MB 4 kap 6 §) create: geometry-type: MULTIPOLYGON fields: @@ -4161,7 +4432,6 @@ layers: publish: skyddade_vattendrag 'svk:ledningar': - # https://ext-geodatakatalog-forv.lansstyrelsen.se/PlaneringsKatalogen/GetMetaDataById?id=08ec56a0-6b5c-4f83-b29e-375e6f1a34b9_C description: Kraftledningar (befintliga) create: geometry-type: MULTILINESTRING @@ -4234,7 +4504,6 @@ layers: minzoom: 7 'svk:transmissionsnatsprojekt': - # https://www.svk.se/utveckling-av-kraftsystemet/transmissionsnatet/transmissionsnatsprojekt/ description: Transmissionsnätsprojekt create: geometry-type: MULTILINESTRING @@ -4283,7 +4552,6 @@ layers: minzoom: 6 'dammar': - # https://www.smhi.se/data/utforskaren-oppna-data/se-hy-dammregistret description: Dammar create: # https://www.smhi.se/polopoly_fs/1.34541!/dammprod%202013_3%2C%20beskrivning%2C%20SVAR2012_2.pdf diff --git a/export_mvt.py b/export_mvt.py index a929b78..d19909c 100644 --- a/export_mvt.py +++ b/export_mvt.py @@ -20,7 +20,7 @@ # pylint: disable=invalid-name, missing-module-docstring, fixme -from os import O_RDONLY, O_WRONLY, O_CREAT, O_EXCL, O_CLOEXEC, O_DIRECTORY, F_OK +from os import O_RDONLY, O_WRONLY, O_CREAT, O_EXCL, O_TRUNC, O_CLOEXEC, O_DIRECTORY, F_OK import os from errno import EAGAIN import json @@ -29,7 +29,7 @@ from pathlib import Path import shutil import tempfile from typing import Any, Iterator, Optional -from time import monotonic as time_monotonic +from time import monotonic as time_monotonic, time_ns import brotli from osgeo import gdal, ogr, osr @@ -274,9 +274,89 @@ def compress_brotli(path : str, os.close(fd_in) return size_in, size_out +def getLayerMetadata(layers : dict[str,Any], + sources : dict[str,Any], + license_info: dict[str,str|dict[str,str]], + last_modified : dict[str,int], + last_updated : int) -> dict[str,int|dict[int|str|dict[str,str]]]: + """Return a dictionary suitable for metadata.json""" + layers2 = {} + for k, v in layers.items(): + layers2[k] = x = {} + if 'description' in v: + x['description'] = v['description'] + source_paths = [] + for src in v.get('sources', []): + if 'source' not in src or src['source'] is None: + continue + if 'path' not in src['source']: + continue + source_path = src['source']['path'] + if source_path is not None: + source_paths.append(source_path) + if len(source_paths) > 0: + # remove duplicates but preserve order + x['source_files'] = list(dict.fromkeys(source_paths)) + + source_files = {} + for source_path in { p for v in layers2.values() for p in v.get('source_files', []) }: + source_files[source_path] = x = {} + if source_path in sources and 'url' in sources[source_path]: + x['url'] = sources[source_path]['url'] + if source_path not in license_info: + logging.warning('Source path %s lacks license information', source_path) + else: + license_info0 = license_info[source_path] + for k in ('description', 'copyright', 'product_url'): + if k in license_info0: + x[k] = license_info0[k] + if 'license' in license_info0: + if isinstance(license_info0['license'], str): + x['license'] = { 'name': license_info0['license'] } + elif isinstance(license_info0['license'], dict): + x['license'] = license_info0['license'].copy() + if source_path not in last_modified: + logging.warning('Source path %s lack last_modified value', source_path) + else: + x['last_modified'] = last_modified[source_path] + + return { + 'layers': layers2, + 'source_files': source_files, + 'last_updated': last_updated + } + +def exportMetadata(basedir : Path, data : dict[str,Any], + dir_fd : Optional[int] = None, + compress : bool = False) -> None: + """Generate metadata.json""" + data = json.dumps(data, ensure_ascii=False, separators=(',',':')).encode('utf-8') + path = basedir.joinpath('metadata.json') + flags = O_WRONLY|O_CREAT|O_TRUNC|O_CLOEXEC + + fd = os.open(str(path), flags, mode=0o644, dir_fd=dir_fd) + try: + write_all(fd, data) + finally: + os.close(fd) + + if not compress: + return + + compressor = brotli.Compressor(mode=brotli.MODE_GENERIC, quality=11) + fd = os.open(str(path.with_suffix('.json.br')), flags, mode=0o644, dir_fd=dir_fd) + try: + write_all(fd, compressor.process(data)) + write_all(fd, compressor.finish()) + finally: + os.close(fd) + # pylint: disable-next=too-many-branches, too-many-statements def exportMVT(ds : gdal.Dataset, layers : dict[str,dict[str,Any]], + sources : dict[str,Any], + license_info: dict[str,str|dict[str,str]], + last_modified : dict[str,int], dst : Path, drvname : str = 'MVT', default_options : dict[str,Any]|None = None, @@ -321,6 +401,7 @@ def exportMVT(ds : gdal.Dataset, start = time_monotonic() os.mkdir(dbname, mode=0o700, dir_fd=dir_fd) basedir = Path(f'/proc/self/fd/{dir_fd}') + creation_time = time_ns() dso = createMVT(drv, path=str(basedir.joinpath(mvtname)), default_options=default_options, options = { @@ -406,11 +487,14 @@ def exportMVT(ds : gdal.Dataset, format_bytes(size_min_z), format_bytes(size_max_z), format_bytes(size_tot_z), format_bytes(round(size_tot_z/tile_count))) - try: - # OpenLayers doesn't make use of that file so delete it - os.unlink(str(Path(mvtname).joinpath('metadata.json')), dir_fd=dir_fd) - except FileNotFoundError: - pass + exportMetadata(basedir=Path(mvtname), + data=getLayerMetadata({k:layers[v] for k,(v,_) in export_layers.items()}, + sources=sources, + license_info=license_info, + last_modified=last_modified, + last_updated=creation_time // 1000000), + dir_fd=dir_fd, + compress=compress) try: # atomically exchange paths diff --git a/webmap-download b/webmap-download index 2c475fe..5e191ad 100755 --- a/webmap-download +++ b/webmap-download @@ -32,8 +32,6 @@ from os import ( O_TMPFILE, path as os_path, curdir as os_curdir, - pardir as os_pardir, - sep as os_sep ) import os import sys @@ -48,7 +46,7 @@ from typing import Optional, NoReturn, Never import requests import common -from common import BadConfiguration, getSourcePathLockFileName +from common import parse_config_dl, getSourcePathLockFileName def download_trystream(url : str, **kwargs) -> requests.Response: """GET a url, trying a number of times. Return immediately after the @@ -167,64 +165,6 @@ def download(dest : str, common.format_time(elapsed), common.format_bytes(int(size/elapsed))) -def _check_key_type(k : str, v : str, known_keys : list[type, tuple[set[str]]]) -> bool: - for t, ks in known_keys: - if k in ks and isinstance(v, t): - return True - return False - -def parse_config_dl(downloads) -> dict[str, dict[str, str|int]]: - """Parse and validate the "downloads" section from the configuration dictionary""" - - if not isinstance(downloads, list): - raise BadConfiguration(f'Invalid download recipe: {downloads}') - - known_keys = [ - (str, {'path', 'url'}), - (int, {'max-age', 'max-size'}) - ] - - destinations = {} - known_keys_set = {k for _,ks in known_keys for k in ks} - for dl in downloads: - if 'url' in dl: - dls = [dl] - elif 'basedir' in dl and 'baseurl' in dl and 'files' in dl and 'path' not in dl: - dls = [] - for filename in dl['files']: - dl2 = { - 'path' : os_path.join(dl['basedir'], filename), - 'url' : dl['baseurl'] + filename - } - for k, v in dl.items(): - if k not in ('basedir', 'baseurl', 'files'): - dl2[k] = v - dls.append(dl2) - else: - raise BadConfiguration(f'Invalid download recipe: {dl}') - - for dl in dls: - path = dl.get('path', None) - if path is None or path in ('', os_curdir, os_pardir) or path.endswith(os_sep): - raise BadConfiguration(f'Invalid destination path "{path}"') - if path in destinations: - raise BadConfiguration(f'Duplicate download recipe for "{path}"') - dl2 = {} - for k, v in dl.items(): - if k == 'path': - continue - if k not in known_keys_set: - logging.warning('Ignoring unknown setting "%s" in download recipe for "%s"', - k, path) - elif not _check_key_type(k, v, known_keys): - logging.warning('Ignoring setting "%s" in download recipe for "%s"' - ' (invalid type)', k, path) - else: - dl2[k] = v - destinations[path] = dl2 - - return destinations - # pylint: disable-next=missing-function-docstring def main() -> NoReturn: common.init_logger(app=os_path.basename(__file__), level=logging.INFO) diff --git a/webmap-import b/webmap-import index c86e7a2..e5a1426 100755 --- a/webmap-import +++ b/webmap-import @@ -47,6 +47,7 @@ from osgeo import gdalconst import common from common import ( BadConfiguration, + parse_config_dl, escape_identifier, escape_literal_str, getSourcePathLockFileName @@ -524,6 +525,26 @@ def areSourceFilesNewer(layername : str, source_path, dt.astimezone().isoformat(timespec='seconds')) return ret +def getLastMTimes(layerdefs : dict[str,Any], basedir : Optional[Path] = None) -> dict[str,int]: + """Return a directing mapping source paths to their last modification time + (as a timestamp in milliseconds).""" + ret = {} + for layerdef in layerdefs: + for source in layerdef['sources']: + source_path = source['source']['path'] + if source_path in ret: + continue + path = source_path if basedir is None else str(basedir.joinpath(source_path)) + try: + st = os.stat(path) + if not S_ISREG(st.st_mode): + raise FileNotFoundError + ret[source_path] = st.st_mtime_ns // 1000000 + except (OSError, ValueError): + #logging.warning('Could not stat(%s)', path) + pass + return ret + def lockSourcePaths(layerdefs : dict[str,Any], lockdir: str) -> dict[str,int]: """Place shared locks on each source path and return their respective file descriptors. We could do that one layerdef at a time (one output layer at a @@ -703,6 +724,9 @@ def main() -> NoReturn: elapsed = time_monotonic() - start logging.info('Processed %d destination layers in %s', n, common.format_time(elapsed)) + # get mtimes before releasing the source locks + last_modified = getLastMTimes(layerdefs=layers.values(), basedir=args.cachedir) + if sourcePathLocks is not None: releaseSourcePathLocks(sourcePathLocks) @@ -716,7 +740,11 @@ def main() -> NoReturn: logging.info('Skipping MVT export for group %s (no changes)', ', '.join(args.groupname) if args.groupname is not None else '*') else: - exportMVT(dso, layers=export_layers, + exportMVT(dso, + layers=export_layers, + sources=parse_config_dl(config.get('downloads', [])), + license_info=config.get('license-info', {}), + last_modified=last_modified, dst=args.mvtdir, default_options=config.get('vector-tiles', None), compress=args.mvt_compress) |