diff options
| author | Guilhem Moulin <guilhem@fripost.org> | 2025-05-27 23:20:26 +0200 | 
|---|---|---|
| committer | Guilhem Moulin <guilhem@fripost.org> | 2025-06-04 15:16:29 +0200 | 
| commit | 052536f62d2e58f6b9b142e035c49cb033458d7f (patch) | |
| tree | 9487344ce453dfa3493dc2a0c9dd899a0070cfb5 | |
| parent | 3e9586f0a9708afca40ab50a2987fc0090256e2f (diff) | |
MVT: Generate metadata.json with copyright and timing information.
So the information can be exposed to the webmap's info dialog.
| -rw-r--r-- | common.py | 60 | ||||
| -rw-r--r-- | config.yml | 312 | ||||
| -rw-r--r-- | export_mvt.py | 98 | ||||
| -rwxr-xr-x | webmap-download | 62 | ||||
| -rwxr-xr-x | webmap-import | 30 | 
5 files changed, 470 insertions, 92 deletions
@@ -19,7 +19,7 @@  # pylint: disable=missing-module-docstring  import os -from os import path as os_path, curdir as os_curdir +from os import path as os_path, curdir as os_curdir, pardir as os_pardir, sep as os_sep  import sys  from fnmatch import fnmatchcase  from pathlib import Path, PosixPath @@ -151,6 +151,64 @@ def parse_config(path : Optional[Path] = None,      return config +def _check_key_type(k : str, v : str, known_keys : list[type, tuple[set[str]]]) -> bool: +    for t, ks in known_keys: +        if k in ks and isinstance(v, t): +            return True +    return False + +def parse_config_dl(downloads) -> dict[str, dict[str, str|int]]: +    """Parse and validate the "downloads" section from the configuration dictionary""" + +    if not isinstance(downloads, list): +        raise BadConfiguration(f'Invalid download recipe: {downloads}') + +    known_keys = [ +        (str, {'path', 'url'}), +        (int, {'max-age', 'max-size'}), +    ] + +    destinations = {} +    known_keys_set = {k for _,ks in known_keys for k in ks} +    for dl in downloads: +        if 'url' in dl: +            dls = [dl] +        elif 'basedir' in dl and 'baseurl' in dl and 'files' in dl and 'path' not in dl: +            dls = [] +            for filename in dl['files']: +                dl2 = { +                    'path' : os_path.join(dl['basedir'], filename), +                    'url' : dl['baseurl'] + filename +                } +                for k, v in dl.items(): +                    if k not in ('basedir', 'baseurl', 'files'): +                        dl2[k] = v +                dls.append(dl2) +        else: +            raise BadConfiguration(f'Invalid download recipe: {dl}') + +        for dl in dls: +            path = dl.get('path', None) +            if path is None or path in ('', os_curdir, os_pardir) or path.endswith(os_sep): +                raise BadConfiguration(f'Invalid destination path "{path}"') +            if path in destinations: +                raise BadConfiguration(f'Duplicate download recipe for "{path}"') +            dl2 = {} +            for k, v in dl.items(): +                if k == 'path': +                    continue +                if k not in known_keys_set: +                    logging.warning('Ignoring unknown setting "%s" in download recipe for "%s"', +                                    k, path) +                elif not _check_key_type(k, v, known_keys): +                    logging.warning('Ignoring setting "%s" in download recipe for "%s"' +                                    ' (invalid type)', k, path) +                else: +                    dl2[k] = v +            destinations[path] = dl2 + +    return destinations +  # pylint: disable-next=invalid-name  def getSourcePathLockFileName(path : str) -> str:      """Return the name of the lockfile associated with a source path.""" @@ -231,6 +231,292 @@ downloads:    - path: svk/SVK_STAMNAT.zip      url: https://gis-services.metria.se/svkfeed/filer/SVK_STAMNAT.zip +license-info: +  # Map source paths to their metada (description, copyright, license +  # information and link). +  nvk/nvr/TILLTRADESFORBUD.zip: +    description: "Skyddade områden: tillträdesförbud (föreskriftsområden)" +    copyright: © Naturvårdsverket +    product_url: https://geodatakatalogen.naturvardsverket.se/geonetwork/srv/swe/catalog.search#/metadata/F2554ED6-3A9B-4955-B4AC-D61B35026C88 +    license: +      name: CC0 1.0 Universiell +      url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv +  nvk/nvr/NP.zip: +    description: "Skyddade områden: nationalparker" +    copyright: © Naturvårdsverket +    product_url: https://geodatakatalogen.naturvardsverket.se/geonetwork/srv/swe/catalog.search#/metadata/bfc33845-ffb9-4835-8355-76af3773d4e0 +    license: +      name: CC0 1.0 Universiell +      url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv +  nvk/nvr/NR.zip: +    description: "Skyddade områden: naturreservat" +    copyright: © Naturvårdsverket +    product_url: https://geodatakatalogen.naturvardsverket.se/geonetwork/srv/swe/catalog.search#/metadata/2921b01a-0baf-4702-a89f-9c5626c97844 +    license: +      name: CC0 1.0 Universiell +      url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv +  nvk/nvr/NVO.zip: +    description: "Skyddade områden: naturvårdsområden" +    copyright: © Naturvårdsverket +    product_url: https://geodatakatalogen.naturvardsverket.se/geonetwork/srv/swe/catalog.search#/metadata/dd8371a0-f692-44e3-bd0b-25de8dee8906 +    license: +      name: CC0 1.0 Universiell +      url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv +  nvk/nvr/DVO.zip: +    description: "Skyddade områden: djur- och växtskyddsområden" +    copyright: © Naturvårdsverket +    product_url: https://geodatakatalogen.naturvardsverket.se/geonetwork/srv/swe/catalog.search#/metadata/b4bb8837-8980-4093-be7e-c09f650df996 +    license: +      name: CC0 1.0 Universiell +      url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv +  nvk/nvr/KR.zip: +    description: "Skyddade områden: kulturreservat" +    copyright: © Naturvårdsverket +    product_url: https://geodatakatalogen.naturvardsverket.se/geonetwork/srv/swe/catalog.search#/metadata/55d17118-f977-46c9-8691-20baf657796e +    license: +      name: CC0 1.0 Universiell +      url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv +  nvk/nvr/VSO.zip: +    description: "Skyddade områden: vattenskyddsområden" +    copyright: © Naturvårdsverket +    product_url: https://geodatakatalogen.naturvardsverket.se/geonetwork/srv/swe/catalog.search#/metadata/ae8d79d2-a799-4e1b-b500-05747a428816 +    license: +      name: CC0 1.0 Universiell +      url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv +  nvk/nvr/LBSO.zip: +    description: "Landskapsbildsskyddsområde" +    copyright: © Naturvårdsverket +    product_url: https://geodatakatalogen.naturvardsverket.se/geonetwork/srv/swe/catalog.search#/metadata/bf435698-15a4-4b0b-85ec-727605a0a6ba +    license: +      name: Inga begränsningar +      url: https://inspire.ec.europa.eu/metadata-codelist/LimitationsOnPublicAccess/noLimitations +  sks/sksBiotopskydd_gpkg.zip: +    description: "Biotopskydd beslutade av Skogsstyrelsen" +    copyright: © Skogsstyrelsen +    product_url: https://www.geodata.se/geodataportalen/GetMetaDataById?ID=772d46b8-25a2-42f7-b3da-4b17f610bc53 +    license: +      name: CC0 1.0 Universiell +      url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv +  nvk/nvr/OBO.zip: +    description: "Skyddade områden: biotopskyddsområden" +    copyright: © Naturvårdsverket +    product_url: https://geodatakatalogen.naturvardsverket.se/geonetwork/srv/swe/catalog.search#/metadata/c3dd73b1-1c82-4db5-aac3-c8c6f240fa25 +    license: +      name: CC0 1.0 Universiell +      url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv +  nvk/nvr/NM.zip: +    description: "Skyddade områden: naturminnen" +    copyright: © Naturvårdsverket +    product_url: https://geodatakatalogen.naturvardsverket.se/geonetwork/srv/swe/catalog.search#/metadata/c6b02e88-8084-4b3f-8a7d-33e5d45349c4 +    license: +      name: CC0 1.0 Universiell +      url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv +  nvk/nvr/IF.zip: +    description: "Skyddade områden: interimistiska förbud" +    copyright: © Naturvårdsverket +    product_url: https://geodatakatalogen.naturvardsverket.se/geonetwork/srv/swe/catalog.search#/metadata/03e6e0d2-9ff8-4234-8dba-1a1ef88cb1ad +    license: +      name: CC0 1.0 Universiell +      url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv +  nvk/nvr/SPA_Rikstackande.zip: +    description: "Skyddade områden: fågeldirektivet (Natura 2000, SPA)" +    copyright: © Naturvårdsverket +    product_url: https://geodatakatalogen.naturvardsverket.se/geonetwork/srv/swe/catalog.search#/metadata/a80bf3d7-e70c-42d1-9b8d-8148e53e011d +    license: +      name: CC0 1.0 Universiell +      url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv +  nvk/nvr/SCI_Rikstackande.zip: +    description: "Skyddade områden: Art- och habitatdirektivet (Natura2000, SCI, SAC)" +    copyright: © Naturvårdsverket +    product_url: https://geodatakatalogen.naturvardsverket.se/geonetwork/srv/swe/catalog.search#/metadata/945e918f-8426-4155-8fd6-3f780a85dd8f +    license: +      name: CC0 1.0 Universiell +      url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv +  nvk/nvr/HELCOM.zip: +    description: "Skyddade områden: marina områden i Sverige enligt Helcom (MPA)" +    copyright: © Naturvårdsverket +    product_url: https://geodatakatalogen.naturvardsverket.se/geonetwork/srv/swe/catalog.search#/metadata/834a442f-310f-4d2d-bd12-d8978d9683c5 +    license: +      name: CC0 1.0 Universiell +      url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv +  nvk/nvr/Ramsar_2018.zip: +    description: "Skyddade områden: Ramsarområden" +    copyright: © Naturvårdsverket +    product_url: https://geodatakatalogen.naturvardsverket.se/geonetwork/srv/swe/catalog.search#/metadata/f2d8691f-8b75-4a62-8d94-7cb1982cceea +    license: +      name: CC0 1.0 Universiell +      url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv +  nvk/nvr/OSPAR.zip: +    description: "Skyddade områden: marina områden i Sverige enligt Ospar" +    copyright: © Naturvårdsverket +    product_url: https://geodatakatalogen.naturvardsverket.se/geonetwork/srv/swe/catalog.search#/metadata/39948786-a278-4cdb-8b95-2ce99f941f65 +    license: +      name: CC0 1.0 Universiell +      url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv +  nvk/nvr/Varldsarv.zip: +    description: "Skyddade områden: världsarv med höga naturvärden" +    copyright: © Naturvårdsverket +    product_url: https://geodatakatalogen.naturvardsverket.se/geonetwork/srv/swe/catalog.search#/metadata/f57de73f-0ce0-4be0-a638-5778bec38cde +    license: +      name: CC0 1.0 Universiell +      url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv +  nvk/nvr/biosfarsomraden.zip: +    description: "Skyddade områden: biosfärsområden" +    copyright: © Naturvårdsverket +    product_url: https://geodatakatalogen.naturvardsverket.se/geonetwork/srv/swe/catalog.search#/metadata/bc2ce857-fa87-42f6-8870-fbdc3a9b113e +    license: +      name: CC0 1.0 Universiell +      url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv +  nvk/nvr/NVA.zip: +    description: "Områden som omfattas av naturvårdsavtal (Naturvårdsverket, Länsstyrelsen)" +    copyright: © Naturvårdsverket +    product_url: https://geodatakatalogen.naturvardsverket.se/geonetwork/srv/swe/catalog.search#/metadata/3a5790ff-8cd3-45ea-bbee-28cf2c1b6b06 +    license: +      name: CC0 1.0 Universiell +      url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv +  sks/sksNaturvardsavtal_gpkg.zip: +    description: "Naturvårdsavtal upprättade av Skogsstyrelsen" +    copyright: © Skogsstyrelsen +    product_url: https://www.geodata.se/geodataportalen/GetMetaDataById?ID=f56d281c-8246-40aa-83cd-9db0d4389d5a +    license: +      name: CC0 1.0 Universiell +      url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv +  sks/sksAvverkAnm_gpkg.zip: +    description: "Avverkningsanmälningar" +    copyright: © Skogsstyrelsen +    product_url: https://www.skogsstyrelsen.se/e-tjanster-och-kartor/karttjanster/skogsstyrelsens-geodata/ +    license: +      # https://www.skogsstyrelsen.se/e-tjanster-och-kartor/karttjanster/geodatatjanster/villkor/ +      name: CC0 1.0 Universiell +      url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv +  sks/sksUtfordAvverk_gpkg.zip: +    description: "Utförda avverkningar" +    copyright: © Skogsstyrelsen +    product_url: https://www.skogsstyrelsen.se/e-tjanster-och-kartor/karttjanster/skogsstyrelsens-geodata/ +    license: +      # https://www.skogsstyrelsen.se/e-tjanster-och-kartor/karttjanster/geodatatjanster/villkor/ +      name: CC0 1.0 Universiell +      url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv +  mrr/mineralrattigheter.zip: +    description: "Mineralrättigheter och prospektering" +    copyright: © Sveriges geologiska undersökning (utdrag ur Bergsstatens mineralrättsregister) +    product_url: https://www.sgu.se/produkter-och-tjanster/geologiska-data/malmer-och-mineral--geologiska-data/mineralrattigheter-och-prospektering/ +    license: +      name: CC0 1.0 Universiell +      url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv +  ren/ren.riks_ren.zip: +    description: "Riksintresse Rennäringen" +    copyright: © Sametinget +    product_url: https://ext-geodatakatalog-forv.lansstyrelsen.se/PlaneringsKatalogen/GetMetaDataById?id=103cf137-9d56-452b-97d6-9b12cba6c864_C +    license: +      name: CC BY 4.0 +      url: https://creativecommons.org/licenses/by/4.0/deed.sv +  ren/ren.omr_riks.zip: +    description: "Riksintresse Rennäringen - Kärnområde" +    copyright: © Sametinget +    product_url: https://ext-geodatakatalog-forv.lansstyrelsen.se/PlaneringsKatalogen/GetMetaDataById?id=b665a528-cd25-4612-8ab3-fa3e692b46c3_C +    license: +      name: CC BY 4.0 +      url: https://creativecommons.org/licenses/by/4.0/deed.sv +  sametinget/Samebyarnas_markanvandningsredovisning.zip: +    description: "Samebyarnas markanvändningsområden" +    copyright: © Sametinget (Rennäringens markanvändningsdatabas IRENMARK) +    product_url: https://ext-geodatakatalog-forv.lansstyrelsen.se/PlaneringsKatalogen/GetMetaDataById?id=19b7addd-a790-4829-991f-f2266009e863_C +    # Licens behövs, cf. 'Samebyarnas markanvändningsredovisning/avtal.pdf' +    license: Se avtal.pdf i zip-filen +  sametinget/Samebyarnas_betesomraden.zip: +    description: "Samebyarnas betesområden" +    copyright: © Sametinget (Rennäringens markanvändningsdatabas, IRENMARK) +    product_url: https://ext-geodatakatalog-forv.lansstyrelsen.se/PlaneringsKatalogen/GetMetaDataById?id=a216dea8-bfcb-4984-a18b-3a421cde2d57_C +    # Licens behövs, cf. 'Samebyarnas_betesomraden/Samebyarnas betesområden/avtal.pdf' +    license: Se avtal.pdf i zip-filen +  nvk/RI_Naturvard.zip: +    description: "Riksintresse naturvård" +    copyright: © Naturvårdsverket +    product_url: https://geodatakatalogen.naturvardsverket.se/geonetwork/srv/swe/catalog.search#/metadata/fb9ff32f-b6f8-4d8e-ac5c-20ebb0986908 +    license: +      name: CC0 1.0 Universiell +      url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv +  nvk/RI_Friluftsliv.zip: +    description: "Riksintresse friluftsliv" +    copyright: © Naturvårdsverket +    product_url: https://geodatakatalogen.naturvardsverket.se/geonetwork/srv/swe/catalog.search#/metadata/22afb5cb-cdb0-4f3a-8b0f-a34344285864 +    license: +      name: CC0 1.0 Universiell +      url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv +  lst/lst.LST_RI_Rorligt_friluftsliv_MB4kap2.zip: +    description: "Riksintresse rörligt friluftsliv (MB 4 kap 1 och 2 §§)" +    copyright: © Länsstyrelsen +    product_url: https://ext-geodatakatalog.lansstyrelsen.se/GeodataKatalogen/srv/api/records/GetMetaDataById?id=072b6b36-2cf6-4717-a616-bbf3fddea83d +    license: +      name: CC BY 4.0 +      url: https://creativecommons.org/licenses/by/4.0/deed.sv +  lst/lst.LST_RI_Obruten_kust_MB4kap3.zip: +    description: "Riksintresse obruten kust (MB 4 kap 3 §)" +    copyright: © Länsstyrelsen — Förvaltningsobjekt Samhällsplanerin +    product_url: https://ext-geodatakatalog.lansstyrelsen.se/GeodataKatalogen/srv/api/records/GetMetaDataById?id=2b5b141f-a9a4-433a-8dc7-bf983acdb859 +    license: +      name: CC0 1.0 Universiell +      url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv +  lst/lst.Lst_RI_Obrutet_fjall_MB4kap5.zip: +    description: "Riksintresse obrutet fjäll (MB 4 kap 5 §)" +    copyright: © Länsstyrelsen +    product_url: https://ext-geodatakatalog.lansstyrelsen.se/GeodataKatalogen/srv/api/records/GetMetaDataById?id=b1d59cb0-2e71-4c08-b99d-e4cc7507cb92 +    license: Inga begränsningar +  lst/lst.LST_RI_Skyddade_vattendrag_MB4kap6.zip: +    description: "Riksintresse skyddade vattendrag (MB 4 kap 6 §)" +    copyright: © Länsstyrelsen +    product_url: https://ext-geodatakatalog.lansstyrelsen.se/GeodataKatalogen/srv/api/records/GetMetaDataById?id=61e21a50-4320-4db4-8e44-56252dab777e +    license: +      name: CC BY 4.0 +      url: https://creativecommons.org/licenses/by/4.0/deed.sv +  vbk/lst.vbk_vindkraftverk.zip: +    description: "Vindbrukskollen vindkraftverk" +    copyright: © Länsstyrelsen +    product_url: https://ext-geodatakatalog.lansstyrelsen.se/GeodataKatalogen/srv/api/records/GetMetaDataById?id=ed5814b2-08bf-493a-a164-7819e1b590d6 +    license: +      name: CC0 1.0 Universiell +      url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv +  vbk/lst.vbk_projekteringsomraden.zip: +    description: "Vindbrukskollen projekteringsområden och vägar" +    copyright: © Länsstyrelsen +    product_url: https://ext-geodatakatalog-forv.lansstyrelsen.se/PlaneringsKatalogen/GetMetaDataById?id=c816bd1e-bc6c-487f-a962-770f05f677b6_C +    license: +      name: CC0 1.0 Universiell +      url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv +  vbk/lst.vbk_havsbaserad_vindkraft.zip: +    description: "LST Vindbrukskollen havsbaserad vindkraft" +    copyright: © Länsstyrelsen +    product_url: https://ext-geodatakatalog.lansstyrelsen.se/GeodataKatalogen/srv/api/records/GetMetaDataById?id=c290bc31-1af8-497e-a9a5-87fcec55d0ce +    license: +      name: CC0 1.0 Universiell +      url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv +  svk/SVK_STAMNAT.zip: +    description: "Transmissionsnät för el i Sverige" +    copyright: © Svenska kraftnät +    product_url: https://ext-geodatakatalog-forv.lansstyrelsen.se/PlaneringsKatalogen/GetMetaDataById?id=08ec56a0-6b5c-4f83-b29e-375e6f1a34b9_C +    license: Okänd +  custom/svk/transmissionsnatsprojekt.geojson: +    description: "Transmissionsnätsprojekt" +    copyright: © Guilhem Moulin (egen ritning baserad på SvK:s tillståndsansökningar och handlingar) +    product_url: https://www.svk.se/utveckling-av-kraftsystemet/transmissionsnatet/transmissionsnatsprojekt/ +    license: +      name: CC0 1.0 Universiell +      url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv +  custom/gigafactories.geojson: +    description: "Stora industrisatsningar" +    copyright: © Guilhem Moulin +    license: +      name: CC0 1.0 Universiell +      url: https://creativecommons.org/publicdomain/zero/1.0/deed.sv +  custom/HY_PhysicalWaters_ManMadeObject.zip: +    description: "Dammregistret" +    copyright: © Sveriges meteorologiska och hydrologiska institut (SMHI) +    product_url: https://www.smhi.se/data/sok-oppna-data-i-utforskaren/se-hy-dammregistret +    license: +      name: CC BY 4.0 +      url: https://creativecommons.org/licenses/by/4.0/deed.sv +  layers:  #  # Dictionary of layer names and source recipes in the output dataset.  If a layer @@ -1210,7 +1496,6 @@ layers:      publish: landskapsbildsskyddsomrade    'nvr:Biotopskydd': -    # https://www.geodata.se/geodataportalen/GetMetaDataById?ID=772d46b8-25a2-42f7-b3da-4b17f610bc53      # https://www.skogsstyrelsen.se/globalassets/sjalvservice/karttjanster/geodatatjanster/produktbeskrivningar/biotopskydd---produktbeskrivning.pdf      description: Biotopskydd i skogsmark (beslutade av Skogsstyrelsen)      create: @@ -1992,7 +2277,6 @@ layers:      publish: biosfarsomraden    'nva:Naturvardsverket_Lansstyrelse': -    # https://geodatakatalogen.naturvardsverket.se/geonetwork/srv/swe/catalog.search#/metadata/3a5790ff-8cd3-45ea-bbee-28cf2c1b6b06      description: Naturvårdsavtal (Naturvårdsverket, Länsstyrelse)      create:        geometry-type: MULTIPOLYGON @@ -2037,7 +2321,6 @@ layers:      publish: naturvardsavtal    'nva:Skogsstyrelsen': -    # https://www.geodata.se/geodataportalen/GetMetaDataById?ID=f56d281c-8246-40aa-83cd-9db0d4389d5a      description: Naturvårdsavtal (Skogsstyrelsen)      create:        geometry-type: MULTIPOLYGON @@ -2275,8 +2558,6 @@ layers:      publish: clearcut_comp    'sametinget:betesomraden': -    # https://ext-geodatakatalog-forv.lansstyrelsen.se/PlaneringsKatalogen/GetMetaDataById?id=a216dea8-bfcb-4984-a18b-3a421cde2d57_C -    # Licens behövs, cf. 'Samebyarnas_betesomraden/Samebyarnas betesområden/avtal.pdf'      description: 'Samebyarnas betesområden: Renbetesområden'      create:        # https://ext-dokument.lansstyrelsen.se/Gemensamt/Geodata/Datadistribution/Information,%20Skiktf%C3%B6rteckning%20och%20f%C3%B6rklaringar.pdf @@ -2328,8 +2609,6 @@ layers:      publish: betesomraden    'sametinget:flyttled': -    # https://ext-geodatakatalog-forv.lansstyrelsen.se/PlaneringsKatalogen/GetMetaDataById?id=19b7addd-a790-4829-991f-f2266009e863_C -    # Licens behövs, cf. 'Samebyarnas_betesomraden/Samebyarnas betesområden/avtal.pdf'      description: 'Samebyarnas markanvändningsredovisning: Flyttled'      create:        # https://ext-dokument.lansstyrelsen.se/Gemensamt/Geodata/Datadistribution/Information,%20Skiktf%C3%B6rteckning%20och%20f%C3%B6rklaringar.pdf @@ -2408,7 +2687,6 @@ layers:      publish: flyttled    'ren:riks_ren': -    # https://ext-geodatakatalog-forv.lansstyrelsen.se/PlaneringsKatalogen/GetMetaDataById?id=103cf137-9d56-452b-97d6-9b12cba6c864_C      description: 'Riksintresse Rennäringen'      create:        geometry-type: MULTIPOLYGON @@ -2451,7 +2729,6 @@ layers:      publish: riks_ren    'ren:omr_riks': -    # https://ext-geodatakatalog-forv.lansstyrelsen.se/PlaneringsKatalogen/GetMetaDataById?id=b665a528-cd25-4612-8ab3-fa3e692b46c3_C      description: 'Riksintresse Rennäringen — Kärnområde'      create:        geometry-type: MULTIPOLYGON @@ -3857,7 +4134,6 @@ layers:              with: null    'ri:naturvard': -    # https://geodatakatalogen.naturvardsverket.se/geonetwork/srv/swe/catalog.search#/metadata/fb9ff32f-b6f8-4d8e-ac5c-20ebb0986908      description: Riksintresse naturvård      create:        geometry-type: MULTIPOLYGON @@ -3909,7 +4185,6 @@ layers:      publish: naturvard    'ri:friluftsliv': -    # https://geodatakatalogen.naturvardsverket.se/geonetwork/srv/swe/catalog.search#/metadata/22afb5cb-cdb0-4f3a-8b0f-a34344285864      description: Riksintresse friluftsliv      create:        geometry-type: MULTIPOLYGON @@ -3976,8 +4251,7 @@ layers:      publish: friluftsliv    'ri:rorligt_friluftsliv': -    # https://ext-geodatakatalog-forv.lansstyrelsen.se/PlaneringsKatalogen/GetMetaDataById?id=22afb5cb-cdb0-4f3a-8b0f-a34344285864_C -    description: Rörligt friluftsliv (MB 4 kap 1§ och 2§) +    description: Rörligt friluftsliv (MB 4 kap 1 och 2 §§)      create:        geometry-type: MULTIPOLYGON        fields: @@ -4020,8 +4294,7 @@ layers:      publish: rorligt_friluftsliv    'ri:obruten_kust': -    # https://ext-geodatakatalog.lansstyrelsen.se/GeodataKatalogen/srv/api/records/GetMetaDataById?id=2b5b141f-a9a4-433a-8dc7-bf983acdb859 -    description: Obruten kust (MB 4 kap 3§) +    description: Obruten kust (MB 4 kap 3 §)      create:        geometry-type: MULTIPOLYGON        fields: @@ -4067,8 +4340,7 @@ layers:      publish: obruten_kust    'ri:obrutet_fjall': -    # https://ext-geodatakatalog.lansstyrelsen.se/GeodataKatalogen/srv/api/records/GetMetaDataById?id=b1d59cb0-2e71-4c08-b99d-e4cc7507cb92 -    description: Obrutet fjäll (MB 4 kap 5§) +    description: Obrutet fjäll (MB 4 kap 5 §)      create:        geometry-type: MULTIPOLYGON        fields: @@ -4109,8 +4381,7 @@ layers:      publish: obrutet_fjall    'ri:skyddade_vattendrag': -    # https://ext-geodatakatalog.lansstyrelsen.se/GeodataKatalogen/srv/api/records/GetMetaDataById?id=61e21a50-4320-4db4-8e44-56252dab777e -    description: Skyddade vattendrag (MB 4 kap 6§) +    description: Skyddade vattendrag (MB 4 kap 6 §)      create:        geometry-type: MULTIPOLYGON        fields: @@ -4161,7 +4432,6 @@ layers:      publish: skyddade_vattendrag    'svk:ledningar': -    # https://ext-geodatakatalog-forv.lansstyrelsen.se/PlaneringsKatalogen/GetMetaDataById?id=08ec56a0-6b5c-4f83-b29e-375e6f1a34b9_C      description: Kraftledningar (befintliga)      create:        geometry-type: MULTILINESTRING @@ -4234,7 +4504,6 @@ layers:          minzoom: 7    'svk:transmissionsnatsprojekt': -    # https://www.svk.se/utveckling-av-kraftsystemet/transmissionsnatet/transmissionsnatsprojekt/      description: Transmissionsnätsprojekt      create:        geometry-type: MULTILINESTRING @@ -4283,7 +4552,6 @@ layers:          minzoom: 6    'dammar': -    # https://www.smhi.se/data/utforskaren-oppna-data/se-hy-dammregistret      description: Dammar      create:        # https://www.smhi.se/polopoly_fs/1.34541!/dammprod%202013_3%2C%20beskrivning%2C%20SVAR2012_2.pdf diff --git a/export_mvt.py b/export_mvt.py index a929b78..d19909c 100644 --- a/export_mvt.py +++ b/export_mvt.py @@ -20,7 +20,7 @@  # pylint: disable=invalid-name, missing-module-docstring, fixme -from os import O_RDONLY, O_WRONLY, O_CREAT, O_EXCL, O_CLOEXEC, O_DIRECTORY, F_OK +from os import O_RDONLY, O_WRONLY, O_CREAT, O_EXCL, O_TRUNC, O_CLOEXEC, O_DIRECTORY, F_OK  import os  from errno import EAGAIN  import json @@ -29,7 +29,7 @@ from pathlib import Path  import shutil  import tempfile  from typing import Any, Iterator, Optional -from time import monotonic as time_monotonic +from time import monotonic as time_monotonic, time_ns  import brotli  from osgeo import gdal, ogr, osr @@ -274,9 +274,89 @@ def compress_brotli(path : str,          os.close(fd_in)      return size_in, size_out +def getLayerMetadata(layers : dict[str,Any], +                     sources : dict[str,Any], +                     license_info: dict[str,str|dict[str,str]], +                     last_modified : dict[str,int], +                     last_updated : int) -> dict[str,int|dict[int|str|dict[str,str]]]: +    """Return a dictionary suitable for metadata.json""" +    layers2 = {} +    for k, v in layers.items(): +        layers2[k] = x = {} +        if 'description' in v: +            x['description'] = v['description'] +        source_paths = [] +        for src in v.get('sources', []): +            if 'source' not in src or src['source'] is None: +                continue +            if 'path' not in src['source']: +                continue +            source_path = src['source']['path'] +            if source_path is not None: +                source_paths.append(source_path) +        if len(source_paths) > 0: +            # remove duplicates but preserve order +            x['source_files'] = list(dict.fromkeys(source_paths)) + +    source_files = {} +    for source_path in { p for v in layers2.values() for p in v.get('source_files', []) }: +        source_files[source_path] = x = {} +        if source_path in sources and 'url' in sources[source_path]: +            x['url'] = sources[source_path]['url'] +        if source_path not in license_info: +            logging.warning('Source path %s lacks license information', source_path) +        else: +            license_info0 = license_info[source_path] +            for k in ('description', 'copyright', 'product_url'): +                if k in license_info0: +                    x[k] = license_info0[k] +            if 'license' in license_info0: +                if isinstance(license_info0['license'], str): +                    x['license'] = { 'name': license_info0['license'] } +                elif isinstance(license_info0['license'], dict): +                    x['license'] = license_info0['license'].copy() +        if source_path not in last_modified: +            logging.warning('Source path %s lack last_modified value', source_path) +        else: +            x['last_modified'] = last_modified[source_path] + +    return { +        'layers': layers2, +        'source_files': source_files, +        'last_updated': last_updated +    } + +def exportMetadata(basedir : Path, data : dict[str,Any], +                   dir_fd : Optional[int] = None, +                   compress : bool = False) -> None: +    """Generate metadata.json""" +    data = json.dumps(data, ensure_ascii=False, separators=(',',':')).encode('utf-8') +    path = basedir.joinpath('metadata.json') +    flags = O_WRONLY|O_CREAT|O_TRUNC|O_CLOEXEC + +    fd = os.open(str(path), flags, mode=0o644, dir_fd=dir_fd) +    try: +        write_all(fd, data) +    finally: +        os.close(fd) + +    if not compress: +        return + +    compressor = brotli.Compressor(mode=brotli.MODE_GENERIC, quality=11) +    fd = os.open(str(path.with_suffix('.json.br')), flags, mode=0o644, dir_fd=dir_fd) +    try: +        write_all(fd, compressor.process(data)) +        write_all(fd, compressor.finish()) +    finally: +        os.close(fd) +  # pylint: disable-next=too-many-branches, too-many-statements  def exportMVT(ds : gdal.Dataset,                layers : dict[str,dict[str,Any]], +              sources : dict[str,Any], +              license_info: dict[str,str|dict[str,str]], +              last_modified : dict[str,int],                dst : Path,                drvname : str = 'MVT',                default_options : dict[str,Any]|None = None, @@ -321,6 +401,7 @@ def exportMVT(ds : gdal.Dataset,          start = time_monotonic()          os.mkdir(dbname, mode=0o700, dir_fd=dir_fd)          basedir = Path(f'/proc/self/fd/{dir_fd}') +        creation_time = time_ns()          dso = createMVT(drv, path=str(basedir.joinpath(mvtname)),                          default_options=default_options,                          options = { @@ -406,11 +487,14 @@ def exportMVT(ds : gdal.Dataset,                      format_bytes(size_min_z), format_bytes(size_max_z),                      format_bytes(size_tot_z), format_bytes(round(size_tot_z/tile_count))) -        try: -            # OpenLayers doesn't make use of that file so delete it -            os.unlink(str(Path(mvtname).joinpath('metadata.json')), dir_fd=dir_fd) -        except FileNotFoundError: -            pass +        exportMetadata(basedir=Path(mvtname), +                       data=getLayerMetadata({k:layers[v] for k,(v,_) in export_layers.items()}, +                                             sources=sources, +                                             license_info=license_info, +                                             last_modified=last_modified, +                                             last_updated=creation_time // 1000000), +                       dir_fd=dir_fd, +                       compress=compress)          try:              # atomically exchange paths diff --git a/webmap-download b/webmap-download index 2c475fe..5e191ad 100755 --- a/webmap-download +++ b/webmap-download @@ -32,8 +32,6 @@ from os import (      O_TMPFILE,      path as os_path,      curdir as os_curdir, -    pardir as os_pardir, -    sep as os_sep  )  import os  import sys @@ -48,7 +46,7 @@ from typing import Optional, NoReturn, Never  import requests  import common -from common import BadConfiguration, getSourcePathLockFileName +from common import parse_config_dl, getSourcePathLockFileName  def download_trystream(url : str, **kwargs) -> requests.Response:      """GET a url, trying a number of times. Return immediately after the @@ -167,64 +165,6 @@ def download(dest : str,                   common.format_time(elapsed),                   common.format_bytes(int(size/elapsed))) -def _check_key_type(k : str, v : str, known_keys : list[type, tuple[set[str]]]) -> bool: -    for t, ks in known_keys: -        if k in ks and isinstance(v, t): -            return True -    return False - -def parse_config_dl(downloads) -> dict[str, dict[str, str|int]]: -    """Parse and validate the "downloads" section from the configuration dictionary""" - -    if not isinstance(downloads, list): -        raise BadConfiguration(f'Invalid download recipe: {downloads}') - -    known_keys = [ -        (str, {'path', 'url'}), -        (int, {'max-age', 'max-size'}) -    ] - -    destinations = {} -    known_keys_set = {k for _,ks in known_keys for k in ks} -    for dl in downloads: -        if 'url' in dl: -            dls = [dl] -        elif 'basedir' in dl and 'baseurl' in dl and 'files' in dl and 'path' not in dl: -            dls = [] -            for filename in dl['files']: -                dl2 = { -                    'path' : os_path.join(dl['basedir'], filename), -                    'url' : dl['baseurl'] + filename -                } -                for k, v in dl.items(): -                    if k not in ('basedir', 'baseurl', 'files'): -                        dl2[k] = v -                dls.append(dl2) -        else: -            raise BadConfiguration(f'Invalid download recipe: {dl}') - -        for dl in dls: -            path = dl.get('path', None) -            if path is None or path in ('', os_curdir, os_pardir) or path.endswith(os_sep): -                raise BadConfiguration(f'Invalid destination path "{path}"') -            if path in destinations: -                raise BadConfiguration(f'Duplicate download recipe for "{path}"') -            dl2 = {} -            for k, v in dl.items(): -                if k == 'path': -                    continue -                if k not in known_keys_set: -                    logging.warning('Ignoring unknown setting "%s" in download recipe for "%s"', -                                    k, path) -                elif not _check_key_type(k, v, known_keys): -                    logging.warning('Ignoring setting "%s" in download recipe for "%s"' -                                    ' (invalid type)', k, path) -                else: -                    dl2[k] = v -            destinations[path] = dl2 - -    return destinations -  # pylint: disable-next=missing-function-docstring  def main() -> NoReturn:      common.init_logger(app=os_path.basename(__file__), level=logging.INFO) diff --git a/webmap-import b/webmap-import index c86e7a2..e5a1426 100755 --- a/webmap-import +++ b/webmap-import @@ -47,6 +47,7 @@ from osgeo import gdalconst  import common  from common import (      BadConfiguration, +    parse_config_dl,      escape_identifier,      escape_literal_str,      getSourcePathLockFileName @@ -524,6 +525,26 @@ def areSourceFilesNewer(layername : str,                           source_path, dt.astimezone().isoformat(timespec='seconds'))      return ret +def getLastMTimes(layerdefs : dict[str,Any], basedir : Optional[Path] = None) -> dict[str,int]: +    """Return a directing mapping source paths to their last modification time +    (as a timestamp in milliseconds).""" +    ret = {} +    for layerdef in layerdefs: +        for source in layerdef['sources']: +            source_path = source['source']['path'] +            if source_path in ret: +                continue +            path = source_path if basedir is None else str(basedir.joinpath(source_path)) +            try: +                st = os.stat(path) +                if not S_ISREG(st.st_mode): +                    raise FileNotFoundError +                ret[source_path] = st.st_mtime_ns // 1000000 +            except (OSError, ValueError): +                #logging.warning('Could not stat(%s)', path) +                pass +    return ret +  def lockSourcePaths(layerdefs : dict[str,Any], lockdir: str) -> dict[str,int]:      """Place shared locks on each source path and return their respective file      descriptors.  We could do that one layerdef at a time (one output layer at a @@ -703,6 +724,9 @@ def main() -> NoReturn:          elapsed = time_monotonic() - start          logging.info('Processed %d destination layers in %s', n, common.format_time(elapsed)) +        # get mtimes before releasing the source locks +        last_modified = getLastMTimes(layerdefs=layers.values(), basedir=args.cachedir) +          if sourcePathLocks is not None:              releaseSourcePathLocks(sourcePathLocks) @@ -716,7 +740,11 @@ def main() -> NoReturn:              logging.info('Skipping MVT export for group %s (no changes)',                           ', '.join(args.groupname) if args.groupname is not None else '*')          else: -            exportMVT(dso, layers=export_layers, +            exportMVT(dso, +                      layers=export_layers, +                      sources=parse_config_dl(config.get('downloads', [])), +                      license_info=config.get('license-info', {}), +                      last_modified=last_modified,                        dst=args.mvtdir,                        default_options=config.get('vector-tiles', None),                        compress=args.mvt_compress)  | 
