Factor sources in config.yml.

This avoid duplications when the same source file is used multiple times
(either by the same layer or by multiple layers).  This change breaks
webmap-import, but that one will be refactored shortly.

It also breaks webmap-import-mrr.py, which is no longer used since
mineralrattigheter.zip can be downloaded from SGU's site directly.

1 files changed, 3 insertions, 77 deletions

diff --git a/common.py b/common.py
index da2927f..acbb5d8 100644
--- a/common.py
+++ b/common.py
@@ -1,6 +1,6 @@
 #----------------------------------------------------------------------
 # Backend utilities for the Klimatanalys Norr project (common module)
-# Copyright © 2024 Guilhem Moulin <info@guilhem.se>
+# Copyright © 2024-2025 Guilhem Moulin <info@guilhem.se>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -19,11 +19,10 @@
 # pylint: disable=missing-module-docstring
 
 import os
-from os import path as os_path, curdir as os_curdir, pardir as os_pardir, sep as os_sep
+from os import path as os_path, curdir as os_curdir
 import sys
 from fnmatch import fnmatchcase
 from pathlib import Path, PosixPath
-from urllib.parse import urlparse
 from stat import S_ISDIR
 import math
 import logging
@@ -70,11 +69,6 @@ def find_config(filename : str = 'config.yml', appname : str = 'webmap') -> Path
             return p
     raise MissingConfiguration(filename)
 
-class BadConfiguration(Exception):
-    """Exception raised when there is a bad configuration"""
-    def __init__(self, config_path : Path, message : str) -> Never:
-        super().__init__(str(config_path) + ': ' + message)
-
 def parse_config(path : Optional[Path] = None,
                  groupnames : Optional[list[str]] = None) -> dict[str, Any]:
     """Parse configuration file"""
@@ -82,77 +76,9 @@ def parse_config(path : Optional[Path] = None,
     config_path = find_config() if path is None else path
     with config_path.open(mode='r', encoding='utf-8') as fp:
         config = yaml.safe_load(fp)
-    layers = config.get('layers', {})
-
-    # validate sources
-    destinations = {}
-    for name, layerdefs in layers.items():
-        if isinstance(layerdefs, dict) and 'sources' not in layerdefs:
-            layers[name] = { 'sources': [layerdefs] }
-            for k in ['description', 'create', 'publish']:
-                if k in layerdefs:
-                    layers[name][k] = layerdefs.pop(k)
-            layerdefs = layers[name]
-
-        if 'sources' not in layerdefs:
-            # pylint: disable-next=broad-exception-raised
-            raise Exception(f'Layer "{name}" does not have any source recipe')
-
-        for sourcedef in layerdefs.get('sources', []):
-            source = sourcedef.get('source', None)
-            if source is None:
-                continue
-            download = source.get('download', None)
-            if download is None:
-                url = None
-                dl_module = None
-            elif isinstance(download, str):
-                url = download
-                dl_module = None
-                source['download'] = download = { 'url': url }
-            else:
-                url = download.get('url', None)
-                dl_module = download.get('module', None)
-            if url is None:
-                urlp = None
-            else:
-                urlp = urlparse(url)
-                if urlp is None:
-                    # pylint: disable-next=broad-exception-raised
-                    raise Exception(f'urlparse({url}) failed')
-
-            cache = source.get('cache', None)
-            if cache is None or isinstance(cache, str):
-                source['cache'] = { 'path': cache }
-            else:
-                cache = cache.get('path', None)
-
-            if cache is None or cache in ['', os_curdir, os_pardir] or cache.endswith(os_sep):
-                # infer filename from the source URL
-                if urlp is None or urlp.path is None or urlp.path == '' or urlp.path.endswith('/'):
-                    # pylint: disable-next=broad-exception-raised
-                    raise Exception(f'Layer "{name}": Could not infer filename from URL {url}')
-                p = PosixPath(urlp.path)
-                if p is None or p.name is None or p.name == '':
-                    # pylint: disable-next=broad-exception-raised
-                    raise Exception(f'Invalid PosixPath({urlp.path})')
-                if cache is None or cache == '':
-                    cache = Path()
-                else:
-                    cache = Path(cache)
-                cache = cache.joinpath(p.name)
-            else:
-                cache = Path(cache)
-            source['cache']['path'] = cache
-
-            v = { 'url': urlp, 'module': dl_module }
-            if cache in destinations and destinations[cache] != v:
-                # allow destination conflicts, but only when the source URL and module match
-                # pylint: disable-next=broad-exception-raised
-                raise Exception(f'Destination conflict for layer "{name}"')
-            destinations[cache] = v
 
     # filter layers that are not of interest
+    layers = config.get('layers', {})
     if groupnames is not None:
         layernames = []
         layer_groups = config.get('layer-groups', {})