From d25c8b8b60fbfabf7544a3a5a44c105f0bc5b4da Mon Sep 17 00:00:00 2001
From: Guilhem Moulin <guilhem@fripost.org>
Date: Sun, 2 Jun 2024 13:24:45 +0200
Subject: Rename script into module and run module.download().

Instead of using a dedicated executable.  There is too much code
duplicate otherwise.
---
 webmap-download | 35 +++++++++++++++++------------------
 1 file changed, 17 insertions(+), 18 deletions(-)

(limited to 'webmap-download')

diff --git a/webmap-download b/webmap-download
index f87cddf..be48bbb 100755
--- a/webmap-download
+++ b/webmap-download
@@ -30,7 +30,12 @@ def download_trystream(url, **kwargs):
             r.raise_for_status()
             return r
 
-def download(url, dest, dir_fd=None, headers={}, max_size=None, session=requests, progress=None):
+def download(url, dest, dir_fd=None, headers={}, session=requests, progress=None):
+    url = None if dl is None else dl.get('url', None)
+    if url is None:
+        logging.error('%s has no source URL, ignoring', dest)
+        return
+    max_size = dl.get('max-size', 2**26) # 64MiB
     logging.info('Downloading %s…', url)
     destPath = Path(dest)
     dest_tmp = destPath.with_stem(f'.{destPath.stem}.new').as_posix()
@@ -175,24 +180,18 @@ if __name__ == '__main__':
     downloads = set()
     for source in sources:
         dl = source.get('download', None)
-        script = None if dl is None else dl.get('script', None)
-        if script is not None and script != os.path.basename(__file__):
-            logging.info('Layer "%s" is not for us (%s != %s), skipping',
-                source['layername'],
-                script, os.path.basename(__file__))
-            continue
-
-        url = None if dl is None else dl.get('url', None)
-        if url is None:
-            logging.error('Layer "%s" has no source URL, ignoring',
-                source['layername'])
-            continue
+        dl_module = None if dl is None else dl.get('module', None)
+        if dl_module is None:
+            fetch = download
+        else:
+            dl_module = __import__(dl_module)
+            fetch = dl_module.download
 
         cache = source.get('cache', None)
         dest = None if cache is None else cache.get('path', None)
         if dest is None:
             raise Exception('Impossible')
-        elif url in downloads:
+        elif dest in downloads:
             logging.info('%s was already downloaded, skipping', dest)
             continue
 
@@ -231,16 +230,16 @@ if __name__ == '__main__':
                                 dest, format_time(s))
                             continue
                     headers['If-Modified-Since'] = formatdate(timeval=st.st_mtime, localtime=False, usegmt=True)
-                max_size = dl.get('max-size', 2**26) # 64MiB
-                download(url, dest, dir_fd=destdir_fd, max_size=max_size,
+                fetch(dl, dest, dir_fd=destdir_fd,
                     headers=headers, session=sessionRequests,
                     progress=pbar)
-                downloads.add(url)
+                downloads.add(dest)
             finally:
                 if lockdir_fd is not None:
                     os.close(lock_fd)
         except Exception:
-            logging.exception('Could not download %s as %s', url, dest)
+            logging.exception('Could not download %s as %s',
+                              dl.get('url', source['layername']), dest)
             if args.exit_code:
                 rv = 1
     exit(rv)
-- 
cgit v1.2.3