diff options
Diffstat (limited to 'webmap-download')
| -rwxr-xr-x | webmap-download | 35 |
1 files changed, 17 insertions, 18 deletions
diff --git a/webmap-download b/webmap-download index f87cddf..be48bbb 100755 --- a/webmap-download +++ b/webmap-download @@ -30,7 +30,12 @@ def download_trystream(url, **kwargs): r.raise_for_status() return r -def download(url, dest, dir_fd=None, headers={}, max_size=None, session=requests, progress=None): +def download(url, dest, dir_fd=None, headers={}, session=requests, progress=None): + url = None if dl is None else dl.get('url', None) + if url is None: + logging.error('%s has no source URL, ignoring', dest) + return + max_size = dl.get('max-size', 2**26) # 64MiB logging.info('Downloading %s…', url) destPath = Path(dest) dest_tmp = destPath.with_stem(f'.{destPath.stem}.new').as_posix() @@ -175,24 +180,18 @@ if __name__ == '__main__': downloads = set() for source in sources: dl = source.get('download', None) - script = None if dl is None else dl.get('script', None) - if script is not None and script != os.path.basename(__file__): - logging.info('Layer "%s" is not for us (%s != %s), skipping', - source['layername'], - script, os.path.basename(__file__)) - continue - - url = None if dl is None else dl.get('url', None) - if url is None: - logging.error('Layer "%s" has no source URL, ignoring', - source['layername']) - continue + dl_module = None if dl is None else dl.get('module', None) + if dl_module is None: + fetch = download + else: + dl_module = __import__(dl_module) + fetch = dl_module.download cache = source.get('cache', None) dest = None if cache is None else cache.get('path', None) if dest is None: raise Exception('Impossible') - elif url in downloads: + elif dest in downloads: logging.info('%s was already downloaded, skipping', dest) continue @@ -231,16 +230,16 @@ if __name__ == '__main__': dest, format_time(s)) continue headers['If-Modified-Since'] = formatdate(timeval=st.st_mtime, localtime=False, usegmt=True) - max_size = dl.get('max-size', 2**26) # 64MiB - download(url, dest, dir_fd=destdir_fd, max_size=max_size, + fetch(dl, dest, dir_fd=destdir_fd, headers=headers, session=sessionRequests, progress=pbar) - downloads.add(url) + downloads.add(dest) finally: if lockdir_fd is not None: os.close(lock_fd) except Exception: - logging.exception('Could not download %s as %s', url, dest) + logging.exception('Could not download %s as %s', + dl.get('url', source['layername']), dest) if args.exit_code: rv = 1 exit(rv) |