Add logic to export raster files (as COG).

Raster data is not stored in the PostGIS database.  Instead, the mtime
of the target directory is used to determine whether the COG is up to
date.

Add a new flag --metadata-compress for JSON metadata compression (which
also applies to MVT metadata), and --rasterdir for the target raster
directory.

1 files changed, 14 insertions, 9 deletions

diff --git a/import_source.py b/import_source.py
index 0431486..d5931ad 100644
--- a/import_source.py
+++ b/import_source.py
@@ -26,7 +26,7 @@ import re
 from fnmatch import fnmatchcase
 from pathlib import Path
 from datetime import datetime, timedelta, UTC
-from typing import Any, Final, Iterator, Optional
+from typing import Any, Callable, Final, Iterator, Optional
 import traceback
 from enum import Enum, unique as enum_unique
 from hashlib import sha256
@@ -37,6 +37,7 @@ from osgeo.gdalconst import (
     OF_ALL as GDAL_OF_ALL,
     OF_READONLY as GDAL_OF_READONLY,
     OF_UPDATE as GDAL_OF_UPDATE,
+    OF_VECTOR as GDAL_OF_VECTOR,
     OF_VERBOSE_ERROR as GDAL_OF_VERBOSE_ERROR,
     DCAP_CREATE as GDAL_DCAP_CREATE,
 )
@@ -56,7 +57,8 @@ def openOutputDS(def_dict : dict[str, Any]) -> gdal.Dataset:
     create-options is a non-empty dictionary."""
 
     path = def_dict['path']
-    kwargs, drv = gdalSetOpenExArgs(def_dict, flags=GDAL_OF_UPDATE|GDAL_OF_VERBOSE_ERROR)
+    kwargs, drv = gdalSetOpenExArgs(def_dict,
+                                    flags=GDAL_OF_VECTOR|GDAL_OF_UPDATE|GDAL_OF_VERBOSE_ERROR)
     try:
         logging.debug('OpenEx(%s, %s)', path, str(kwargs))
         return gdal.OpenEx(path, **kwargs)
@@ -485,10 +487,11 @@ def importSources(dso : gdal.Dataset, lyr : ogr.Layer,
         clearLayer(dso, lyr) # TODO conditional (only if not new)?
 
         for source in sources:
-            _importSource(lyr, **source['source'],
+            importSource0(lyr, **source['source'],
                           args=source['import'],
                           cachedir=cachedir,
-                          extent=extent)
+                          extent=extent,
+                          callback=_importSource2)
 
         # force the PG driver to call EndCopy() to detect errors and trigger a
         # rollback if needed
@@ -550,15 +553,17 @@ def importSources(dso : gdal.Dataset, lyr : ogr.Layer,
     return rv
 
 # pylint: disable-next=dangerous-default-value
-def _importSource(lyr : ogr.Layer,
+def importSource0(lyr : ogr.Layer|None = None,
                   path : str = '/nonexistent',
                   unar : dict[str,Any]|None = None,
                   args : dict[str,Any] = {},
                   cachedir : Path|None = None,
-                  extent : ogr.Geometry|None = None) -> None:
+                  extent : ogr.Geometry|None = None,
+                  callback : Callable[[ogr.Layer|None, str, dict[str,Any], Path|None,
+                                      ogr.Geometry|None], None]|None = None) -> None:
     """Import a source layer"""
     if unar is None:
-        return _importSource2(lyr, path, args=args, basedir=cachedir, extent=extent)
+        return callback(lyr, path, args=args, basedir=cachedir, extent=extent)
 
     ds_srcpath = Path(args['path'])
     if ds_srcpath.is_absolute():
@@ -574,7 +579,7 @@ def _importSource(lyr : ogr.Layer,
                        fmt=unar.get('format', None),
                        patterns=unar.get('patterns', None),
                        exact_matches=[ds_srcpath])
-        return _importSource2(lyr, ds_srcpath, args=args, basedir=Path(tmpdir), extent=extent)
+        return callback(lyr, ds_srcpath, args=args, basedir=Path(tmpdir), extent=extent)
 
 def setFieldMapValue(fld : ogr.FieldDefn,
                      idx : int,
@@ -613,7 +618,7 @@ def _importSource2(lyr_dst : ogr.Layer, path : str, args : dict[str,Any],
     calling StartTransaction() https://github.com/OSGeo/gdal/issues/3403
     while we want a single transaction for the entire desination layer,
     including truncation, source imports, and metadata changes."""
-    kwargs, _ = gdalSetOpenExArgs(args, flags=GDAL_OF_READONLY|GDAL_OF_VERBOSE_ERROR)
+    kwargs, _ = gdalSetOpenExArgs(args, flags=GDAL_OF_VECTOR|GDAL_OF_READONLY|GDAL_OF_VERBOSE_ERROR)
     path2 = path if basedir is None else str(basedir.joinpath(path))
 
     logging.debug('OpenEx(%s, %s)', path2, str(kwargs))