From ad38438a0b980ee816e1573bf18362e72345fa4d Mon Sep 17 00:00:00 2001 From: Guilhem Moulin Date: Tue, 22 Apr 2025 21:33:42 +0200 Subject: webmap-import: Fingerprint destination layers to detect changes. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Comparing modification times is not enough since some sources (for instance Naturvårdsverket's SCI_Rikstackande) are updated on the server even though no objects are being added; the source layer remains unchanged but the file differs because of OBJECTID changes we are not interested in. --- webmap-import | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'webmap-import') diff --git a/webmap-import b/webmap-import index c930730..c801b6d 100755 --- a/webmap-import +++ b/webmap-import @@ -289,7 +289,8 @@ def processOutputLayer(ds : gdal.Dataset, return importSources(dso=ds, lyr=lyr, sources=sources, cachedir=cachedir, extent=extent, dsoTransaction=dsTransaction, - lyrcache=lyrcache) + lyrcache=lyrcache, + force=force) def validate_sources(layers : dict[str, Any]) -> None: """Mangle and validate layer sources and import definitions""" @@ -394,6 +395,8 @@ def validateCacheLayer(ds : gdal.Dataset, name : str) -> bool: 'nullable': False, 'unique': True, 'width': 255 }, { 'name': 'last_updated', 'typ': ogr.OFTDateTime, 'nullable': False }, + { 'name': 'fingerprint', 'typ': ogr.OFTBinary, + 'nullable': False, 'width': 32 }, ] m = len(fields) n = defn.GetFieldCount() @@ -495,8 +498,11 @@ def areSourceFilesNewer(layername : str, microsecond=round(ms*1000000), tzinfo=tz ) - logging.debug('Found entry in layer cache for "%s", last_updated=%s', layername, - dt.isoformat(timespec='microseconds')) + fpr = feature.GetFieldAsBinary(2) if feature.IsFieldSetAndNotNull(2) else None + logging.debug('Found entry in layer cache for "%s", last_updated=%s, fingerprint=%s', + layername, + dt.isoformat(timespec='microseconds'), + fpr.hex() if fpr is not None else 'NULL') ret = int(dt.timestamp() * 1000000.) * 1000 < t if lyrcache.GetNextFeature() is not None: -- cgit v1.2.3