From 1af347391f9f54b370dfc7395464b8ed637a79ca Mon Sep 17 00:00:00 2001 From: Guilhem Moulin Date: Fri, 6 Mar 2026 13:50:24 +0100 Subject: Rename "webmap" to the less generic "geodata". The database has uses beyond the webmap. Cf. ca91a579770c89d25aefae220079bf336fa88dc9 in tools. --- files/etc/systemd/system/geodata-download@.service | 37 ++ files/etc/systemd/system/geodata-import@.service | 41 +++ files/etc/systemd/system/geodata-raster@.service | 40 +++ files/etc/systemd/system/geodata-update@.target | 3 + files/etc/systemd/system/geodata-update@.timer | 11 + files/etc/systemd/system/webmap-download@.service | 37 -- files/etc/systemd/system/webmap-import@.service | 41 --- files/etc/systemd/system/webmap-raster@.service | 40 --- files/etc/systemd/system/webmap-update@.target | 3 - files/etc/systemd/system/webmap-update@.timer | 11 - files/etc/tmpfiles.d/geodata.conf | 8 + files/etc/tmpfiles.d/webmap.conf | 8 - group_vars/all.yml | 8 +- setup.yml | 4 + tasks/geodata.yml | 211 ++++++++++++ tasks/postgis.yml | 158 +++++++++ tasks/webmap.yml | 379 +-------------------- .../geodata-update@.timer.d/override.conf.j2 | 3 + templates/etc/systemd/system/webmap-cgi.service | 2 +- .../system/webmap-update@.timer.d/override.conf.j2 | 3 - webmap-tools | 2 +- 21 files changed, 526 insertions(+), 524 deletions(-) create mode 100644 files/etc/systemd/system/geodata-download@.service create mode 100644 files/etc/systemd/system/geodata-import@.service create mode 100644 files/etc/systemd/system/geodata-raster@.service create mode 100644 files/etc/systemd/system/geodata-update@.target create mode 100644 files/etc/systemd/system/geodata-update@.timer delete mode 100644 files/etc/systemd/system/webmap-download@.service delete mode 100644 files/etc/systemd/system/webmap-import@.service delete mode 100644 files/etc/systemd/system/webmap-raster@.service delete mode 100644 files/etc/systemd/system/webmap-update@.target delete mode 100644 files/etc/systemd/system/webmap-update@.timer create mode 100644 files/etc/tmpfiles.d/geodata.conf delete mode 100644 files/etc/tmpfiles.d/webmap.conf create mode 100644 tasks/geodata.yml create mode 100644 tasks/postgis.yml create mode 100644 templates/etc/systemd/system/geodata-update@.timer.d/override.conf.j2 delete mode 100644 templates/etc/systemd/system/webmap-update@.timer.d/override.conf.j2 diff --git a/files/etc/systemd/system/geodata-download@.service b/files/etc/systemd/system/geodata-download@.service new file mode 100644 index 0000000..2a8c940 --- /dev/null +++ b/files/etc/systemd/system/geodata-download@.service @@ -0,0 +1,37 @@ +[Unit] +Description=Geodata updater service (download ‘%I’) +# Chaining logic from https://serverfault.com/questions/1079993/why-does-my-systemd-timer-only-trigger-once-when-the-unit-is-a-target#answer-1128671 +# XXX Looks like Upholds= prevents running a single unit, as it causes +# geodata-update@%i.target to start upon `systemctl start geodata-download@foo.service` +After=network-online.target geodata-update@%i.target +Upholds=geodata-update@%i.target + +[Service] +User=_geodata-download +Group=_geodata + +Nice=15 +IOSchedulingClass=idle + +Type=oneshot +ExecStart=/usr/local/bin/geodata-download \ + --cachedir=%C/geodata \ + --lockdir=%t/lock/geodata/cache \ + --no-exit-code \ + --quiet \ + -- %I + +# Hardening +NoNewPrivileges=yes +ProtectHome=yes +ProtectSystem=strict +PrivateDevices=yes +ProtectControlGroups=yes +ProtectKernelModules=yes +ProtectKernelTunables=yes +RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 +ReadWritePaths=%C/geodata +ReadWritePaths=%t/lock/geodata/cache + +[Install] +WantedBy=geodata-update@%i.target diff --git a/files/etc/systemd/system/geodata-import@.service b/files/etc/systemd/system/geodata-import@.service new file mode 100644 index 0000000..7d652ea --- /dev/null +++ b/files/etc/systemd/system/geodata-import@.service @@ -0,0 +1,41 @@ +[Unit] +Description=Geodata updater service (import ‘%I’ to PostGIS) +After=postgresql.service geodata-update@%i.target +After=geodata-download@%i.service +Upholds=geodata-update@%i.target + +[Service] +User=_geodata +Group=_geodata + +Nice=15 +IOSchedulingClass=idle + +# Point TMPDIR to something that is not a tmpfs as we need to unpack large archives +Environment=TMPDIR=/var/tmp + +Type=oneshot +ExecStart=/usr/local/bin/geodata-import \ + --cachedir=%C/geodata \ + --lockfile=%t/lock/geodata/lock \ + --lockdir-sources=%t/lock/geodata/cache \ + --mvtdir=/var/www/webmap/tiles/%I \ + --mvt-compress \ + --metadata-compress \ + -- %I + +# Hardening +NoNewPrivileges=yes +ProtectHome=yes +ProtectSystem=strict +PrivateDevices=yes +ProtectControlGroups=yes +ProtectKernelModules=yes +ProtectKernelTunables=yes +RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 +ReadWritePaths=%t/lock/geodata +ReadWritePaths=/var/www/webmap/tiles +PrivateTmp=yes + +[Install] +WantedBy=geodata-update@%i.target diff --git a/files/etc/systemd/system/geodata-raster@.service b/files/etc/systemd/system/geodata-raster@.service new file mode 100644 index 0000000..aed7930 --- /dev/null +++ b/files/etc/systemd/system/geodata-raster@.service @@ -0,0 +1,40 @@ +[Unit] +Description=Geodata updater service (export ‘%I’ to COG) +After=geodata-update@%i.target +After=geodata-download@%i.service +Upholds=geodata-update@%i.target + +[Service] +User=_geodata +Group=_geodata + +Nice=15 +IOSchedulingClass=idle + +# Point TMPDIR to something that is not a tmpfs as we need to unpack large archives +Environment=TMPDIR=/var/tmp + +Type=oneshot +ExecStart=/usr/local/bin/geodata-import \ + --cachedir=%C/geodata \ + --lockfile=%t/lock/geodata/lock \ + --lockdir-sources=%t/lock/geodata/cache \ + --rasterdir=/var/www/webmap/raster/%I \ + --metadata-compress \ + -- %I + +# Hardening +NoNewPrivileges=yes +ProtectHome=yes +ProtectSystem=strict +PrivateDevices=yes +ProtectControlGroups=yes +ProtectKernelModules=yes +ProtectKernelTunables=yes +RestrictAddressFamilies=AF_UNIX +ReadWritePaths=%t/lock/geodata +ReadWritePaths=/var/www/webmap/raster +PrivateTmp=yes + +[Install] +WantedBy=geodata-update@%i.target diff --git a/files/etc/systemd/system/geodata-update@.target b/files/etc/systemd/system/geodata-update@.target new file mode 100644 index 0000000..e7cdecb --- /dev/null +++ b/files/etc/systemd/system/geodata-update@.target @@ -0,0 +1,3 @@ +[Unit] +Description=Geodata updater (target unit ‘%I’) +StopWhenUnneeded=true diff --git a/files/etc/systemd/system/geodata-update@.timer b/files/etc/systemd/system/geodata-update@.timer new file mode 100644 index 0000000..90fd865 --- /dev/null +++ b/files/etc/systemd/system/geodata-update@.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Geodata updater (timer unit) + +[Timer] +OnCalendar=*-*-* 01:00:00 +AccuracySec=1s +RandomizedDelaySec=3599 +Unit=geodata-update@%i.target + +[Install] +WantedBy=timers.target diff --git a/files/etc/systemd/system/webmap-download@.service b/files/etc/systemd/system/webmap-download@.service deleted file mode 100644 index d7a49dc..0000000 --- a/files/etc/systemd/system/webmap-download@.service +++ /dev/null @@ -1,37 +0,0 @@ -[Unit] -Description=Webmap updater service (download ‘%I’) -# Chaining logic from https://serverfault.com/questions/1079993/why-does-my-systemd-timer-only-trigger-once-when-the-unit-is-a-target#answer-1128671 -# XXX Looks like Upholds= prevents running a single unit, as it causes -# webmap-update@%i.target to start upon `systemctl start webmap-download@foo.service` -After=network-online.target webmap-update@%i.target -Upholds=webmap-update@%i.target - -[Service] -User=_webmap-download -Group=_webmap - -Nice=15 -IOSchedulingClass=idle - -Type=oneshot -ExecStart=/usr/local/bin/webmap-download \ - --cachedir=%C/webmap \ - --lockdir=%t/lock/webmap/cache \ - --no-exit-code \ - --quiet \ - -- %I - -# Hardening -NoNewPrivileges=yes -ProtectHome=yes -ProtectSystem=strict -PrivateDevices=yes -ProtectControlGroups=yes -ProtectKernelModules=yes -ProtectKernelTunables=yes -RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 -ReadWritePaths=%C/webmap -ReadWritePaths=%t/lock/webmap/cache - -[Install] -WantedBy=webmap-update@%i.target diff --git a/files/etc/systemd/system/webmap-import@.service b/files/etc/systemd/system/webmap-import@.service deleted file mode 100644 index e2a6eb4..0000000 --- a/files/etc/systemd/system/webmap-import@.service +++ /dev/null @@ -1,41 +0,0 @@ -[Unit] -Description=Webmap updater service (import ‘%I’ to PostGIS) -After=postgresql.service webmap-update@%i.target -After=webmap-download@%i.service -Upholds=webmap-update@%i.target - -[Service] -User=_webmap -Group=_webmap - -Nice=15 -IOSchedulingClass=idle - -# Point TMPDIR to something that is not a tmpfs as we need to unpack large archives -Environment=TMPDIR=/var/tmp - -Type=oneshot -ExecStart=/usr/local/bin/webmap-import \ - --cachedir=%C/webmap \ - --lockfile=%t/lock/webmap/lock \ - --lockdir-sources=%t/lock/webmap/cache \ - --mvtdir=/var/www/webmap/tiles/%I \ - --mvt-compress \ - --metadata-compress \ - -- %I - -# Hardening -NoNewPrivileges=yes -ProtectHome=yes -ProtectSystem=strict -PrivateDevices=yes -ProtectControlGroups=yes -ProtectKernelModules=yes -ProtectKernelTunables=yes -RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 -ReadWritePaths=%t/lock/webmap -ReadWritePaths=/var/www/webmap/tiles -PrivateTmp=yes - -[Install] -WantedBy=webmap-update@%i.target diff --git a/files/etc/systemd/system/webmap-raster@.service b/files/etc/systemd/system/webmap-raster@.service deleted file mode 100644 index 42a97cf..0000000 --- a/files/etc/systemd/system/webmap-raster@.service +++ /dev/null @@ -1,40 +0,0 @@ -[Unit] -Description=Webmap updater service (export ‘%I’ to COG) -After=webmap-update@%i.target -After=webmap-download@%i.service -Upholds=webmap-update@%i.target - -[Service] -User=_webmap -Group=_webmap - -Nice=15 -IOSchedulingClass=idle - -# Point TMPDIR to something that is not a tmpfs as we need to unpack large archives -Environment=TMPDIR=/var/tmp - -Type=oneshot -ExecStart=/usr/local/bin/webmap-import \ - --cachedir=%C/webmap \ - --lockfile=%t/lock/webmap/lock \ - --lockdir-sources=%t/lock/webmap/cache \ - --rasterdir=/var/www/webmap/raster/%I \ - --metadata-compress \ - -- %I - -# Hardening -NoNewPrivileges=yes -ProtectHome=yes -ProtectSystem=strict -PrivateDevices=yes -ProtectControlGroups=yes -ProtectKernelModules=yes -ProtectKernelTunables=yes -RestrictAddressFamilies=AF_UNIX -ReadWritePaths=%t/lock/webmap -ReadWritePaths=/var/www/webmap/raster -PrivateTmp=yes - -[Install] -WantedBy=webmap-update@%i.target diff --git a/files/etc/systemd/system/webmap-update@.target b/files/etc/systemd/system/webmap-update@.target deleted file mode 100644 index 840de96..0000000 --- a/files/etc/systemd/system/webmap-update@.target +++ /dev/null @@ -1,3 +0,0 @@ -[Unit] -Description=Webmap updater (target unit ‘%I’) -StopWhenUnneeded=true diff --git a/files/etc/systemd/system/webmap-update@.timer b/files/etc/systemd/system/webmap-update@.timer deleted file mode 100644 index 74fb848..0000000 --- a/files/etc/systemd/system/webmap-update@.timer +++ /dev/null @@ -1,11 +0,0 @@ -[Unit] -Description=Webmap updater (timer unit) - -[Timer] -OnCalendar=*-*-* 01:00:00 -AccuracySec=1s -RandomizedDelaySec=3599 -Unit=webmap-update@%i.target - -[Install] -WantedBy=timers.target diff --git a/files/etc/tmpfiles.d/geodata.conf b/files/etc/tmpfiles.d/geodata.conf new file mode 100644 index 0000000..a299e0f --- /dev/null +++ b/files/etc/tmpfiles.d/geodata.conf @@ -0,0 +1,8 @@ +d %t/lock/geodata 00755 root root + +# for `geodata-download --lockdir` *and* `geodata-import --lockdir-sources` +# (hence the set-group-ID bit and g+w) +d %t/lock/geodata/cache 02775 _geodata-download _geodata + +# for `geodata-import --lockfile` +f %t/lock/geodata/lock 00644 _geodata _geodata diff --git a/files/etc/tmpfiles.d/webmap.conf b/files/etc/tmpfiles.d/webmap.conf deleted file mode 100644 index 786e6dd..0000000 --- a/files/etc/tmpfiles.d/webmap.conf +++ /dev/null @@ -1,8 +0,0 @@ -d %t/lock/webmap 00755 root root - -# for `webmap-download --lockdir` *and* `webmap-import --lockdir-sources` -# (hence the set-group-ID bit and g+w) -d %t/lock/webmap/cache 02775 _webmap-download _webmap - -# for `webmap-import --lockfile` -f %t/lock/webmap/lock 00644 _webmap _webmap diff --git a/group_vars/all.yml b/group_vars/all.yml index 4caa775..3531f04 100644 --- a/group_vars/all.yml +++ b/group_vars/all.yml @@ -1,7 +1,7 @@ --- # The list of layer groups to process, see # webmap-tools/config.yml:layer-groups. -webmap_layer_groups: +geodata_layer_groups: - adm - mrr - skydd @@ -13,15 +13,15 @@ webmap_layer_groups: - misc - nv -webmap_raster: +geodata_raster: - kskog -webmap_layer_groups_nodownload: +geodata_layer_groups_nodownload: - adm - misc # adjust calendar events for individual units -webmap_layer_groups_update_calendar: +geodata_layer_groups_update_calendar: mrr: "*-*-* 05:00:00" # updated daily at 04:33 CEST # PostgreSQL's version number and cluster name diff --git a/setup.yml b/setup.yml index 33780a8..00adee4 100644 --- a/setup.yml +++ b/setup.yml @@ -22,6 +22,10 @@ tags: - mail - postfix + - import_tasks: ./tasks/geodata.yml + tags: geodata + - import_tasks: ./tasks/postgis.yml + tags: postgis - import_tasks: ./tasks/webmap.yml tags: webmap - import_tasks: ./tasks/httpd.yml diff --git a/tasks/geodata.yml b/tasks/geodata.yml new file mode 100644 index 0000000..fcf3471 --- /dev/null +++ b/tasks/geodata.yml @@ -0,0 +1,211 @@ +- name: Install gdal-bin + apt: pkg=gdal-bin install-recommends=true + +- name: Install unzip + apt: pkg=unzip + +- name: Install python dependencies + apt: pkg={{ packages }} + vars: + packages: + - python3 + - python3-brotli + - python3-gdal + - python3-requests + - python3-systemd + - python3-tqdm + - python3-urllib3 + - python3-xdg + - python3-yaml + +- name: Create directory /etc/geodata + file: path=/etc/geodata + state=directory + owner=root group=root + mode=0755 + +- name: Copy /etc/geodata/config.yml + copy: src=webmap-tools/config.yml + dest=/etc/geodata/config.yml + owner=root group=root + mode=0644 + +- name: Create directory /usr/local/share/geodata + file: path=/usr/local/share/geodata + state=directory + owner=root group=root + mode=0755 + +- name: Copy /usr/local/share/geodata/*.py modules + copy: src=webmap-tools/{{ item }} + dest=/usr/local/share/geodata/{{ item }} + owner=root group=root + mode=0644 + with_items: + # TODO these should be compiled + - common.py + - common_gdal.py + - import_source.py + - export_mvt.py + - export_raster.py + - rename_exchange.py + +- name: Copy geodata-update@.target + copy: src=etc/systemd/system/geodata-update@.target + dest=/etc/systemd/system/geodata-update@.target + owner=root group=root + mode=0644 + notify: + - systemctl daemon-reload + +- name: Copy geodata-update@.timer + copy: src=etc/systemd/system/geodata-update@.timer + dest=/etc/systemd/system/geodata-update@.timer + owner=root group=root + mode=0644 + notify: + - systemctl daemon-reload + +- name: Create directory /etc/systemd/system/geodata-update@*.timer.d + file: path=/etc/systemd/system/geodata-update@{{ item }}.timer.d + state=directory + owner=root group=root + mode=0755 + with_items: "{{ geodata_layer_groups_update_calendar.keys() | list }}" + +- name: Copy /etc/systemd/system/geodata-update@*.timer.d/override.conf + template: src=etc/systemd/system/geodata-update@.timer.d/override.conf.j2 + dest=/etc/systemd/system/geodata-update@{{ item }}.timer.d/override.conf + owner=root group=root + mode=0644 + with_items: "{{ geodata_layer_groups_update_calendar.keys() | list }}" + notify: + - systemctl daemon-reload + +- name: Enable geodata-update.timer + service: name=geodata-update@{{ item }}.timer state=started enabled=true + with_items: "{{ geodata_layer_groups | union(geodata_raster) }}" + +- meta: flush_handlers + + +- name: Create system group '_geodata' + group: name=_geodata system=true + state=present + +- name: Create system user '_geodata-download' + user: name=_geodata-download system=true + group=_geodata + createhome=false + home=/nonexistent + shell=/usr/sbin/nologin + comment="geodata update (download)" + password="!" + state=present + +- name: Copy /usr/local/share/geodata/download.py + copy: src=webmap-tools/geodata-download + dest=/usr/local/share/geodata/download.py + owner=root group=root + mode=0755 + +- name: Create /usr/local/bin/geodata-download + file: src=../share/geodata/download.py + dest=/usr/local/bin/geodata-download + owner=root group=root + state=link force=yes + +- name: Create directory /var/cache/geodata + file: path=/var/cache/geodata + state=directory + owner=_geodata-download group=root + mode=0755 + +- name: Create directory /var/cache/geodata/custom + file: path=/var/cache/geodata/custom + state=directory + owner=root group=root + mode=0755 + +- name: Copy custom layers into /var/cache/geodata/custom + copy: src=webmap-tools/layers/custom/ + dest=/var/cache/geodata/custom/ + owner=root group=root + mode=0644 + directory_mode=0755 + +- name: Copy geodata-download@.service + copy: src=etc/systemd/system/geodata-download@.service + dest=/etc/systemd/system/geodata-download@.service + owner=root group=root + mode=0644 + notify: + - systemctl daemon-reload + +- name: Enable geodata-download@.service + service: name=geodata-download@{{ item }}.service enabled=true + with_items: "{{ geodata_layer_groups | union(geodata_raster) | difference(geodata_layer_groups_nodownload) }}" + +- name: Disable some geodata-download@.service + service: name=geodata-download@{{ item }}.service enabled=false + with_items: "{{ geodata_layer_groups_nodownload }}" + +- meta: flush_handlers + + +- name: Copy /etc/tmpfiles.d/geodata.conf + copy: src=etc/tmpfiles.d/geodata.conf + dest=/etc/tmpfiles.d/geodata.conf + owner=root group=root + mode=0644 + notify: + - systemd-tmpfiles --create + +- meta: flush_handlers + + +- name: Create system user '_geodata' + user: name=_geodata system=true + group=_geodata + createhome=false + home=/nonexistent + shell=/usr/sbin/nologin + comment="geodata update (extract/import)" + password="!" + state=present + +- name: Copy /usr/local/share/geodata/import.py + copy: src=webmap-tools/geodata-import + dest=/usr/local/share/geodata/import.py + owner=root group=root + mode=0755 + +- name: Create /usr/local/bin/geodata-import + file: src=../share/geodata/import.py + dest=/usr/local/bin/geodata-import + owner=root group=root + state=link force=yes + +- name: Copy geodata-import@.service + copy: src=etc/systemd/system/geodata-import@.service + dest=/etc/systemd/system/geodata-import@.service + owner=root group=root + mode=0644 + notify: + - systemctl daemon-reload + +- name: Enable geodata-import@.service + service: name=geodata-import@{{ item }}.service enabled=true + with_items: "{{ geodata_layer_groups }}" + +- name: Copy geodata-raster@.service + copy: src=etc/systemd/system/geodata-raster@.service + dest=/etc/systemd/system/geodata-raster@.service + owner=root group=root + mode=0644 + notify: + - systemctl daemon-reload + +- name: Enable geodata-raster@.service + service: name=geodata-raster@{{ item }}.service enabled=true + with_items: "{{ geodata_raster }}" diff --git a/tasks/postgis.yml b/tasks/postgis.yml new file mode 100644 index 0000000..3e156a9 --- /dev/null +++ b/tasks/postgis.yml @@ -0,0 +1,158 @@ +- name: Install PostgreSQL and PostGIS + apt: pkg={{ packages }} + vars: + packages: + - postgresql + - postgresql-postgis + - postgis + # for ansible + - python3-psycopg + +- name: Generate sv_SE.UTF-8 locales + locale_gen: name=sv_SE.UTF-8 state=present + # PostgreSQL needs to be restarted to see the new locale + notify: Restart PostgreSQL + +- name: Configure PostgreSQL + copy: src=etc/postgresql/postgresql.conf + dest=/etc/postgresql/{{ postgresql.version }}/{{ postgresql.cluster }}/conf.d/local.conf + owner=postgres group=postgres + mode=0644 + notify: Restart PostgreSQL + +- name: Start PostgreSQL + service: name=postgresql@{{ postgresql.version }}-{{ postgresql.cluster }}.service state=started + +- meta: flush_handlers + +# Usage: \sudo -u postgres psql