diff options
31 files changed, 737 insertions, 148 deletions
diff --git a/ansible.cfg b/ansible.cfg index a35402e..cf64db8 100644 --- a/ansible.cfg +++ b/ansible.cfg @@ -1,5 +1,6 @@ [defaults] inventory = ./hosts.ini +max_diff_size = 524288 [privilege_escalation] become = True diff --git a/files/etc/apt/listchanges.conf b/files/etc/apt/listchanges.conf index 96910a0..713cbbd 100644 --- a/files/etc/apt/listchanges.conf +++ b/files/etc/apt/listchanges.conf @@ -6,4 +6,4 @@ email_format=text confirm=false headers=false reverse=false -save_seen=/var/lib/apt/listchanges.db +save_seen=/var/lib/apt/listchanges diff --git a/files/etc/nginx/sites-available/webmap b/files/etc/nginx/sites-available/webmap index d16ab60..bbc25c8 100644 --- a/files/etc/nginx/sites-available/webmap +++ b/files/etc/nginx/sites-available/webmap @@ -33,8 +33,9 @@ server { } server { - listen 443 ssl http2 default_server; - listen [::]:443 ssl http2 default_server; + listen 443 ssl default_server; + listen [::]:443 ssl default_server; + http2 on; server_name karta.klimatanalysnorr.se; @@ -45,36 +46,83 @@ server { ssl_certificate_key /etc/nginx/ssl/webmap.rsa.key; include snippets/ssl.conf; - add_header Referrer-Policy "no-referrer"; - add_header X-Frame-Options "SAMEORIGIN"; + root /var/www/webmap; + index index.html; + + add_header Referrer-Policy "same-origin"; + add_header X-Frame-Options "DENY"; add_header X-Content-Type-Options "nosniff"; add_header X-XSS-Protection "1; mode=block"; add_header Strict-Transport-Security "max-age=31557600; includeSubDomains" always; - add_header Content-Security-Policy "default-src 'none'; connect-src 'self'; font-src 'self'; frame-src 'self'; img-src 'self' data: https://minkarta.lantmateriet.se/map/; script-src 'self'; style-src 'self'; frame-ancestors 'self'; form-action 'none'; base-uri 'self'"; - - root /var/www/webmap; - index index.html; + add_header Content-Security-Policy "default-src 'none'; frame-ancestors 'none'; form-action 'none'; base-uri 'self'"; + #add_header Access-Control-Allow-Origin $http_origin always; + + include mime.types; + types { + # application/protobuf and application/vnd.google.protobuf might be valid types too, cf. + # https://stackoverflow.com/questions/30505408/what-is-the-correct-protobuf-content-type + application/x-protobuf pbf; + } location ^~ /assets/ { expires 7d; - gzip_static on; try_files $uri =404; + location ~ "\.(?:css|js|svg)$" { + brotli_static on; + } } location ^~ /tiles/ { - expires 1d; - gzip_static on; + expires 30m; + brotli_static on; try_files $uri =404; + # service an empty payload to save bandwidth error_page 404 /_.txt; } + location ^~ /raster/ { + expires 30m; + try_files $uri =404; + # service an empty payload to save bandwidth + error_page 404 /_.txt; + location ~ "\.json$" { + brotli_static on; + } + } + location = /q { + expires epoch; + limit_except POST { deny all; } + #if ($request_method = OPTIONS) { + # add_header Strict-Transport-Security "max-age=31557600; includeSubDomains"; + # add_header Access-Control-Allow-Origin $http_origin; + # add_header Access-Control-Allow-Methods "POST, GET, OPTIONS"; + # add_header Access-Control-Allow-Headers "Accept, Content-Type"; + # add_header Access-Control-Max-Age 28800; + # return 204; + #} + client_max_body_size 64k; + gzip on; + gzip_types application/json text/plain; + include uwsgi_params; + uwsgi_pass unix:/run/webmap-cgi.socket; + } location = /_.txt { - # cache 404 responses + # cache 404 responses for 30m like for valid tiles add_header Strict-Transport-Security "max-age=31557600; includeSubDomains" always; - add_header Cache-Control "public; max-age=86400" always; + add_header Cache-Control "public; max-age=1800" always; + #add_header Access-Control-Allow-Origin $http_origin always; internal; } location / { + add_header Referrer-Policy "same-origin"; + add_header X-Frame-Options "SAMEORIGIN"; + add_header X-Content-Type-Options "nosniff"; + add_header X-XSS-Protection "1; mode=block"; + add_header Strict-Transport-Security "max-age=31557600; includeSubDomains" always; + add_header Content-Security-Policy "default-src 'none'; connect-src 'self'; font-src 'self'; frame-src 'self'; img-src 'self' data: https://minkarta.lantmateriet.se/map/; script-src 'self'; style-src 'self'; frame-ancestors 'self'; form-action 'none'; worker-src blob:; base-uri 'self'"; + + expires 1h; + brotli_static on; try_files $uri $uri/ =404; } } diff --git a/files/etc/nginx/snippets/ssl.conf b/files/etc/nginx/snippets/ssl.conf index 0bce30a..b86f5e3 100644 --- a/files/etc/nginx/snippets/ssl.conf +++ b/files/etc/nginx/snippets/ssl.conf @@ -7,10 +7,3 @@ ssl_dhparam /etc/ssl/dhparams.pem; ssl_protocols TLSv1.2 TLSv1.3; ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305; ssl_prefer_server_ciphers off; - -ssl_stapling on; -ssl_stapling_verify on; - -ssl_trusted_certificate /usr/share/lacme/ca-certificates.crt; - -resolver 127.0.0.53; diff --git a/files/etc/postgresql/postgresql.conf b/files/etc/postgresql/postgresql.conf new file mode 100644 index 0000000..038438a --- /dev/null +++ b/files/etc/postgresql/postgresql.conf @@ -0,0 +1,4 @@ +shared_buffers = 768MB +temp_buffers = 128MB +work_mem = 16MB +effective_cache_size = 1536MB diff --git a/files/etc/systemd/system/webmap-download@.service b/files/etc/systemd/system/geodata-download@.service index a928a13..2a8c940 100644 --- a/files/etc/systemd/system/webmap-download@.service +++ b/files/etc/systemd/system/geodata-download@.service @@ -1,22 +1,22 @@ [Unit] -Description=Webmap updater service (download %I) +Description=Geodata updater service (download ‘%I’) # Chaining logic from https://serverfault.com/questions/1079993/why-does-my-systemd-timer-only-trigger-once-when-the-unit-is-a-target#answer-1128671 # XXX Looks like Upholds= prevents running a single unit, as it causes -# webmap-update@%i.target to start upon `systemctl start webmap-download@foo.service` -After=network-online.target webmap-update@%i.target -Upholds=webmap-update@%i.target +# geodata-update@%i.target to start upon `systemctl start geodata-download@foo.service` +After=network-online.target geodata-update@%i.target +Upholds=geodata-update@%i.target [Service] -User=_webmap-download -Group=nogroup +User=_geodata-download +Group=_geodata Nice=15 IOSchedulingClass=idle Type=oneshot -ExecStart=/usr/local/bin/webmap-download \ - --cachedir=/var/cache/webmap \ - --lockdir=%t/webmap-download \ +ExecStart=/usr/local/bin/geodata-download \ + --cachedir=%C/geodata \ + --lockdir=%t/lock/geodata/cache \ --no-exit-code \ --quiet \ -- %I @@ -30,9 +30,8 @@ ProtectControlGroups=yes ProtectKernelModules=yes ProtectKernelTunables=yes RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 -ReadWritePaths=/var/cache/webmap -RuntimeDirectory=webmap-download -RuntimeDirectoryPreserve=yes +ReadWritePaths=%C/geodata +ReadWritePaths=%t/lock/geodata/cache [Install] -WantedBy=webmap-update@%i.target +WantedBy=geodata-update@%i.target diff --git a/files/etc/systemd/system/geodata-import@.service b/files/etc/systemd/system/geodata-import@.service new file mode 100644 index 0000000..7d652ea --- /dev/null +++ b/files/etc/systemd/system/geodata-import@.service @@ -0,0 +1,41 @@ +[Unit] +Description=Geodata updater service (import ‘%I’ to PostGIS) +After=postgresql.service geodata-update@%i.target +After=geodata-download@%i.service +Upholds=geodata-update@%i.target + +[Service] +User=_geodata +Group=_geodata + +Nice=15 +IOSchedulingClass=idle + +# Point TMPDIR to something that is not a tmpfs as we need to unpack large archives +Environment=TMPDIR=/var/tmp + +Type=oneshot +ExecStart=/usr/local/bin/geodata-import \ + --cachedir=%C/geodata \ + --lockfile=%t/lock/geodata/lock \ + --lockdir-sources=%t/lock/geodata/cache \ + --mvtdir=/var/www/webmap/tiles/%I \ + --mvt-compress \ + --metadata-compress \ + -- %I + +# Hardening +NoNewPrivileges=yes +ProtectHome=yes +ProtectSystem=strict +PrivateDevices=yes +ProtectControlGroups=yes +ProtectKernelModules=yes +ProtectKernelTunables=yes +RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 +ReadWritePaths=%t/lock/geodata +ReadWritePaths=/var/www/webmap/tiles +PrivateTmp=yes + +[Install] +WantedBy=geodata-update@%i.target diff --git a/files/etc/systemd/system/geodata-raster@.service b/files/etc/systemd/system/geodata-raster@.service new file mode 100644 index 0000000..aed7930 --- /dev/null +++ b/files/etc/systemd/system/geodata-raster@.service @@ -0,0 +1,40 @@ +[Unit] +Description=Geodata updater service (export ‘%I’ to COG) +After=geodata-update@%i.target +After=geodata-download@%i.service +Upholds=geodata-update@%i.target + +[Service] +User=_geodata +Group=_geodata + +Nice=15 +IOSchedulingClass=idle + +# Point TMPDIR to something that is not a tmpfs as we need to unpack large archives +Environment=TMPDIR=/var/tmp + +Type=oneshot +ExecStart=/usr/local/bin/geodata-import \ + --cachedir=%C/geodata \ + --lockfile=%t/lock/geodata/lock \ + --lockdir-sources=%t/lock/geodata/cache \ + --rasterdir=/var/www/webmap/raster/%I \ + --metadata-compress \ + -- %I + +# Hardening +NoNewPrivileges=yes +ProtectHome=yes +ProtectSystem=strict +PrivateDevices=yes +ProtectControlGroups=yes +ProtectKernelModules=yes +ProtectKernelTunables=yes +RestrictAddressFamilies=AF_UNIX +ReadWritePaths=%t/lock/geodata +ReadWritePaths=/var/www/webmap/raster +PrivateTmp=yes + +[Install] +WantedBy=geodata-update@%i.target diff --git a/files/etc/systemd/system/geodata-update@.target b/files/etc/systemd/system/geodata-update@.target new file mode 100644 index 0000000..e7cdecb --- /dev/null +++ b/files/etc/systemd/system/geodata-update@.target @@ -0,0 +1,3 @@ +[Unit] +Description=Geodata updater (target unit ‘%I’) +StopWhenUnneeded=true diff --git a/files/etc/systemd/system/webmap-update@.timer b/files/etc/systemd/system/geodata-update@.timer index 74fb848..90fd865 100644 --- a/files/etc/systemd/system/webmap-update@.timer +++ b/files/etc/systemd/system/geodata-update@.timer @@ -1,11 +1,11 @@ [Unit] -Description=Webmap updater (timer unit) +Description=Geodata updater (timer unit) [Timer] OnCalendar=*-*-* 01:00:00 AccuracySec=1s RandomizedDelaySec=3599 -Unit=webmap-update@%i.target +Unit=geodata-update@%i.target [Install] WantedBy=timers.target diff --git a/files/etc/systemd/system/webmap-cgi.socket b/files/etc/systemd/system/webmap-cgi.socket new file mode 100644 index 0000000..2828985 --- /dev/null +++ b/files/etc/systemd/system/webmap-cgi.socket @@ -0,0 +1,11 @@ +[Unit] +Description=Webmap CGI (Common Gateway Interface) activation socket +After=syslog.target network.target + +[Socket] +ListenStream=%t/webmap-cgi.socket +SocketUser=www-data +SocketMode=0666 + +[Install] +WantedBy=sockets.target diff --git a/files/etc/systemd/system/webmap-update@.target b/files/etc/systemd/system/webmap-update@.target deleted file mode 100644 index 3d9fb7f..0000000 --- a/files/etc/systemd/system/webmap-update@.target +++ /dev/null @@ -1,3 +0,0 @@ -[Unit] -Description=Webmap updater (target unit %I) -StopWhenUnneeded=true diff --git a/files/etc/tmpfiles.d/geodata.conf b/files/etc/tmpfiles.d/geodata.conf new file mode 100644 index 0000000..a299e0f --- /dev/null +++ b/files/etc/tmpfiles.d/geodata.conf @@ -0,0 +1,8 @@ +d %t/lock/geodata 00755 root root + +# for `geodata-download --lockdir` *and* `geodata-import --lockdir-sources` +# (hence the set-group-ID bit and g+w) +d %t/lock/geodata/cache 02775 _geodata-download _geodata + +# for `geodata-import --lockfile` +f %t/lock/geodata/lock 00644 _geodata _geodata diff --git a/group_vars/all.yml b/group_vars/all.yml index a0d35c6..3531f04 100644 --- a/group_vars/all.yml +++ b/group_vars/all.yml @@ -1,9 +1,43 @@ --- # The list of layer groups to process, see # webmap-tools/config.yml:layer-groups. -webmap_layer_groups: +geodata_layer_groups: + - adm - mrr - - nvr - - sks - - st + - skydd + - avverk + - ren - vbk + - ri + - svk + - misc + - nv + +geodata_raster: + - kskog + +geodata_layer_groups_nodownload: + - adm + - misc + +# adjust calendar events for individual units +geodata_layer_groups_update_calendar: + mrr: "*-*-* 05:00:00" # updated daily at 04:33 CEST + +# PostgreSQL's version number and cluster name +postgresql: + version: 17 + cluster: main + +postgis_schemas: + - { name: ogr_system_tables, comment: "OGR metadata" } + - { name: lm_topo250, comment: "Lantmäteriets Topografi 250 (vektor)" } + - { name: mrr, comment: "Mineralrättigheter och prospektering" } + - { name: lst, comment: "Länsstyrelserna" } + - { name: sks, comment: "Skogsstyrelsen" } + - { name: nvk, comment: "Naturvårdsverket" } + - { name: nvr, comment: "Naturvårdsregistret" } + - { name: svk, comment: "Svenska kraftnät" } + - { name: vbk, comment: "Vindbrukskollen" } + - { name: sametinget, comment: "Sametinget" } + - { name: misc, comment: "Övriga skikt" } diff --git a/handlers/main.yml b/handlers/main.yml index c241ecc..b6ee548 100644 --- a/handlers/main.yml +++ b/handlers/main.yml @@ -5,6 +5,9 @@ - name: systemctl daemon-reload command: /usr/bin/systemctl daemon-reload +- name: systemd-tmpfiles --create + command: /usr/bin/systemd-tmpfiles --create + - name: Refresh hostname command: /usr/bin/hostnamectl hostname {{ inventory_hostname_short }} @@ -46,3 +49,12 @@ - name: Reload nginx service: name=nginx.service state=reloaded + +- name: Restart PostgreSQL + service: name=postgresql.service state=restarted + +- name: Reload PostgreSQL + service: name=postgresql.service state=reloaded + +- name: Stop webmap-cgi.service + service: name=webmap-cgi.service state=stopped @@ -22,6 +22,10 @@ tags: - mail - postfix + - import_tasks: ./tasks/geodata.yml + tags: geodata + - import_tasks: ./tasks/postgis.yml + tags: postgis - import_tasks: ./tasks/webmap.yml tags: webmap - import_tasks: ./tasks/httpd.yml diff --git a/tasks/apt.yml b/tasks/apt.yml index 1023908..f17a2e4 100644 --- a/tasks/apt.yml +++ b/tasks/apt.yml @@ -5,8 +5,13 @@ - apt - lsb-release -- name: Remove /etc/apt/sources.list - file: path=/etc/apt/sources.list state=absent +# something keeps recreating (without content) it if we delete it, so we +# leave it instead but ensure it's empty instead +- name: Create empty /etc/apt/sources.list + copy: content="" + dest=/etc/apt/sources.list + owner=root group=root + mode=0644 notify: - apt-get update diff --git a/tasks/firewall.yml b/tasks/firewall.yml index fa46ade..a4a7fee 100644 --- a/tasks/firewall.yml +++ b/tasks/firewall.yml @@ -3,7 +3,7 @@ - name: Configure nftables copy: src=etc/nftables.conf - dest=/etc/nftables + dest=/etc/nftables.conf owner=root group=root mode=0644 notify: diff --git a/tasks/geodata.yml b/tasks/geodata.yml new file mode 100644 index 0000000..fcf3471 --- /dev/null +++ b/tasks/geodata.yml @@ -0,0 +1,211 @@ +- name: Install gdal-bin + apt: pkg=gdal-bin install-recommends=true + +- name: Install unzip + apt: pkg=unzip + +- name: Install python dependencies + apt: pkg={{ packages }} + vars: + packages: + - python3 + - python3-brotli + - python3-gdal + - python3-requests + - python3-systemd + - python3-tqdm + - python3-urllib3 + - python3-xdg + - python3-yaml + +- name: Create directory /etc/geodata + file: path=/etc/geodata + state=directory + owner=root group=root + mode=0755 + +- name: Copy /etc/geodata/config.yml + copy: src=webmap-tools/config.yml + dest=/etc/geodata/config.yml + owner=root group=root + mode=0644 + +- name: Create directory /usr/local/share/geodata + file: path=/usr/local/share/geodata + state=directory + owner=root group=root + mode=0755 + +- name: Copy /usr/local/share/geodata/*.py modules + copy: src=webmap-tools/{{ item }} + dest=/usr/local/share/geodata/{{ item }} + owner=root group=root + mode=0644 + with_items: + # TODO these should be compiled + - common.py + - common_gdal.py + - import_source.py + - export_mvt.py + - export_raster.py + - rename_exchange.py + +- name: Copy geodata-update@.target + copy: src=etc/systemd/system/geodata-update@.target + dest=/etc/systemd/system/geodata-update@.target + owner=root group=root + mode=0644 + notify: + - systemctl daemon-reload + +- name: Copy geodata-update@.timer + copy: src=etc/systemd/system/geodata-update@.timer + dest=/etc/systemd/system/geodata-update@.timer + owner=root group=root + mode=0644 + notify: + - systemctl daemon-reload + +- name: Create directory /etc/systemd/system/geodata-update@*.timer.d + file: path=/etc/systemd/system/geodata-update@{{ item }}.timer.d + state=directory + owner=root group=root + mode=0755 + with_items: "{{ geodata_layer_groups_update_calendar.keys() | list }}" + +- name: Copy /etc/systemd/system/geodata-update@*.timer.d/override.conf + template: src=etc/systemd/system/geodata-update@.timer.d/override.conf.j2 + dest=/etc/systemd/system/geodata-update@{{ item }}.timer.d/override.conf + owner=root group=root + mode=0644 + with_items: "{{ geodata_layer_groups_update_calendar.keys() | list }}" + notify: + - systemctl daemon-reload + +- name: Enable geodata-update.timer + service: name=geodata-update@{{ item }}.timer state=started enabled=true + with_items: "{{ geodata_layer_groups | union(geodata_raster) }}" + +- meta: flush_handlers + + +- name: Create system group '_geodata' + group: name=_geodata system=true + state=present + +- name: Create system user '_geodata-download' + user: name=_geodata-download system=true + group=_geodata + createhome=false + home=/nonexistent + shell=/usr/sbin/nologin + comment="geodata update (download)" + password="!" + state=present + +- name: Copy /usr/local/share/geodata/download.py + copy: src=webmap-tools/geodata-download + dest=/usr/local/share/geodata/download.py + owner=root group=root + mode=0755 + +- name: Create /usr/local/bin/geodata-download + file: src=../share/geodata/download.py + dest=/usr/local/bin/geodata-download + owner=root group=root + state=link force=yes + +- name: Create directory /var/cache/geodata + file: path=/var/cache/geodata + state=directory + owner=_geodata-download group=root + mode=0755 + +- name: Create directory /var/cache/geodata/custom + file: path=/var/cache/geodata/custom + state=directory + owner=root group=root + mode=0755 + +- name: Copy custom layers into /var/cache/geodata/custom + copy: src=webmap-tools/layers/custom/ + dest=/var/cache/geodata/custom/ + owner=root group=root + mode=0644 + directory_mode=0755 + +- name: Copy geodata-download@.service + copy: src=etc/systemd/system/geodata-download@.service + dest=/etc/systemd/system/geodata-download@.service + owner=root group=root + mode=0644 + notify: + - systemctl daemon-reload + +- name: Enable geodata-download@.service + service: name=geodata-download@{{ item }}.service enabled=true + with_items: "{{ geodata_layer_groups | union(geodata_raster) | difference(geodata_layer_groups_nodownload) }}" + +- name: Disable some geodata-download@.service + service: name=geodata-download@{{ item }}.service enabled=false + with_items: "{{ geodata_layer_groups_nodownload }}" + +- meta: flush_handlers + + +- name: Copy /etc/tmpfiles.d/geodata.conf + copy: src=etc/tmpfiles.d/geodata.conf + dest=/etc/tmpfiles.d/geodata.conf + owner=root group=root + mode=0644 + notify: + - systemd-tmpfiles --create + +- meta: flush_handlers + + +- name: Create system user '_geodata' + user: name=_geodata system=true + group=_geodata + createhome=false + home=/nonexistent + shell=/usr/sbin/nologin + comment="geodata update (extract/import)" + password="!" + state=present + +- name: Copy /usr/local/share/geodata/import.py + copy: src=webmap-tools/geodata-import + dest=/usr/local/share/geodata/import.py + owner=root group=root + mode=0755 + +- name: Create /usr/local/bin/geodata-import + file: src=../share/geodata/import.py + dest=/usr/local/bin/geodata-import + owner=root group=root + state=link force=yes + +- name: Copy geodata-import@.service + copy: src=etc/systemd/system/geodata-import@.service + dest=/etc/systemd/system/geodata-import@.service + owner=root group=root + mode=0644 + notify: + - systemctl daemon-reload + +- name: Enable geodata-import@.service + service: name=geodata-import@{{ item }}.service enabled=true + with_items: "{{ geodata_layer_groups }}" + +- name: Copy geodata-raster@.service + copy: src=etc/systemd/system/geodata-raster@.service + dest=/etc/systemd/system/geodata-raster@.service + owner=root group=root + mode=0644 + notify: + - systemctl daemon-reload + +- name: Enable geodata-raster@.service + service: name=geodata-raster@{{ item }}.service enabled=true + with_items: "{{ geodata_raster }}" diff --git a/tasks/httpd.yml b/tasks/httpd.yml index 22757d5..2e1fa98 100644 --- a/tasks/httpd.yml +++ b/tasks/httpd.yml @@ -2,6 +2,12 @@ - name: Install nginx apt: pkg=nginx-light +- name: Install nginx modules + apt: pkg={{ packages }} + vars: + packages: + - libnginx-mod-http-brotli-static + - name: Start nginx service: name=nginx enabled=true state=started diff --git a/tasks/mail.yml b/tasks/mail.yml index 8f58c8a..9c5ef5e 100644 --- a/tasks/mail.yml +++ b/tasks/mail.yml @@ -6,10 +6,10 @@ regexp='^{{ item.src }}{{':'}} ' line='{{ item.src }}{{':'}} {{ item.dst }}' with_items: - - { src: mailer-daemon, dst: 'postmaster' } - - { src: postmaster, dst: 'root' } - - { src: nobody, dst: 'root' } - - { src: root, dst: 'hostmaster@{{ ansible_domain }}' } + - { src: mailer-daemon, dst: 'postmaster' } + - { src: postmaster, dst: 'root' } + - { src: nobody, dst: 'root' } + - { src: root, dst: 'hostmaster@{{ ansible_facts.domain }}' } notify: - Run newaliases diff --git a/tasks/network.yml b/tasks/network.yml index 1551f82..5c1af1a 100644 --- a/tasks/network.yml +++ b/tasks/network.yml @@ -18,6 +18,7 @@ - ifupdown - isc-dhcp-client - isc-dhcp-common + - dhcpcd-base - name: Remove /etc/network/interfaces and /etc/network/interfaces.d file: path={{ item }} state=absent @@ -32,6 +33,7 @@ packages: - systemd-resolved - libnss-resolve + - libnss-myhostname - name: Create directory /etc/systemd/resolved.conf.d file: path=/etc/systemd/resolved.conf.d diff --git a/tasks/postgis.yml b/tasks/postgis.yml new file mode 100644 index 0000000..3e156a9 --- /dev/null +++ b/tasks/postgis.yml @@ -0,0 +1,158 @@ +- name: Install PostgreSQL and PostGIS + apt: pkg={{ packages }} + vars: + packages: + - postgresql + - postgresql-postgis + - postgis + # for ansible + - python3-psycopg + +- name: Generate sv_SE.UTF-8 locales + locale_gen: name=sv_SE.UTF-8 state=present + # PostgreSQL needs to be restarted to see the new locale + notify: Restart PostgreSQL + +- name: Configure PostgreSQL + copy: src=etc/postgresql/postgresql.conf + dest=/etc/postgresql/{{ postgresql.version }}/{{ postgresql.cluster }}/conf.d/local.conf + owner=postgres group=postgres + mode=0644 + notify: Restart PostgreSQL + +- name: Start PostgreSQL + service: name=postgresql@{{ postgresql.version }}-{{ postgresql.cluster }}.service state=started + +- meta: flush_handlers + +# Usage: \sudo -u postgres psql </usr/local/share/geodata/schema.sql +- name: Copy /usr/local/share/geodata/schema.sql + copy: src=webmap-tools/schema.sql + dest=/usr/local/share/geodata/schema.sql + owner=root group=root + mode=0644 + +- name: Create PostgreSQL database + become: true + # XXX: this creates /var/lib/postgresql/.ansible/tmp + become_user: postgres + community.postgresql.postgresql_db: + name: geodata + comment: Backend PostGIS database for KlimatanalysNorr tooling + encoding: UTF-8 + lc_collate: sv_SE.UTF-8 + lc_ctype: sv_SE.UTF-8 + locale_provider: icu + icu_locale: sv-SE-x-icu + template: template0 + owner: postgres + +- name: Create 'geodata' and 'guest' PostgreSQL users (roles) + become: true + become_user: postgres + community.postgresql.postgresql_user: + login_db: geodata + name: "{{ item }}" + with_items: + - geodata + - guest + +- name: Add a rule for 'geodata' user in pg_hba.conf + ansible.builtin.lineinfile: + path: /etc/postgresql/{{ postgresql.version }}/{{ postgresql.cluster }}/pg_hba.conf + regexp: '^local\s+geodata\s' + line: 'local geodata all peer map=pgmap_geodata' + # must come before 'local all all peer', cf. + # https://dba.stackexchange.com/questions/177142/postgresql-cannot-peer-authenticate-using-usermap-provided-user-name-dbuser + insertbefore: '^local\s+all\s+all\s' + create: false + notify: Reload PostgreSQL + +- name: Add a mapping rule for 'geodata' user in pg_ident.conf + ansible.builtin.lineinfile: + path: /etc/postgresql/{{ postgresql.version }}/{{ postgresql.cluster }}/pg_ident.conf + regexp: '^pgmap_geodata\s.*\sgeodata\s*$' + line: 'pgmap_geodata _geodata geodata' + create: false + notify: Reload PostgreSQL + +- name: Add a mapping rule for 'guest' user in pg_ident.conf + ansible.builtin.lineinfile: + path: /etc/postgresql/{{ postgresql.version }}/{{ postgresql.cluster }}/pg_ident.conf + regexp: '^pgmap_geodata\s.*\sguest\s*$' + line: 'pgmap_geodata /^_?[a-zA-Z][a-zA-Z0-9_\-]*[a-zA-Z0-9]$ guest' + create: false + notify: Reload PostgreSQL + +- name: Create PostgreSQL schemas + become: true + become_user: postgres + community.postgresql.postgresql_schema: + login_db: geodata + name: "{{ item.name }}" + owner: postgres + comment: "{{ item.comment }}" + with_items: "{{ postgis_schemas }}" + +- name: Install 'postgis' PostgreSQL extension to the geodata database + become: true + become_user: postgres + community.postgresql.postgresql_ext: + name: postgis + login_db: geodata + comment: Geographic objects support for PostgreSQL + +- name: GRANT CONNECT ON DATABASE geodata TO geodata, guest + become: true + become_user: postgres + community.postgresql.postgresql_privs: + login_db: geodata + privs: CONNECT + type: database + role: geodata,guest + +- name: GRANT USAGE ON SCHEMA * TO geodata, guest + become: true + become_user: postgres + community.postgresql.postgresql_privs: + login_db: geodata + privs: USAGE + type: schema + objs: "{{ (['public'] + (postgis_schemas | map(attribute='name'))) | join(',') }}" + role: geodata,guest + +# tooling should TRUNCATE existing output layers instead +- name: REVOKE CREATE ON SCHEMA * FROM geodata + become: true + become_user: postgres + community.postgresql.postgresql_privs: + login_db: geodata + privs: CREATE + type: schema + objs: "{{ (['public'] + (postgis_schemas | map(attribute='name'))) | join(',') }}" + role: geodata + state: absent + +- name: GRANT SELECT ON TABLES IN SCHEMA * TO guest + become: true + become_user: postgres + community.postgresql.postgresql_privs: + login_db: geodata + privs: SELECT + type: table + obj: ALL_IN_SCHEMA + schema: "{{ item }}" + role: guest + with_items: "{{ ['public'] + (postgis_schemas | map(attribute='name')) }}" + +- name: GRANT USAGE, SELECT ON SEQUENCES IN SCHEMA * TO guest + become: true + become_user: postgres + community.postgresql.postgresql_privs: + login_db: geodata + privs: USAGE,SELECT + type: sequence + obj: ALL_IN_SCHEMA + schema: "{{ item }}" + role: guest + with_items: "{{ ['public'] + (postgis_schemas | map(attribute='name')) }}" diff --git a/tasks/ssh.yml b/tasks/ssh.yml index 341a96d..e568036 100644 --- a/tasks/ssh.yml +++ b/tasks/ssh.yml @@ -16,5 +16,5 @@ notify: - Restart OpenSSH -- name: Start Openssh +- name: Start OpenSSH service: name=ssh enabled=true state=started diff --git a/tasks/webmap.yml b/tasks/webmap.yml index 9b60359..682e785 100644 --- a/tasks/webmap.yml +++ b/tasks/webmap.yml @@ -1,113 +1,83 @@ -- name: Install gdal-bin - apt: pkg=gdal-bin install-recommends=true - -- name: Install unzip - apt: pkg=unzip - -- name: Install python dependencies - apt: pkg={{ packages }} - vars: - packages: - - python3 - - python3-gdal - - python3-lxml - - python3-requests - - python3-tqdm - - python3-urllib3 - - python3-xdg - - python3-yaml - -- name: Create directory /etc/webmap - file: path=/etc/webmap +- name: Install brotli + apt: pkg=brotli + +- name: Build administrative-codes.json* + become: false + delegate_to: localhost + community.general.make: + chdir: ./webmap-tools/administrative-codes + target: all + +- name: Create directory /var/www/webmap/data + file: path=/var/www/webmap/data state=directory owner=root group=root mode=0755 -- name: Copy /etc/webmap/config.yml - copy: src=webmap-tools/config.yml - dest=/etc/webmap/config.yml +- name: Copy /var/www/webmap/data/administrative-codes.json* + copy: src=./webmap-tools/administrative-codes/{{ item }} + dest=/var/www/webmap/data/{{ item }} owner=root group=root mode=0644 + with_items: + - administrative-codes.json + - administrative-codes.json.br -- name: Create directory /usr/local/share/webmap - file: path=/usr/local/share/webmap +- meta: flush_handlers + + +- name: Create directory /var/www/webmap/tiles + file: path=/var/www/webmap/tiles state=directory - owner=root group=root + owner=_geodata group=root mode=0755 -- name: Copy /usr/local/share/webmap/common.py - copy: src=webmap-tools/common.py - dest=/usr/local/share/webmap/common.py - owner=root group=root - mode=0644 +- name: Create directory /var/www/webmap/raster + file: path=/var/www/webmap/raster + state=directory + owner=_geodata group=root + mode=0755 + +- meta: flush_handlers + -- name: Copy webmap-update@.target - copy: src=etc/systemd/system/webmap-update@.target - dest=/etc/systemd/system/webmap-update@.target +- name: Install Python/WSGI dependencies + apt: pkg={{ packages }} + vars: + packages: + - uwsgi-core + - uwsgi-plugin-python3 + - python3-psycopg-c + +- name: Copy webmap-cgi.socket + copy: src=etc/systemd/system/webmap-cgi.socket + dest=/etc/systemd/system/webmap-cgi.socket owner=root group=root mode=0644 notify: - systemctl daemon-reload -- name: Copy webmap-update@.timer - copy: src=etc/systemd/system/webmap-update@.timer - dest=/etc/systemd/system/webmap-update@.timer - owner=root group=root - mode=0644 +- name: Copy webmap-cgi.service + template: src=etc/systemd/system/webmap-cgi.service + dest=/etc/systemd/system/webmap-cgi.service + owner=root group=root + mode=0644 notify: - systemctl daemon-reload + - Stop webmap-cgi.service -- name: Enable webmap-update.timer - service: name=webmap-update@{{ item }}.timer state=started enabled=true - with_items: "{{ webmap_layer_groups }}" - -- meta: flush_handlers - - -- name: Create system user '_webmap-download' - user: name=_webmap-download system=true - group=nogroup - createhome=false - home=/nonexistent - shell=/usr/sbin/nologin - comment="Webmap update (download)" - password="!" - state=present - -- name: Copy /usr/local/share/webmap/download.py - copy: src=webmap-tools/webmap-download - dest=/usr/local/share/webmap/download.py +- name: Copy /usr/local/libexec/webmap-cgi + copy: src=./webmap-tools/webmap-cgi + dest=/usr/local/libexec/webmap-cgi owner=root group=root mode=0755 - -- name: Create /usr/local/bin/webmap-download - file: src=../share/webmap/download.py - dest=/usr/local/bin/webmap-download - owner=root group=root - state=link force=yes - -- name: Copy /usr/local/share/webmap/webmap-download-mrr.py - copy: src=webmap-tools/webmap-download-mrr.py - dest=/usr/local/share/webmap/webmap-download-mrr.py - owner=root group=root - mode=0644 - -- name: Create directory /var/cache/webmap - file: path=/var/cache/webmap - state=directory - owner=_webmap-download group=nogroup - mode=0755 - -- name: Copy webmap-download@.service - copy: src=etc/systemd/system/webmap-download@.service - dest=/etc/systemd/system/webmap-download@.service - owner=root group=root - mode=0644 notify: - - systemctl daemon-reload - -- name: Enable webmap-download@.service - service: name=webmap-download@{{ item }}.service enabled=true - with_items: "{{ webmap_layer_groups }}" + - Stop webmap-cgi.service - meta: flush_handlers + +- name: Enable webmap-cgi.socket + service: name=webmap-cgi.socket state=started enabled=true + +- name: Disable webmap-cgi.service + service: name=webmap-cgi.service enabled=false diff --git a/templates/etc/apt/sources.list.d/debian.sources.j2 b/templates/etc/apt/sources.list.d/debian.sources.j2 index 980daaf..c859a4e 100644 --- a/templates/etc/apt/sources.list.d/debian.sources.j2 +++ b/templates/etc/apt/sources.list.d/debian.sources.j2 @@ -1,9 +1,11 @@ Types: deb URIs: https://deb.debian.org/debian -Suites: {{ ansible_lsb.codename }} {{ ansible_lsb.codename }}-updates +Suites: {{ ansible_facts.lsb.codename }} {{ ansible_facts.lsb.codename }}-updates Components: main non-free-firmware +Signed-By: /usr/share/keyrings/debian-archive-keyring.pgp Types: deb URIs: https://deb.debian.org/debian-security -Suites: {{ ansible_lsb.codename }}-security +Suites: {{ ansible_facts.lsb.codename }}-security Components: main non-free-firmware +Signed-By: /usr/share/keyrings/debian-archive-keyring.pgp diff --git a/templates/etc/postfix/main.cf.j2 b/templates/etc/postfix/main.cf.j2 index 35a6790..10313b4 100644 --- a/templates/etc/postfix/main.cf.j2 +++ b/templates/etc/postfix/main.cf.j2 @@ -9,8 +9,8 @@ compatibility_level = 3.6 smtputf8_enable = no myorigin = /etc/mailname -myhostname = {{ ansible_fqdn }} -mydomain = {{ ansible_domain }} +myhostname = {{ ansible_facts.fqdn }} +mydomain = {{ ansible_facts.domain }} append_dot_mydomain = no # This server is for internal use only diff --git a/templates/etc/systemd/network/01-wired.network.j2 b/templates/etc/systemd/network/01-wired.network.j2 index 7be5d21..dc85b2e 100644 --- a/templates/etc/systemd/network/01-wired.network.j2 +++ b/templates/etc/systemd/network/01-wired.network.j2 @@ -1,13 +1,13 @@ [Match] -Name={{ ansible_default_ipv4.interface }} +Name={{ ansible_facts.default_ipv4.interface }} [Network] DHCP=yes -{% if ansible_default_ipv6.get('scope', '') == 'global' %} +{% if ansible_facts.default_ipv6.get('scope', '') == 'global' %} [Address] -Address={{ ansible_default_ipv6.address }}/{{ ansible_default_ipv6.prefix }} +Address={{ ansible_facts.default_ipv6.address }}/{{ ansible_facts.default_ipv6.prefix }} [Route] -Gateway={{ ansible_default_ipv6.gateway }} +Gateway={{ ansible_facts.default_ipv6.gateway }} {%- endif %} diff --git a/templates/etc/systemd/system/geodata-update@.timer.d/override.conf.j2 b/templates/etc/systemd/system/geodata-update@.timer.d/override.conf.j2 new file mode 100644 index 0000000..103fbde --- /dev/null +++ b/templates/etc/systemd/system/geodata-update@.timer.d/override.conf.j2 @@ -0,0 +1,3 @@ +[Timer] +OnCalendar= +OnCalendar={{ geodata_layer_groups_update_calendar[item] }} diff --git a/templates/etc/systemd/system/webmap-cgi.service b/templates/etc/systemd/system/webmap-cgi.service new file mode 100644 index 0000000..9c9ffe9 --- /dev/null +++ b/templates/etc/systemd/system/webmap-cgi.service @@ -0,0 +1,37 @@ +[Unit] +Description=Webmap CGI (Common Gateway Interface) +After=syslog.target network.target postgresql.service +StopPropagatedFrom=postgresql.service postgresql@{{ postgresql.version }}-{{ postgresql.cluster }}.service + +[Service] +DynamicUser=yes +User=_webmap-cgi +# Note: the "WARNING: you have enabled harakiri without post buffering" can +# be ignored because body requests are in fact buffered on the nginx side +ExecStart=/usr/bin/uwsgi -M -p2 \ + --single-interpreter --die-on-term \ + --close-on-exec --close-on-exec2 \ + --max-requests 1000 \ + --max-worker-lifetime 86400 \ + --max-worker-lifetime-delta 11 \ + --harakiri 60 \ + --lazy-apps \ + --plugins python3 \ + --pythonpath /usr/local/share/geodata \ + --wsgi-file /usr/local/libexec/webmap-cgi +Nice=10 +RestartSec=15s +Restart=always + +# Hardening +NoNewPrivileges=yes +ProtectHome=yes +ProtectSystem=strict +PrivateDevices=yes +ProtectControlGroups=yes +ProtectKernelModules=yes +ProtectKernelTunables=yes +RestrictAddressFamilies=AF_UNIX + +[Install] +WantedBy=multi-user.target diff --git a/webmap-tools b/webmap-tools -Subproject 729a5df4ba9889aebcd51787ec11a4d0d1ea547 +Subproject baa8667a38a5cc79571da15844d525c665234ed |