diff options
-rw-r--r-- | ansible.cfg | 1 | ||||
-rw-r--r-- | files/etc/nginx/sites-available/webmap | 57 | ||||
-rw-r--r-- | files/etc/postgresql/postgresql.conf | 4 | ||||
-rw-r--r-- | files/etc/systemd/system/webmap-cgi.service | 36 | ||||
-rw-r--r-- | files/etc/systemd/system/webmap-cgi.socket | 11 | ||||
-rw-r--r-- | files/etc/systemd/system/webmap-download@.service | 10 | ||||
-rw-r--r-- | files/etc/systemd/system/webmap-import@.service | 16 | ||||
-rw-r--r-- | files/etc/systemd/system/webmap-publish@.service | 40 | ||||
-rw-r--r-- | files/etc/systemd/system/webmap-update@.target | 2 | ||||
-rw-r--r-- | files/etc/tmpfiles.d/webmap.conf | 15 | ||||
-rw-r--r-- | group_vars/all.yml | 8 | ||||
-rw-r--r-- | handlers/main.yml | 3 | ||||
-rw-r--r-- | tasks/apt.yml | 9 | ||||
-rw-r--r-- | tasks/network.yml | 1 | ||||
-rw-r--r-- | tasks/webmap.yml | 157 | ||||
m--------- | webmap-tools | 0 |
16 files changed, 240 insertions, 130 deletions
diff --git a/ansible.cfg b/ansible.cfg index a35402e..cf64db8 100644 --- a/ansible.cfg +++ b/ansible.cfg @@ -1,5 +1,6 @@ [defaults] inventory = ./hosts.ini +max_diff_size = 524288 [privilege_escalation] become = True diff --git a/files/etc/nginx/sites-available/webmap b/files/etc/nginx/sites-available/webmap index 92809e2..efb062a 100644 --- a/files/etc/nginx/sites-available/webmap +++ b/files/etc/nginx/sites-available/webmap @@ -45,36 +45,75 @@ server { ssl_certificate_key /etc/nginx/ssl/webmap.rsa.key; include snippets/ssl.conf; + root /var/www/webmap; + index index.html; + add_header Referrer-Policy "no-referrer"; - add_header X-Frame-Options "SAMEORIGIN"; + add_header X-Frame-Options "DENY"; add_header X-Content-Type-Options "nosniff"; add_header X-XSS-Protection "1; mode=block"; add_header Strict-Transport-Security "max-age=31557600; includeSubDomains" always; - add_header Content-Security-Policy "default-src 'none'; connect-src 'self'; font-src 'self'; frame-src 'self'; img-src 'self' data: https://minkarta.lantmateriet.se/map/; script-src 'self'; style-src 'self'; frame-ancestors 'self'; form-action 'none'; base-uri 'self'"; - - root /var/www/webmap; - index index.html; + add_header Content-Security-Policy "default-src 'none'; frame-ancestors 'none'; form-action 'none'; base-uri 'self'"; + #add_header Access-Control-Allow-Origin $http_origin always; + + include mime.types; + types { + # application/protobuf and application/vnd.google.protobuf might be valid types too, cf. + # https://stackoverflow.com/questions/30505408/what-is-the-correct-protobuf-content-type + application/x-protobuf pbf; + } location ^~ /assets/ { expires 7d; - gzip_static on; + brotli_static on; try_files $uri =404; } location ^~ /tiles/ { - expires 1d; + expires 30m; brotli_static on; try_files $uri =404; + # service an empty payload to save bandwidth error_page 404 /_.txt; } + location = /q { + expires epoch; + limit_except POST { deny all; } + #if ($request_method = OPTIONS) { + # add_header Strict-Transport-Security "max-age=31557600; includeSubDomains"; + # add_header Access-Control-Allow-Origin $http_origin; + # add_header Access-Control-Allow-Methods "POST, GET, OPTIONS"; + # add_header Access-Control-Allow-Headers "Accept, Content-Type"; + # add_header Access-Control-Max-Age 28800; + # return 204; + #} + client_max_body_size 64k; + gzip on; + gzip_types application/json text/plain; + include uwsgi_params; + uwsgi_pass unix:/run/webmap-cgi.socket; + } + location = /tiles/metadata.json { + expires epoch; + brotli_static on; + try_files $uri =404; + } location = /_.txt { - # cache 404 responses + # cache 404 responses for 30m like for valid tiles add_header Strict-Transport-Security "max-age=31557600; includeSubDomains" always; - add_header Cache-Control "public; max-age=86400" always; + add_header Cache-Control "public; max-age=1800" always; + #add_header Access-Control-Allow-Origin $http_origin always; internal; } location / { + add_header Referrer-Policy "no-referrer"; + add_header X-Frame-Options "SAMEORIGIN"; + add_header X-Content-Type-Options "nosniff"; + add_header X-XSS-Protection "1; mode=block"; + add_header Strict-Transport-Security "max-age=31557600; includeSubDomains" always; + add_header Content-Security-Policy "default-src 'none'; connect-src 'self'; font-src 'self'; frame-src 'self'; img-src 'self' data: https://minkarta.lantmateriet.se/map/; script-src 'self'; style-src 'self'; frame-ancestors 'self'; form-action 'none'; base-uri 'self'"; + try_files $uri $uri/ =404; } } diff --git a/files/etc/postgresql/postgresql.conf b/files/etc/postgresql/postgresql.conf new file mode 100644 index 0000000..038438a --- /dev/null +++ b/files/etc/postgresql/postgresql.conf @@ -0,0 +1,4 @@ +shared_buffers = 768MB +temp_buffers = 128MB +work_mem = 16MB +effective_cache_size = 1536MB diff --git a/files/etc/systemd/system/webmap-cgi.service b/files/etc/systemd/system/webmap-cgi.service new file mode 100644 index 0000000..88f22e5 --- /dev/null +++ b/files/etc/systemd/system/webmap-cgi.service @@ -0,0 +1,36 @@ +[Unit] +Description=Webmap CGI (Common Gateway Interface) +After=syslog.target network.target postgresql.service + +[Service] +DynamicUser=yes +User=_webmap-cgi +# Note: the "WARNING: you have enabled harakiri without post buffering" can +# be ignored because body requests are in fact buffered on the nginx side +ExecStart=/usr/bin/uwsgi -M -p2 \ + --single-interpreter --die-on-term \ + --close-on-exec --close-on-exec2 \ + --max-requests 1000 \ + --max-worker-lifetime 86400 \ + --max-worker-lifetime-delta 11 \ + --harakiri 60 \ + --lazy-apps \ + --plugins python3 \ + --pythonpath /usr/local/share/webmap \ + --wsgi-file /usr/libexec/webmap-cgi +Nice=10 +RestartSec=15s +Restart=always + +# Hardening +NoNewPrivileges=yes +ProtectHome=yes +ProtectSystem=strict +PrivateDevices=yes +ProtectControlGroups=yes +ProtectKernelModules=yes +ProtectKernelTunables=yes +RestrictAddressFamilies=AF_UNIX + +[Install] +WantedBy=multi-user.target diff --git a/files/etc/systemd/system/webmap-cgi.socket b/files/etc/systemd/system/webmap-cgi.socket new file mode 100644 index 0000000..2828985 --- /dev/null +++ b/files/etc/systemd/system/webmap-cgi.socket @@ -0,0 +1,11 @@ +[Unit] +Description=Webmap CGI (Common Gateway Interface) activation socket +After=syslog.target network.target + +[Socket] +ListenStream=%t/webmap-cgi.socket +SocketUser=www-data +SocketMode=0666 + +[Install] +WantedBy=sockets.target diff --git a/files/etc/systemd/system/webmap-download@.service b/files/etc/systemd/system/webmap-download@.service index 2c5a3e4..d7a49dc 100644 --- a/files/etc/systemd/system/webmap-download@.service +++ b/files/etc/systemd/system/webmap-download@.service @@ -1,5 +1,5 @@ [Unit] -Description=Webmap updater service (download %I) +Description=Webmap updater service (download ‘%I’) # Chaining logic from https://serverfault.com/questions/1079993/why-does-my-systemd-timer-only-trigger-once-when-the-unit-is-a-target#answer-1128671 # XXX Looks like Upholds= prevents running a single unit, as it causes # webmap-update@%i.target to start upon `systemctl start webmap-download@foo.service` @@ -15,8 +15,8 @@ IOSchedulingClass=idle Type=oneshot ExecStart=/usr/local/bin/webmap-download \ - --cachedir=/var/cache/webmap \ - --lockdir=%t/lock/webmap/download \ + --cachedir=%C/webmap \ + --lockdir=%t/lock/webmap/cache \ --no-exit-code \ --quiet \ -- %I @@ -30,8 +30,8 @@ ProtectControlGroups=yes ProtectKernelModules=yes ProtectKernelTunables=yes RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 -ReadWritePaths=/var/cache/webmap -ReadWritePaths=%t/lock/webmap/download +ReadWritePaths=%C/webmap +ReadWritePaths=%t/lock/webmap/cache [Install] WantedBy=webmap-update@%i.target diff --git a/files/etc/systemd/system/webmap-import@.service b/files/etc/systemd/system/webmap-import@.service index 06d204c..964c37d 100644 --- a/files/etc/systemd/system/webmap-import@.service +++ b/files/etc/systemd/system/webmap-import@.service @@ -1,17 +1,11 @@ [Unit] -Description=Webmap updater service (import %I to PostgreSQL) +Description=Webmap updater service (import ‘%I’ to PostGIS) After=postgresql.service webmap-update@%i.target After=webmap-download@%i.service Upholds=webmap-update@%i.target -# XXX webmap-download write cached files atomatically but there is no -# guarantee that GDAL/OGR opens them atomically. It'd therefore make -# sense to use the following Conflict= directive, however systemd skips -# webmap-download@%i.service in that case. -#Conflicts=webmap-download@%i.service - [Service] -User=_webmap-import +User=_webmap Group=_webmap Nice=15 @@ -19,8 +13,11 @@ IOSchedulingClass=idle Type=oneshot ExecStart=/usr/local/bin/webmap-import \ - --cachedir=/var/cache/webmap \ + --cachedir=%C/webmap \ --lockfile=%t/lock/webmap/lock \ + --lockdir-sources=%t/lock/webmap/cache \ + --mvtdir=/var/www/webmap/tiles/%I \ + --mvt-compress \ -- %I # Hardening @@ -33,6 +30,7 @@ ProtectKernelModules=yes ProtectKernelTunables=yes RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 ReadWritePaths=%t/lock/webmap +ReadWritePaths=/var/www/webmap/tiles PrivateTmp=yes [Install] diff --git a/files/etc/systemd/system/webmap-publish@.service b/files/etc/systemd/system/webmap-publish@.service deleted file mode 100644 index e2f8e6b..0000000 --- a/files/etc/systemd/system/webmap-publish@.service +++ /dev/null @@ -1,40 +0,0 @@ -[Unit] -Description=Webmap updater service (publish %I as MVT) -#After=postgresql.service webmap-update@%i.target -#After=webmap-download@%i.service -#After=webmap-import@%i.service -#Upholds=webmap-update@%i.target - -[Service] -User=_webmap-publish -Group=_webmap - -Nice=15 -IOSchedulingClass=idle - -Type=oneshot -ExecStart=/usr/local/bin/webmap-publish \ - --lockfile=%t/lock/webmap/lock \ - --destdir=/var/www/webmap/tiles/%i \ - --name=%I \ - --webroot=/var/www/webmap \ - --metadata=/var/www/webmap/tiles/metadata.json \ - --metadata-lockfile=%t/lock/webmap/tiles.lock \ - --compress \ - -- %I - -# Hardening -NoNewPrivileges=yes -ProtectHome=yes -ProtectSystem=strict -PrivateDevices=yes -ProtectControlGroups=yes -ProtectKernelModules=yes -ProtectKernelTunables=yes -RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 -ReadWritePaths=/var/www/webmap/tiles -ReadWritePaths=%t/lock/webmap -PrivateTmp=yes - -#[Install] -#WantedBy=webmap-update@%i.target diff --git a/files/etc/systemd/system/webmap-update@.target b/files/etc/systemd/system/webmap-update@.target index 3d9fb7f..840de96 100644 --- a/files/etc/systemd/system/webmap-update@.target +++ b/files/etc/systemd/system/webmap-update@.target @@ -1,3 +1,3 @@ [Unit] -Description=Webmap updater (target unit %I) +Description=Webmap updater (target unit ‘%I’) StopWhenUnneeded=true diff --git a/files/etc/tmpfiles.d/webmap.conf b/files/etc/tmpfiles.d/webmap.conf index 620cd24..786e6dd 100644 --- a/files/etc/tmpfiles.d/webmap.conf +++ b/files/etc/tmpfiles.d/webmap.conf @@ -1,11 +1,8 @@ -d %t/lock/webmap 0755 root root +d %t/lock/webmap 00755 root root -# for webmap-download's --lockdir -d %t/lock/webmap/download 0755 _webmap-download _webmap +# for `webmap-download --lockdir` *and* `webmap-import --lockdir-sources` +# (hence the set-group-ID bit and g+w) +d %t/lock/webmap/cache 02775 _webmap-download _webmap -# for webmap-import's *and* webmap-publish's --lockfile (hence the -# ownership and g+w) -f %t/lock/webmap/lock 0664 root _webmap - -# for webmap-publish's --metadata-lockfile -f %t/lock/webmap/tiles.lock 0644 _webmap-publish _webmap +# for `webmap-import --lockfile` +f %t/lock/webmap/lock 00644 _webmap _webmap diff --git a/group_vars/all.yml b/group_vars/all.yml index 5d4474a..e90c73e 100644 --- a/group_vars/all.yml +++ b/group_vars/all.yml @@ -5,8 +5,14 @@ webmap_layer_groups: - mrr - nvr - sks - - st + - ren - vbk + - ri + - svk + - misc + +webmap_layer_groups_nodownload: + - misc # PostgreSQL's version number and cluster name postgresql: diff --git a/handlers/main.yml b/handlers/main.yml index 281951b..b6ee548 100644 --- a/handlers/main.yml +++ b/handlers/main.yml @@ -55,3 +55,6 @@ - name: Reload PostgreSQL service: name=postgresql.service state=reloaded + +- name: Stop webmap-cgi.service + service: name=webmap-cgi.service state=stopped diff --git a/tasks/apt.yml b/tasks/apt.yml index 1023908..f17a2e4 100644 --- a/tasks/apt.yml +++ b/tasks/apt.yml @@ -5,8 +5,13 @@ - apt - lsb-release -- name: Remove /etc/apt/sources.list - file: path=/etc/apt/sources.list state=absent +# something keeps recreating (without content) it if we delete it, so we +# leave it instead but ensure it's empty instead +- name: Create empty /etc/apt/sources.list + copy: content="" + dest=/etc/apt/sources.list + owner=root group=root + mode=0644 notify: - apt-get update diff --git a/tasks/network.yml b/tasks/network.yml index 1551f82..a02b07c 100644 --- a/tasks/network.yml +++ b/tasks/network.yml @@ -32,6 +32,7 @@ packages: - systemd-resolved - libnss-resolve + - libnss-myhostname - name: Create directory /etc/systemd/resolved.conf.d file: path=/etc/systemd/resolved.conf.d diff --git a/tasks/webmap.yml b/tasks/webmap.yml index 9e2c2f6..d694387 100644 --- a/tasks/webmap.yml +++ b/tasks/webmap.yml @@ -15,7 +15,6 @@ - python3 - python3-brotli - python3-gdal - - python3-lxml - python3-requests - python3-systemd - python3-tqdm @@ -41,11 +40,18 @@ owner=root group=root mode=0755 -- name: Copy /usr/local/share/webmap/common.py - copy: src=webmap-tools/common.py - dest=/usr/local/share/webmap/common.py +- name: Copy /usr/local/share/webmap/*.py modules + copy: src=webmap-tools/{{ item }} + dest=/usr/local/share/webmap/{{ item }} owner=root group=root mode=0644 + with_items: + # TODO these should be compiled + - common.py + - common_gdal.py + - import_source.py + - export_mvt.py + - rename_exchange.py - name: Copy webmap-update@.target copy: src=etc/systemd/system/webmap-update@.target @@ -96,18 +102,25 @@ owner=root group=root state=link force=yes -- name: Copy /usr/local/share/webmap/webmap-download-mrr.py - copy: src=webmap-tools/webmap-download-mrr.py - dest=/usr/local/share/webmap/webmap-download-mrr.py - owner=root group=root - mode=0644 - - name: Create directory /var/cache/webmap file: path=/var/cache/webmap state=directory owner=_webmap-download group=root mode=0755 +- name: Create directory /var/cache/webmap/custom + file: path=/var/cache/webmap/custom + state=directory + owner=root group=root + mode=0755 + +- name: Copy custom layers into /var/cache/webmap/custom + copy: src=webmap-tools/layers/custom/ + dest=/var/cache/webmap/custom/ + owner=root group=root + mode=0644 + directory_mode=0755 + - name: Copy webmap-download@.service copy: src=etc/systemd/system/webmap-download@.service dest=/etc/systemd/system/webmap-download@.service @@ -118,13 +131,17 @@ - name: Enable webmap-download@.service service: name=webmap-download@{{ item }}.service enabled=true - with_items: "{{ webmap_layer_groups }}" + with_items: "{{ webmap_layer_groups | difference(webmap_layer_groups_nodownload) }}" + +- name: Disable some webmap-download@.service + service: name=webmap-download@{{ item }}.service enabled=false + with_items: "{{ webmap_layer_groups_nodownload }}" - meta: flush_handlers -- name: Create system user '_webmap-import' - user: name=_webmap-import system=true +- name: Create system user '_webmap' + user: name=_webmap system=true group=_webmap createhome=false home=/nonexistent @@ -148,6 +165,13 @@ # PostgreSQL needs to be restarted to see the new locale notify: Restart PostgreSQL +- name: Configure PostgreSQL + copy: src=etc/postgresql/postgresql.conf + dest=/etc/postgresql/{{ postgresql.version }}/{{ postgresql.cluster }}/conf.d/local.conf + owner=postgres group=postgres + mode=0644 + notify: Restart PostgreSQL + - name: Start PostgreSQL service: name=postgresql@{{ postgresql.version }}-{{ postgresql.cluster }}.service state=started @@ -179,7 +203,7 @@ become: true become_user: postgres community.postgresql.postgresql_user: - db: webmap + login_db: webmap name: "{{ item }}" with_items: - webmap_import @@ -200,7 +224,7 @@ ansible.builtin.lineinfile: path: /etc/postgresql/{{ postgresql.version }}/{{ postgresql.cluster }}/pg_ident.conf regexp: '^pgmap_webmap\s.*\swebmap_import\s*$' - line: 'pgmap_webmap _webmap-import webmap_import' + line: 'pgmap_webmap _webmap webmap_import' create: false notify: Reload PostgreSQL @@ -217,7 +241,7 @@ become_user: postgres community.postgresql.postgresql_schema: name: postgis - db: webmap + login_db: webmap owner: postgres - name: Install 'postgis' PostgreSQL extension to the webmap database in the postgis schema @@ -225,7 +249,7 @@ become_user: postgres community.postgresql.postgresql_ext: name: postgis - db: webmap + login_db: webmap schema: postgis comment: Geographic objects support for PostgreSQL @@ -233,7 +257,7 @@ become: true become_user: postgres community.postgresql.postgresql_privs: - database: webmap + login_db: webmap privs: CONNECT type: database role: webmap_import,webmap_guest @@ -242,7 +266,7 @@ become: true become_user: postgres community.postgresql.postgresql_privs: - database: webmap + login_db: webmap privs: USAGE type: schema obj: postgis @@ -253,7 +277,7 @@ become: true become_user: postgres community.postgresql.postgresql_privs: - database: webmap + login_db: webmap privs: CREATE type: schema obj: postgis @@ -264,7 +288,7 @@ become: true become_user: postgres community.postgresql.postgresql_privs: - database: webmap + login_db: webmap privs: SELECT type: table obj: ALL_IN_SCHEMA @@ -275,7 +299,7 @@ become: true become_user: postgres community.postgresql.postgresql_privs: - database: webmap + login_db: webmap privs: USAGE,SELECT type: sequence obj: ALL_IN_SCHEMA @@ -306,6 +330,7 @@ service: name=webmap-import@{{ item }}.service enabled=true with_items: "{{ webmap_layer_groups }}" + - name: Build administrative-codes.json* become: false local_action: @@ -313,56 +338,80 @@ chdir: ./webmap-tools/administrative-codes target: all -- meta: flush_handlers - - -- name: Create system user '_webmap-publish' - user: name=_webmap-publish system=true - group=_webmap - createhome=false - home=/nonexistent - shell=/usr/sbin/nologin - comment="Webmap update (publication as MVT)" - password="!" - state=present - -- name: Copy /usr/local/share/webmap/publish.py - copy: src=webmap-tools/webmap-publish - dest=/usr/local/share/webmap/publish.py +- name: Create directory /var/www/webmap/data + file: path=/var/www/webmap/data + state=directory owner=root group=root mode=0755 -- name: Create /usr/local/bin/webmap-publish - file: src=../share/webmap/publish.py - dest=/usr/local/bin/webmap-publish +- name: Copy /var/www/webmap/data/administrative-codes.json* + copy: src=./webmap-tools/administrative-codes/{{ item }} + dest=/var/www/webmap/data/{{ item }} owner=root group=root - state=link force=yes + mode=0644 + with_items: + - administrative-codes.json + - administrative-codes.json.br + +- meta: flush_handlers + - name: Create directory /var/www/webmap/tiles file: path=/var/www/webmap/tiles state=directory - owner=_webmap-publish group=root + owner=_webmap group=root mode=0755 -- name: Copy webmap-publish@.service - copy: src=etc/systemd/system/webmap-publish@.service - dest=/etc/systemd/system/webmap-publish@.service + +- name: Copy /etc/tmpfiles.d/webmap.conf + copy: src=etc/tmpfiles.d/webmap.conf + dest=/etc/tmpfiles.d/webmap.conf owner=root group=root mode=0644 notify: - - systemctl daemon-reload + - systemd-tmpfiles --create -#- name: Enable webmap-publish@.service -# service: name=webmap-publish@{{ item }}.service enabled=true -# with_items: "{{ webmap_layer_groups }}" +- meta: flush_handlers -- name: Copy /etc/tmpfiles.d/webmap.conf - copy: src=etc/tmpfiles.d/webmap.conf - dest=/etc/tmpfiles.d/webmap.conf +- name: Install Python/WSGI dependencies + apt: pkg={{ packages }} + vars: + packages: + - uwsgi-core + - uwsgi-plugin-python3 + # TODO[trixie]: install python3-psycopg-c instead + - python3-psycopg + +- name: Copy webmap-cgi.socket + copy: src=etc/systemd/system/webmap-cgi.socket + dest=/etc/systemd/system/webmap-cgi.socket owner=root group=root mode=0644 notify: - - systemd-tmpfiles --create + - systemctl daemon-reload + +- name: Copy webmap-cgi.service + copy: src=etc/systemd/system/webmap-cgi.service + dest=/etc/systemd/system/webmap-cgi.service + owner=root group=root + mode=0644 + notify: + - systemctl daemon-reload + - Stop webmap-cgi.service + +- name: Copy /usr/libexec/webmap-cgi + copy: src=./webmap-tools/webmap-cgi + dest=/usr/libexec/webmap-cgi + owner=root group=root + mode=0755 + notify: + - Stop webmap-cgi.service - meta: flush_handlers + +- name: Enable webmap-cgi.socket + service: name=webmap-cgi.socket state=started enabled=true + +- name: Disable webmap-cgi.service + service: name=webmap-cgi.service enabled=false diff --git a/webmap-tools b/webmap-tools -Subproject 7cda119879cf48ba72ba34522fa9cdf9ef6d9b4 +Subproject 3e9586f0a9708afca40ab50a2987fc0090256e2 |