29 Jun, 2024
I've run this on a £3.20/month digital ocean droplet and it seems to work at about 2-6k requests per second for hello world depending on the concurrency. You'll need to answer some questions as you go for ufw and certbot.
WARNING: I haven't left what is deployed with this particular script running long enough to check renewal is actually successful or whether logs actually rotate. I'll update the post once I do.
#!/bin/bash
set -e
# Define environment variables
SANIC_USER="sanic"
GROUP_NAME="ssl-cert"
APP_DIR="/home/${SANIC_USER}/app"
VENV_DIR="/home/${SANIC_USER}/venv"
DOMAIN="hello.example.com"
WELL_KNOWN_DIR="${APP_DIR}/certbot"
# Update and upgrade existing packages
# apt update
apt -y upgrade
# Install necessary packages
apt install -y certbot acl fail2ban ntp unattended-upgrades logrotate htop python3-venv authbind ufw
# Configure Fail2Ban
cat <<EOL > /etc/fail2ban/jail.local
[DEFAULT]
bantime = 10m
findtime = 10m
maxretry = 5
[sshd]
enabled = true
port = ssh
backend = systemd
EOL
systemctl restart fail2ban
# Configure automatic updates
cat <<EOL > /etc/apt/apt.conf.d/20auto-upgrades
APT::Periodic::Update-Package-Lists "1";
APT::Periodic::Unattended-Upgrade "1";
EOL
# Configure UFW to allow SSH, HTTP, and HTTPS
ufw default deny incoming
ufw default allow outgoing
ufw allow ssh
ufw allow http
ufw allow https
ufw enable
# Check if the user exists
if id "$SANIC_USER" &>/dev/null; then
echo "User $SANIC_USER exists."
else
echo "User $SANIC_USER does not exist. Creating user..."
sudo adduser --disabled-password --gecos "" ${SANIC_USER}
sudo usermod -s /usr/sbin/nologin ${SANIC_USER}
sudo mkdir -p "/home/${SANIC_USER}/app"
sudo chown "${SANIC_USER}:${SANIC_USER}" -R "/home/${SANIC_USER}"
fi
# Ensure the application directory exists
mkdir -p ${APP_DIR}
chown -R ${SANIC_USER}:${SANIC_USER} ${APP_DIR}
sudo -u ${SANIC_USER} python3 -m venv ${VENV_DIR}
sudo -u ${SANIC_USER} ${VENV_DIR}/bin/pip install sanic
# Create hello.py
cat <<EOL > ${APP_DIR}/hello.py
from sanic import Sanic, response
from sanic.exceptions import SanicException
app = Sanic("hello")
@app.route("/")
async def hello_world(request):
return response.json({"message": "Hello, world!"})
# Custom error handler for various HTTP status codes
async def custom_error_handler(request, exception):
status_code = exception.status_code if isinstance(exception, SanicException) else 500
return response.json(
{"message": f"An error occurred: {status_code}"},
status=status_code
)
# Register the custom handler for common error statuses
app.error_handler.add(Exception, custom_error_handler)
app.error_handler.add(400, custom_error_handler)
app.error_handler.add(401, custom_error_handler)
app.error_handler.add(403, custom_error_handler)
app.error_handler.add(404, custom_error_handler)
app.error_handler.add(500, custom_error_handler)
if __name__ == "__main__":
app.run()
EOL
# Change ownership of the hello.py file
chown ${SANIC_USER}:${SANIC_USER} ${APP_DIR}/hello.py
# Create httpredirect.py
cat <<EOL > ${APP_DIR}/httpredirect.py
from sanic import Sanic, exceptions, response
app = Sanic("http_redir")
# Serve ACME/certbot files without HTTPS, for certificate renewals
app.static("/.well-known", "${WELL_KNOWN_DIR}/.well-known", resource_type="dir")
@app.exception(exceptions.NotFound, exceptions.MethodNotSupported)
async def redirect_everything_else(request, exception):
server, path = request.server_name, request.path
if server and path.startswith("/"):
return response.redirect(f"https://{server}{path}", status=308)
return response.text("Bad Request. Please use HTTPS!", status=400)
# Custom error handler for various HTTP status codes
async def custom_error_handler(request, exception):
status_code = exception.status_code if isinstance(exception, exceptions.SanicException) else 500
return response.json(
{"message": f"An error occurred: {status_code}"},
status=status_code
)
# Register the custom handler for common error statuses
app.error_handler.add(Exception, custom_error_handler)
app.error_handler.add(400, custom_error_handler)
app.error_handler.add(401, custom_error_handler)
app.error_handler.add(403, custom_error_handler)
app.error_handler.add(404, custom_error_handler)
app.error_handler.add(500, custom_error_handler)
if __name__ == "__main__":
app.run()
EOL
# Change ownership of the httpredirect.py file
chown ${SANIC_USER}:${SANIC_USER} ${APP_DIR}/httpredirect.py
# Configure authbind for Sanic to bind to ports 80 and 443
touch /etc/authbind/byport/80
touch /etc/authbind/byport/443
chown ${SANIC_USER}:${SANIC_USER} /etc/authbind/byport/80
chown ${SANIC_USER}:${SANIC_USER} /etc/authbind/byport/443
chmod 500 /etc/authbind/byport/80
chmod 500 /etc/authbind/byport/443
# Create the systemd service file for httpredirect
cat <<EOL > /etc/systemd/system/httpredirect.service
[Unit]
Description=Sanic HTTP Redirect Service
After=network.target
[Service]
User=${SANIC_USER}
WorkingDirectory=${APP_DIR}
ExecStart=/usr/bin/authbind ${VENV_DIR}/bin/sanic httpredirect:app --host 0.0.0.0 --port 80
Restart=always
[Install]
WantedBy=multi-user.target
EOL
# Reload systemd and start the httpredirect service
systemctl daemon-reload
systemctl enable httpredirect
systemctl restart httpredirect
# Ensure the certbot directory exists and has the correct permissions
mkdir -p ${WELL_KNOWN_DIR}/.well-known/acme-challenge
chown -R ${SANIC_USER}:${SANIC_USER} ${WELL_KNOWN_DIR}
# Check if the fullchain.pem exists
if [ ! -f "/etc/letsencrypt/live/${DOMAIN}/fullchain.pem" ]; then
# Obtain the Let's Encrypt certificate with recommended settings
certbot certonly --key-type ecdsa --preferred-chain "ISRG Root X1" --webroot -w ${WELL_KNOWN_DIR} -d ${DOMAIN}
# If you do a test run with the sandbox, the certificates might end up in a directory with -0001 suffix. If you don't delete this, the real certbot command uses the same directory.
fi
# Update certbot renewal configuration
cat <<EOL > /etc/letsencrypt/renewal/${DOMAIN}.conf
# renew_before_expiry = 30 days
version = 2.1.0
archive_dir = /etc/letsencrypt/archive/${DOMAIN}
cert = /etc/letsencrypt/live/${DOMAIN}/cert.pem
privkey = /etc/letsencrypt/live/${DOMAIN}/privkey.pem
chain = /etc/letsencrypt/live/${DOMAIN}/chain.pem
fullchain = /etc/letsencrypt/live/${DOMAIN}/fullchain.pem
# Options used in the renewal process
[renewalparams]
account = 04fxxx
authenticator = webroot
webroot_path = ${WELL_KNOWN_DIR}
server = https://acme-v02.api.letsencrypt.org/directory
key_type = ecdsa
EOL
if getent group "$GROUP_NAME" >/dev/null 2>&1; then
echo "Group $GROUP_NAME exists."
else
echo "Group $GROUP_NAME does not exist."
# Create a new group for SSL certificates
groupadd $GROUP_NAME
# Add the sanic user to the ssl-cert group
usermod -aG $GROUP_NAME $SANIC_USER
fi
# Set permissions for Let's Encrypt directories and files
chgrp -R $GROUP_NAME /etc/letsencrypt
chmod -R 750 /etc/letsencrypt
chmod -R 750 /etc/letsencrypt/live
chmod -R 750 /etc/letsencrypt/archive
chmod 640 /etc/letsencrypt/archive/${DOMAIN}/privkey1.pem
chmod 640 /etc/letsencrypt/archive/${DOMAIN}/fullchain1.pem
chmod 640 /etc/letsencrypt/archive/${DOMAIN}/cert1.pem
chmod 640 /etc/letsencrypt/archive/${DOMAIN}/chain1.pem
# Set ACLs to ensure new files have the correct permissions
setfacl -R -m "g:${GROUP_NAME}:rx" /etc/letsencrypt/live /etc/letsencrypt/archive
setfacl -R -m "d:g:${GROUP_NAME}:rx" /etc/letsencrypt/live /etc/letsencrypt/archive
# Create the systemd service file for the Sanic app
cat <<EOL > /etc/systemd/system/sanic.service
[Unit]
Description=Sanic Application
After=network.target
[Service]
User=${SANIC_USER}
WorkingDirectory=${APP_DIR}
ExecStart=/usr/bin/authbind ${VENV_DIR}/bin/sanic hello:app --host 0.0.0.0 --port 443 --tls /etc/letsencrypt/live/${DOMAIN}
Restart=always
[Install]
WantedBy=multi-user.target
EOL
# Reload systemd and start the Sanic service
systemctl daemon-reload
systemctl enable sanic
systemctl restart sanic
# Perform a dry run of certbot renew to ensure everything is set up correctly
certbot renew --dry-run
echo "Sanic setup with HTTP redirect, Fail2Ban, and Certbot configuration completed successfully."
This also sets up fail2ban, ufw, logrotate and automatic updates, but doesn't disable the root user's SSH or move SSH to a non-standard port (both of which should be done but I felt might interfere with the web console agent Digital Ocean provides). There is htop for some easier monitoring too and authbind allows the non-privileged sanic user to bind to 80 and 433.
Let me know if there are any potential issues you can spot.
Benchmarking:
$ curl https://hello.example.com/
{"message":"Hello, world!"}
$ wrk -t 8 -d 10 -c 256 https://hello.exampe.com/
Running 10s test @ https://hello.example.com/
8 threads and 256 connections
Thread Stats Avg Stdev Max +/- Stdev
Latency 38.93ms 36.54ms 896.85ms 98.30%
Req/Sec 791.96 230.74 1.16k 77.40%
60807 requests in 10.03s, 7.07MB read
Requests/sec: 6061.31
Transfer/sec: 722.15KB
By default, sanic starts quite a few workers which uses up a bit more RAM. It also seems to work fine if you add the --single-process flag to both services.
With a low memory machine you can get errors like this when installing packages:
update-initramfs: Generating /boot/initrd.img-6.1.0-21-amd64
Killed
E: mkinitramfs failure zstd -q -9 -T0 137
update-initramfs: failed for /boot/initrd.img-6.1.0-21-amd64 with 1.
dpkg: error processing package initramfs-tools (--configure):
installed initramfs-tools package post-installation script subprocess returned error exit status 1
Errors were encountered while processing:
initramfs-tools
E: Sub-process /usr/bin/dpkg returned an error code (1)
Add some swap space and you should be all good:
sudo fallocate -l 512M /swapfile
sudo chmod 600 /swapfile
sudo mkswap /swapfile
sudo swapon /swapfile
If you want the swap space mounted each boot (probably a good idea) you can run this once to add the correct line to /etc/fstab:
echo '/swapfile none swap sw 0 0' | sudo tee -a /etc/fstab
NOTE: setguid might be a better way of sharing access to the /etc/letsencrypt directory:
# Set group ownership to ssl-cert
sudo chgrp -R ssl-cert /etc/letsencrypt
# Apply setgid bit to ensure new files inherit the group
sudo find /etc/letsencrypt -type d -exec chmod g+s {} +
Also there's a bug with hard coding the certbot account ID.
Update Nov 2024:
root@debian-s-1vcpu-512mb-10gb-ams3-01:~# sudo systemctl status certbot.timer
● certbot.timer - Run certbot twice daily
Loaded: loaded (/lib/systemd/system/certbot.timer; enabled; preset: enabled)
Active: active (waiting) since Mon 2024-07-01 10:32:18 UTC; 4 months 8 days ago
Trigger: Fri 2024-11-08 19:52:21 UTC; 6h left
Triggers: ● certbot.service
Notice: journal has been rotated since unit was started, output may be incomplete.
root@debian-s-1vcpu-512mb-10gb-ams3-01:~# sudo systemctl status certbot.service
× certbot.service - Certbot
Loaded: loaded (/lib/systemd/system/certbot.service; static)
Active: failed (Result: exit-code) since Fri 2024-11-08 03:11:43 UTC; 10h ago
TriggeredBy: ● certbot.timer
Docs: file:///usr/share/doc/python-certbot-doc/html/index.html
https://certbot.eff.org/docs
Process: 2057673 ExecStart=/usr/bin/certbot -q renew --no-random-sleep-on-renew (code=exited, status=1/FAILU>
Main PID: 2057673 (code=exited, status=1/FAILURE)
CPU: 938ms
Nov 08 03:11:42 debian-s-1vcpu-512mb-10gb-ams3-01 systemd[1]: Starting certbot.service - Certbot...
Nov 08 03:11:43 debian-s-1vcpu-512mb-10gb-ams3-01 certbot[2057673]: Failed to renew certificate hello.productspe>
Nov 08 03:11:43 debian-s-1vcpu-512mb-10gb-ams3-01 certbot[2057673]: All renewals failed. The following certifica>
Nov 08 03:11:43 debian-s-1vcpu-512mb-10gb-ams3-01 certbot[2057673]: /etc/letsencrypt/live/hello.productspeople>
Nov 08 03:11:43 debian-s-1vcpu-512mb-10gb-ams3-01 certbot[2057673]: 1 renew failure(s), 0 parse failure(s)
Nov 08 03:11:43 debian-s-1vcpu-512mb-10gb-ams3-01 systemd[1]: certbot.service: Main process exited, code=exited,>
Nov 08 03:11:43 debian-s-1vcpu-512mb-10gb-ams3-01 systemd[1]: certbot.service: Failed with result 'exit-code'.
Nov 08 03:11:43 debian-s-1vcpu-512mb-10gb-ams3-01 systemd[1]: Failed to start certbot.service - Certbot.
root@debian-s-1vcpu-512mb-10gb-ams3-01:~# sudo tail -n 50 /var/log/letsencrypt/letsencrypt.log
Initialized: <certbot._internal.plugins.webroot.Authenticator object at 0x7fce62c6c9d0>
Prep: True
2024-11-08 03:11:43,378:DEBUG:certbot._internal.plugins.selection:Selected authenticator <certbot._internal.plugins.webroot.Authenticator object at 0x7fce62c6c9d0> and installer None
2024-11-08 03:11:43,379:INFO:certbot._internal.plugins.selection:Plugins selected: Authenticator webroot, Installer None
2024-11-08 03:11:43,381:ERROR:certbot._internal.renewal:Failed to renew certificate hello.productspeoplecareabout.com with error: Account at /etc/letsencrypt/accounts/acme-v01.api.letsencrypt.org/directory/04fxxx does not exist
2024-11-08 03:11:43,386:DEBUG:certbot._internal.renewal:Traceback was:
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/certbot/_internal/renewal.py", line 532, in handle_renewal_request
main.renew_cert(lineage_config, plugins, renewal_candidate)
File "/usr/lib/python3/dist-packages/certbot/_internal/main.py", line 1538, in renew_cert
le_client = _init_le_client(config, auth, installer)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3/dist-packages/certbot/_internal/main.py", line 827, in _init_le_client
acc, acme = _determine_account(config)
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3/dist-packages/certbot/_internal/main.py", line 721, in _determine_account
acc = account_storage.load(config.account)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3/dist-packages/certbot/_internal/account.py", line 244, in load
return self._load_for_server_path(account_id, self.config.server_path)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3/dist-packages/certbot/_internal/account.py", line 220, in _load_for_server_path
prev_loaded_account = self._load_for_server_path(account_id, prev_server_path)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3/dist-packages/certbot/_internal/account.py", line 229, in _load_for_server_path
raise errors.AccountNotFound(f"Account at {account_dir_path} does not exist")
certbot.errors.AccountNotFound: Account at /etc/letsencrypt/accounts/acme-v01.api.letsencrypt.org/directory/04fxxx does not exist
2024-11-08 03:11:43,386:DEBUG:certbot._internal.display.obj:Notifying user:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
2024-11-08 03:11:43,386:ERROR:certbot._internal.renewal:All renewals failed. The following certificates could not be renewed:
2024-11-08 03:11:43,387:ERROR:certbot._internal.renewal: /etc/letsencrypt/live/hello.productspeoplecareabout.com/fullchain.pem (failure)
2024-11-08 03:11:43,387:DEBUG:certbot._internal.display.obj:Notifying user: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
2024-11-08 03:11:43,387:DEBUG:certbot._internal.log:Exiting abnormally:
Traceback (most recent call last):
File "/usr/bin/certbot", line 33, in <module>
sys.exit(load_entry_point('certbot==2.1.0', 'console_scripts', 'certbot')())
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3/dist-packages/certbot/main.py", line 19, in main
return internal_main.main(cli_args)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3/dist-packages/certbot/_internal/main.py", line 1736, in main
return config.func(config, plugins)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3/dist-packages/certbot/_internal/main.py", line 1629, in renew
renewal.handle_renewal_request(config)
File "/usr/lib/python3/dist-packages/certbot/_internal/renewal.py", line 558, in handle_renewal_request
raise errors.Error(
certbot.errors.Error: 1 renew failure(s), 0 parse failure(s)
2024-11-08 03:11:43,388:ERROR:certbot._internal.log:1 renew failure(s), 0 parse failure(s)
root@debian-s-1vcpu-512mb-10gb-ams3-01:~# ls -la /etc/letsencrypt/accounts/acme-v01.api.letsencrypt.org/directory/
ls: cannot access '/etc/letsencrypt/accounts/acme-v01.api.letsencrypt.org/directory/': No such file or directory
root@debian-s-1vcpu-512mb-10gb-ams3-01:~# ls -la /etc/letsencrypt/accounts/acme-v01.api.letsencrypt.orgls: cannot access '/etc/letsencrypt/accounts/acme-v01.api.letsencrypt.org': No such file or directory
root@debian-s-1vcpu-512mb-10gb-ams3-01:~# ls -la /etc/letsencrypt/accounts/
total 16
drwxr-x--- 4 root ssl-cert 4096 Jun 29 06:32 .
drwxr-x--- 9 root ssl-cert 4096 Nov 8 03:11 ..
drwxr-x--- 3 root ssl-cert 4096 Jun 29 06:32 acme-staging-v02.api.letsencrypt.org
drwxr-x--- 3 root ssl-cert 4096 Jun 29 06:32 acme-v02.api.letsencrypt.org
root@debian-s-1vcpu-512mb-10gb-ams3-01:~# ls -la /etc/letsencrypt/accounts/acme-v02.api.letsencrypt.org/directory/
total 12
drwxr-x--- 3 root ssl-cert 4096 Jun 29 06:32 .
drwxr-x--- 3 root ssl-cert 4096 Jun 29 06:32 ..
drwxr-x--- 2 root ssl-cert 4096 Jun 29 06:32 daxxx
root@debian-s-1vcpu-512mb-10gb-ams3-01:~# sudo vim /etc/letsencrypt/renewal/hello.productspeoplecareabout.com.conf
Then it works again. So it seems like 01 became deprecated, and there was a 02 account registered, but although the auto update updated all packages, the certbot renew failed by trying to use the old account ID until the config was updated.
Let's see if renews next time.
No renewal hook ran so once the certificate renewed, the service didn't restart:
sudo certbot renew --deploy-hook "systemctl restart sanic"
Add it:
sudo vim /etc/letsencrypt/renewal/hello.productspeoplecareabout.com.conf
[renewparams]
renew_hook = systemctl restart sanic
sudo certbot renew --force-renewal
Be the first to comment.
Copyright James Gardner 1996-2020 All Rights Reserved. Admin.