Got it working on my instance by using the following config for the nginx outside of the docker container (you'll need to change the server_name from the server sections and the path to the ssl keys) :

upstream lemmy {
    # depending on your setup, you may want to update this
    server 127.0.0.1:1380;
}

server {
    listen 80;
    listen [::]:80;
    server_name lemmy.pierre-couy.fr;
    location / { return 301 https://$host$request_uri; }
}
server {
    listen      443 ssl;
    listen [::]:443 ssl;
    server_name lemmy.pierre-couy.fr;

    # TLS
    ssl_protocols TLSv1.2;
    ssl_ciphers HIGH:!MEDIUM:!LOW:!aNULL:!NULL:!SHA;
    ssl_prefer_server_ciphers on;
    ssl_session_cache shared:SSL:10m;
    ssl_certificate     /etc/ssl/certs/pcouy.pem;
    ssl_certificate_key /etc/ssl/private/pcouy.key;

    # HSTS
    add_header Strict-Transport-Security "max-age=31536000";

    location / {
        proxy_pass http://lemmy;
        proxy_set_header Host $host;
        # include proxy_params;
    }

    location ~ ^/(api|pictrs|feeds|nodeinfo|.well-known) {
        proxy_pass "http://lemmy";
        # proxy common stuff
        proxy_http_version 1.1;
        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection "upgrade";

        # Send actual client IP upstream
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header Host $host;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
    }
}