Hey,
Hoping to get some help from folks who are much smarter and more experienced in homelabbing than me.
Brief description of my configuration:
– Traefik swarm on 4 nodes (3 managers + 1 worker) running inside ubuntu VMs on 4 separate Proxmox physical instances
– OPNsense on bare metal as router / firewall
– 2 wired APs
I am using host override features under Services > Unbound in OPNsense to redirect my local domains to my server IP
Now the problem is that all this is working perfectly when I am using a single server defined in the host override section, but once I host a service in any different server, the traefik returns bad gateway error or goes into infinite looping, I tried defining all 4 server IPs in host override for round robin but it didn't work as well.
Important thing to note here is that prior moving to OPNsense this week, I tried to make the swarm work with Technitium as the local dns and defined zones to achieve the same objective but again it used to work fine for a single server but not for all.
I have been going through various forums for weeks now but can't seem to figure out the missing piece in my setup or a completely different setup that works.
I would really appreciate for some pointers, willing to provide any more information to help resolve this issue.
Attaching my network architecture
TIA
docker-stack.yml:
services: traefik: image: traefik:v3.7 # container_name: traefik hostname: '{{.Node.Hostname}}' # security_opt: # - no-new-privileges:true networks: - proxy-swarm ports: # listen on host ports without ingress network - target: 80 published: 80 protocol: tcp mode: host - target: 443 published: 443 protocol: tcp mode: host - 443:443/udp # Uncomment if you want HTTP3 environment: CF_DNS_API_TOKEN_FILE: /run/secrets/cf_api_token # note using _FILE for docker secrets # CF_DNS_API_TOKEN: ${CF_DNS_API_TOKEN} # if using .env TRAEFIK_DASHBOARD_CREDENTIALS: xxxx secrets: - cf_api_token # env_file: .env # use .env volumes: - /etc/localtime:/etc/localtime:ro - /var/run/docker.sock:/var/run/docker.sock:ro - /docker/traefik/data/traefik.yml:/traefik.yml:ro - /docker/traefik/data/acme.json:/acme.json - /docker/traefik/config.yml:/config.yml:ro - /docker/traefik/traefik.log:/var/log/traefik.log deploy: mode: global restart_policy: condition: on-failure placement: constraints: - node.role == manager labels: - "traefik.enable=true" - "traefik.http.routers.traefik.entrypoints=http" - "traefik.http.routers.traefik.rule=Host(`traefik-dashboard.example.cc`)" # - "traefik.http.middlewares.traefik-auth.basicauth.users=${TRAEFIK_DASHBOARD_CREDENTIALS}" - "traefik.http.middlewares.traefik-auth.basicauth.users=xxxx" - "traefik.http.middlewares.traefik-https-redirect.redirectscheme.scheme=https" - "traefik.http.middlewares.default-headers.headers.customRequestHeaders.X-Forwarded-Proto=https" - "traefik.http.routers.traefik.middlewares=traefik-https-redirect" - "traefik.http.routers.traefik-secure.entrypoints=https" - "traefik.http.routers.traefik-secure.rule=Host(`traefik-dashboard.example.com`)" - "traefik.http.routers.traefik-secure.middlewares=traefik-auth" - "traefik.http.routers.traefik-secure.tls=true" - "traefik.http.routers.traefik-secure.tls.certresolver=cloudflare" - "traefik.http.routers.traefik-secure.tls.domains[0].main=example.com" - "traefik.http.routers.traefik-secure.tls.domains[0].sans=*.example.com" - "traefik.http.routers.traefik-secure.service=api@internal" - "traefik.http.services.dummy-svc.loadbalancer.server.port=9999" secrets: cf_api_token: file: /docker/traefik/cf_api_token.txt networks: proxy-swarm: external: true traefik.yml:
api: dashboard: true debug: true log: filePath: "/var/log/traefik.log" format: json level: ERROR accessLog: # JSON format format: json # Filter on status codes, retry attempts and minimal duration filters: statusCodes: - "400-599" retryAttempts: true minDuration: "10ms" entryPoints: http: address: ":80" http: redirections: entryPoint: to: https scheme: https https: address: ":443" serversTransport: insecureSkipVerify: true providers: docker: endpoint: "unix:///var/run/docker.sock" exposedByDefault: false swarm: endpoint: "unix:///var/run/docker.sock" exposedByDefault: false network: proxy-swarm file: filename: /config.yml certificatesResolvers: cloudflare: acme: email: xxxx storage: acme.json caServer: https://acme-v02.api.letsencrypt.org/directory # prod (default) dnsChallenge: provider: cloudflare #disablePropagationCheck: true # uncomment this if you have issues pulling certificates through cloudflare, By setting this flag to true disables the need to wait for the propagation of the TXT record to all authoritative name servers. #delayBeforeCheck: 60s # uncomment along with disablePropagationCheck if needed to ensure the TXT record is ready before verification is attempted resolvers: - "1.1.1.1:53" - "1.0.0.1:53" config.yml:
middlewares: https-redirectscheme: redirectScheme: scheme: https permanent: true default-headers: headers: frameDeny: true browserXssFilter: true contentTypeNosniff: true forceSTSHeader: true stsIncludeSubdomains: true stsPreload: true stsSeconds: 15552000 customFrameOptionsValue: SAMEORIGIN customRequestHeaders: X-Forwarded-Proto: https default-whitelist: ipAllowList: sourceRange: - "172.20.0.0/16" - "192.168.0.0/24" secured: chain: middlewares: - default-whitelist - default-headers Source: r/Traefik · by /u/Lone_Assassin