EdgeRouter X LAN Failover Config

Hello! I'm trying to setup a failover for my WAN connection. I'd like my second WAN (eth2) to only be used when the promary fails (eth0).

At the moment, both interfaces are getting IP addresses correctly. The issue is that my second WAN is not working properly. When the primary fails, I loose my Internet connection. I've also tried physically switching the ethernet cords for the two wan connections and the failover to eth2 still fails. This at least, hopefully, tells me that it's not an issue with the WAN connections themselfs.

Here's my config, any help would be greatly appreciated! If any other snippets of the config would be usefull, do let me know.

load-balance {
    group FAILOVER {
        interface eth0 {
            route-test {
                type {
                    ping {
                        target 8.8.8.8
                    }
                }
            }
        }
        interface eth2 {
            failover-only
            route-test {
                type {
                    ping {
                        target 8.8.8.8
                    }
                }
            }
        }
    }
}

    switch switch0 {
        address 192.168.1.1/24
        description Local
        firewall {
            in {
                modify WAN_FAILOVER
            }
        }
        mtu 1500
        switch-port {
            interface eth1
            interface eth3
        }
        traffic-policy {
            out download
        }
    }

firewall {
    all-ping enable
    broadcast-ping disable
    group {
        network-group PRIVATE_LANS {
            description "Non-Routeable Private Subnets"
            network 10.0.0.0/8
            network 172.16.0.0/12
            network 192.168.0.0/16
        }
    }
    ipv6-receive-redirects disable
    ipv6-src-route disable
    ip-src-route disable
    log-martians enable
    modify WAN_FAILOVER {
        rule 1 {
            action accept
            description "Exclude private subnets from load balancing"
            destination {
                group {
                    network-group PRIVATE_LANS
                }
            }
        }
        rule 10 {
            action modify
            modify {
                lb-group FAILOVER
            }
        }
    }