Hey Folks
we have set up a router in the following scenario on our EdgeRouter-4:
- Dual WAN (eth0 via Cable Router and eth1/pppoe0 via VDSL modem - both public static IPs)
- Failover with primary usage of the cable line
- Dual IPSEC tunnels to different remote gateways (each remote gateway is bound to one connection with static routes)
- IPSEC is configured with GRE tunnel interfaces on top with OSPF on top of that
- several internal subnets (eth2) which reach other internal systems via IPSEC but forward all other traffic directly through the local internet connections
Every once in a while the internal subnets are offline because traffic reaches the EdgeRouter via VPN but gets then forwarded to the internet instead of the eth2 (or its vlan sub interfaces).
What we have observed: at some point, all "connected" routes vanish from the additional routing tables 201/202 and therefore all traffic is routed via the default route directly to the internet.
From what I have learned, the router triggers the script /usr/sbin/ubnt-add-connected.pl periodically - this script fails in our scenario:
/usr/sbin/ubnt-add-connected.pl
Error no prefix found
I took a look at the code and it calls vtysh to get a list of routes:
vtysh -c "show ip route json"
In our scenario, this returns a list of prefix objects (in that case a lot, because it lists all the internal OSPF routes received via IPSEC). However, it also contains an additional list item, which fails the script's sanity check (as there is no 'pfx' item):
[...]
{
"pfx" : "1.2.3.0/30",
"nh" : [
{
"up" : "00:10:23",
"via" : "10.0.0.2",
"intf" : "tun1",
"metric" : "110/2010",
"t" : "O>*"
}
]
},
{
"pfx" : "1.2.4.0/30",
"nh" : [
{
"t" : "O>*",
"metric" : "110/2010",
"intf" : "tun1",
"via" : "10.0.0.2",
"up" : "00:10:23"
}
]
},
{
"too_many_routes" : "1"
}
]
It seems to be related to this part of the code:
foreach my $r (@routes) {
my $pfx = $r->{pfx};
if (!defined $pfx) {
print "Error no prefix found\n";
exit 1;
}
Does anyone know a sensible quick fix for this?
For completeness, this is our current config and we are running 1.10.5 as of today:
firewall {
all-ping enable
broadcast-ping disable
group {
network-group Other-Workstations-Nets {
network 1.2.3.128/25
network 4.5.6.0/23
}
network-group Our-Prefixes {
description "All of our public Networks"
network 1.2.0.0/20
network 4.5.6.0/20
}
}
ipv6-receive-redirects disable
ipv6-src-route disable
ip-src-route disable
log-martians enable
modify WAN_POLICY {
rule 10 {
action modify
destination {
group {
network-group !Our-Prefixes
}
}
modify {
lb-group WAN_FAILOVER
}
}
rule 20 {
action modify
destination {
group {
network-group Our-Prefixes
}
}
modify {
lb-group WAN_FAILOVER
}
source {
address 192.168.0.0/24
}
}
}
name VPN_IN {
default-action drop
description "VPN to local workstation network"
rule 05 {
action accept
description "Allow OSPF on GRE tunnel"
protocol ospf
}
rule 10 {
action accept
description "Allow established/related"
state {
established enable
related enable
}
}
rule 20 {
action drop
state {
invalid enable
}
}
rule 30 {
action accept
description "Allow Traffic from other Workstation networks"
source {
group {
network-group Other-Workstations-Nets
}
}
}
rule 40 {
action accept
description "Allow incoming ICMP via tunnel"
protocol icmp
}
}
name VPN_LOCAL {
default-action drop
description "WAN to router"
rule 05 {
action accept
description "Allow OSPF on tunnel interface"
protocol ospf
}
rule 10 {
action accept
description "Allow established/related"
state {
established enable
related enable
}
}
rule 20 {
action drop
description "Drop invalid state"
state {
invalid enable
}
}
rule 30 {
action accept
description "Allow Management SSH access"
destination {
port 22
}
protocol tcp
source {
group {
network-group Other-Workstations-Nets
}
}
}
rule 40 {
action accept
description "Allow incoming ICMP via tunnel"
protocol icmp
}
}
name WAN_IN {
default-action drop
description "WAN to internal"
rule 10 {
action accept
description "Allow established/related"
state {
established enable
related enable
}
}
rule 20 {
action drop
description "Drop invalid state"
state {
invalid enable
}
}
}
name WAN_LOCAL {
default-action drop
description "WAN to router"
rule 10 {
action accept
description "Allow established/related"
state {
established enable
related enable
}
}
rule 20 {
action drop
description "Drop invalid state"
state {
invalid enable
}
}
rule 30 {
action accept
description "Allow Management SSH access"
destination {
port 22
}
protocol tcp
source {
group {
network-group Other-Workstations-Nets
}
}
}
rule 40 {
action accept
description "Allow sipgate VPN"
source {
address 1.2.0.10
}
}
}
receive-redirects disable
send-redirects enable
source-validation disable
syn-cookies enable
}
interfaces {
ethernet eth0 {
address 7.7.5.6/30
description Cable-Connection
duplex auto
firewall {
in {
name WAN_IN
}
local {
name WAN_LOCAL
}
}
speed auto
}
ethernet eth1 {
description VDSL-Connection
duplex auto
speed auto
vif 7 {
description VDSL-Vlan
pppoe 0 {
default-route none
firewall {
in {
name WAN_IN
}
local {
name WAN_LOCAL
}
}
mtu 1492
name-server none
password SECRET
user-id SECRET@SECRET.COM
}
}
}
ethernet eth2 {
address 192.168.1.1/24
description LAN
duplex auto
speed auto
vif 100 {
address 1.2.5.129/25
description Workstations
firewall {
in {
modify WAN_POLICY
}
}
}
vif 408 {
address 192.168.0.1/24
description Guests
firewall {
in {
modify WAN_POLICY
}
}
}
}
ethernet eth3 {
disable
duplex auto
speed auto
}
loopback lo {
address 1.2.0.5/32
}
tunnel tun0 {
address 10.0.0.2/30
encapsulation gre
firewall {
in {
name VPN_IN
}
local {
name VPN_LOCAL
}
}
ip {
ospf {
cost 10
dead-interval 4
hello-interval 1
network point-to-point
priority 1
retransmit-interval 5
transmit-delay 1
}
}
local-ip 7.7.5.5
mtu 1300
multicast disable
remote-ip 1.2.0.11
ttl 255
}
tunnel tun1 {
address 10.0.0.6/30
encapsulation gre
firewall {
in {
name VPN_IN
}
local {
name VPN_LOCAL
}
}
ip {
ospf {
cost 1000
dead-interval 4
hello-interval 1
network point-to-point
priority 1
retransmit-interval 5
transmit-delay 1
}
}
local-ip 8.8.9.9
mtu 1300
multicast disable
remote-ip 1.2.0.10
ttl 255
}
}
load-balance {
group WAN_FAILOVER {
interface eth0 {
}
interface pppoe0 {
failover-only
}
lb-local enable
lb-local-metric-change disable
}
}
protocols {
ospf {
area 0.0.0.0 {
network 10.0.0.0/30
network 10.0.0.4/30
}
area 0.0.0.12 {
network 1.2.5.128/25
network 192.168.1.0/24
network 11.2.0.5/32
}
parameters {
abr-type standard
router-id 11.2.0.5
}
passive-interface eth2.100
passive-interface eth2
passive-interface eth2.408
passive-interface lo
}
static {
interface-route 0.0.0.0/0 {
next-hop-interface pppoe0 {
}
}
interface-route 1.2.0.10/32 {
next-hop-interface pppoe0 {
}
}
route 0.0.0.0/0 {
next-hop 7.7.5.5 {
}
}
route 1.2.0.11/32 {
next-hop 7.7.5.5 {
}
}
}
}
service {
dhcp-server {
disabled false
hostfile-update disable
shared-network-name Guest-WLAN {
authoritative enable
subnet 192.168.0.0/24 {
default-router 192.168.0.1
dns-server 192.168.0.1
domain-name localdomain.com
lease 86400
start 192.168.0.2 {
stop 192.168.0.254
}
}
}
shared-network-name Management {
authoritative enable
subnet 192.168.1.0/24 {
default-router 192.168.1.1
dns-server 192.168.1.1
domain-name localdomain.com
lease 86400
start 192.168.1.5 {
stop 192.168.1.254
}
}
}
shared-network-name Workstations {
authoritative enable
subnet 1.2.5.128/25 {
default-router 1.2.5.129
dns-server 1.2.5.129
domain-name localdomain.com
lease 86400
start 1.2.5.131 {
stop 1.2.5.254
}
}
}
static-arp enable
use-dnsmasq disable
}
dns {
forwarding {
cache-size 150
listen-on eth2
listen-on eth2.408
listen-on eth2.100
}
}
nat {
rule 5001 {
outbound-interface eth0
type masquerade
}
rule 5002 {
outbound-interface pppoe0
type masquerade
}
}
ssh {
port 22
protocol-version v2
}
unms {
disable
}
}
system {
domain-name localdomain.com
host-name router-office
login {
user admin {
authentication {
encrypted-password SECRET
}
level admin
}
}
name-server 8.8.8.8
ntp {
server 0.ubnt.pool.ntp.org {
}
server 1.ubnt.pool.ntp.org {
}
server 2.ubnt.pool.ntp.org {
}
server 3.ubnt.pool.ntp.org {
}
}
offload {
hwnat disable
ipsec enable
ipv4 {
forwarding enable
gre enable
pppoe enable
}
}
syslog {
global {
facility all {
level notice
}
facility protocols {
level debug
}
}
}
time-zone Europe/Berlin
}
vpn {
ipsec {
auto-firewall-nat-exclude enable
esp-group office-provider-1-esp {
compression disable
lifetime 3600
mode tunnel
pfs enable
proposal 1 {
encryption aes256
hash sha1
}
}
esp-group office-provider-2-esp {
compression disable
lifetime 3600
mode tunnel
pfs enable
proposal 1 {
encryption aes256
hash sha1
}
}
ike-group office-provider-1-ike {
ikev2-reauth no
key-exchange ikev2
lifetime 28800
proposal 1 {
dh-group 5
encryption aes256
hash sha1
}
}
ike-group office-provider-2-ike {
ikev2-reauth no
key-exchange ikev2
lifetime 28800
proposal 1 {
dh-group 5
encryption aes256
hash sha1
}
}
site-to-site {
peer 1.2.0.11 {
authentication {
mode pre-shared-secret
pre-shared-secret SECRET
}
connection-type initiate
default-esp-group office-provider-2-esp
ike-group office-provider-2-ike
ikev2-reauth inherit
local-address 7.7.5.6
tunnel 1 {
allow-nat-networks disable
allow-public-networks disable
protocol gre
}
}
peer 1.2.0.10 {
authentication {
mode pre-shared-secret
pre-shared-secret SECRET
}
connection-type initiate
default-esp-group office-provider-1-esp
ike-group office-provider-1-ike
ikev2-reauth inherit
local-address 8.8.9.9
tunnel 1 {
allow-nat-networks disable
allow-public-networks disable
protocol gre
}
}
}
}
}