Problem configuring NethServer as gateway in Proxmox

Andy_Wismer · February 29, 2020, 10:01am

Show the output of

nano /etc/network/interfaces

capote · February 29, 2020, 10:02am

root@proxmox:~# cat /etc/network/interfaces
auto lo
iface lo inet loopback

auto enp3s0
iface enp3s0 inet manual
#Connected to Switch

auto enp2s0
iface enp2s0 inet manual
#connected: to DSL router

auto vmbr0
iface vmbr0 inet static
address 192.168.3.200
netmask 255.255.255.0
gateway 192.168.3.1
bridge-ports enp3s0
bridge-stp off
bridge-fd 0

auto vmbr1
iface vmbr1 inet manual
bridge-ports enp2s0
bridge-stp off
bridge-fd 0
root@proxmox:~#

inside the new nethserver:

[root@v-nethserver2 ~]# ifconfig
eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet 192.168.2.100 netmask 255.255.255.0 broadcast 192.168.2.255
inet6 fe80::e091:9fff:fea8:3121 prefixlen 64 scopeid 0x20
inet6 fd65:3544:2664:1:e091:9fff:fea8:3121 prefixlen 64 scopeid 0x0
inet6 xxxx:xx:xxxx:xxxx:e091:9fff:fea8:3121 prefixlen 64 scopeid 0x0
ether e2:91:9f:a8:31:21 txqueuelen 1000 (Ethernet)
RX packets 65761 bytes 187650469 (178.9 MiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 48243 bytes 7118254 (6.7 MiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0

lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536
inet 127.0.0.1 netmask 255.0.0.0
inet6 ::1 prefixlen 128 scopeid 0x10
loop txqueuelen 1000 (Lokale Schleife)
RX packets 1260 bytes 1845727 (1.7 MiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 1260 bytes 1845727 (1.7 MiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0

[root@v-nethserver2 ~]#

Andy_Wismer · February 29, 2020, 10:10am

ipconfig is deprecated in Debian since Stretch, the replacement is simply:

ip a

This is the output of my friends Proxmox Server:

root@abho-pve-1:~# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: eno1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq master vmbr0 state UP group default qlen 1000
link/ether 34:64:a9:9a:d5:9c brd ff:ff:ff:ff:ff:ff
3: eno2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq master vmbr1 state UP group default qlen 1000
link/ether 34:64:a9:9a:d5:9d brd ff:ff:ff:ff:ff:ff
4: vmbr0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
link/ether 34:64:a9:9a:d5:9c brd ff:ff:ff:ff:ff:ff
inet 192.168.209.61/24 brd 192.168.209.255 scope global vmbr0
valid_lft forever preferred_lft forever
inet6 fe80::3664:a9ff:fe9a:d59c/64 scope link
valid_lft forever preferred_lft forever
5: vmbr1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
link/ether 34:64:a9:9a:d5:9d brd ff:ff:ff:ff:ff:ff
inet6 fe80::3664:a9ff:fe9a:d59d/64 scope link
valid_lft forever preferred_lft forever
6: tap119i0: <BROADCAST,MULTICAST,PROMISC,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast master vmbr0 state UNKNOWN group default qlen 1000
link/ether 4a:fb:65:d0:5b:b7 brd ff:ff:ff:ff:ff:ff
7: tap119i1: <BROADCAST,MULTICAST,PROMISC,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast master vmbr1 state UNKNOWN group default qlen 1000
link/ether 56:f4:de:fd:b6:07 brd ff:ff:ff:ff:ff:ff
8: tap100i0: <BROADCAST,MULTICAST,PROMISC,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast master vmbr0 state UNKNOWN group default qlen 1000
link/ether 26:a6:5b:ae:47:b1 brd ff:ff:ff:ff:ff:ff
13: veth202i0@if12: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue master vmbr0 state UP group default qlen 1000
link/ether fe:52:27:16:fa:ac brd ff:ff:ff:ff:ff:ff link-netnsid 1
17: veth203i0@if16: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue master vmbr0 state UP group default qlen 1000
link/ether fe:cf:90:fc:8b:6e brd ff:ff:ff:ff:ff:ff link-netnsid 0
root@abho-pve-1:~#

We need to know, if Proxmox has 2 NICs running…
(vmbr0 and vmbr1)

capote · February 29, 2020, 10:13am

that was inside the v-nethserver2 / Centos 7

Proxmox:
root@proxmox:~# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: enp2s0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast master vmbr1 state UP group default qlen 1000
link/ether 00:01:2e:81:e1:bb brd ff:ff:ff:ff:ff:ff
3: enp3s0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast master vmbr0 state UP group default qlen 1000
link/ether 00:01:2e:81:e1:bc brd ff:ff:ff:ff:ff:ff
4: wlp1s0: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN group default qlen 1000
link/ether 00:e1:8c:3a:98:b9 brd ff:ff:ff:ff:ff:ff
5: vmbr0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
link/ether 00:01:2e:81:e1:bc brd ff:ff:ff:ff:ff:ff
inet 192.168.3.200/24 scope global vmbr0
valid_lft forever preferred_lft forever
inet6 fe80::201:2eff:fe81:e1bc/64 scope link
valid_lft forever preferred_lft forever
6: vmbr1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
link/ether 00:01:2e:81:e1:bb brd ff:ff:ff:ff:ff:ff
inet6 fd65:3544:2664:1:201:2eff:fe81:e1bb/64 scope global dynamic mngtmpaddr
valid_lft 1814364sec preferred_lft 604764sec
inet6 xxx:xx:xxxx:xxxx:201:2eff:fe81:e1bb/64 scope global dynamic mngtmpaddr
valid_lft 604764sec preferred_lft 86364sec
inet6 fe80::201:2eff:fe81:e1bb/64 scope link
valid_lft forever preferred_lft forever
7: tap101i0: <BROADCAST,MULTICAST,PROMISC,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast master vmbr1 state UNKNOWN group default qlen 1000
link/ether 26:5d:a2:4a:82:82 brd ff:ff:ff:ff:ff:ff
9: tap100i0: <BROADCAST,MULTICAST,PROMISC,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast master vmbr1 state UNKNOWN group default qlen 1000
link/ether 42:af:95:3a:40:63 brd ff:ff:ff:ff:ff:ff
root@proxmox:~#

Nethserver2:

[root@v-nethserver2 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether e2:91:9f:a8:31:21 brd ff:ff:ff:ff:ff:ff
inet 192.168.2.100/24 brd 192.168.2.255 scope global dynamic eth0
valid_lft 1576sec preferred_lft 1576sec
inet6 fd65:3544:2664:1:e091:9fff:fea8:3121/64 scope global mngtmpaddr dynamic
valid_lft 1814369sec preferred_lft 604769sec
inet6 2003:d1:bf42:b24:e091:9fff:fea8:3121/64 scope global mngtmpaddr dynamic
valid_lft 604769sec preferred_lft 86369sec
inet6 fe80::e091:9fff:fea8:3121/64 scope link
valid_lft forever preferred_lft forever
[root@v-nethserver2 ~]#

Andy_Wismer · February 29, 2020, 10:23am

This part looks good…

These also look good:

5: vmbr0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
link/ether 00:01:2e:81:e1:bc brd ff:ff:ff:ff:ff:ff
inet 192.168.3.200/24 scope global vmbr0
valid_lft forever preferred_lft forever
inet6 fe80::201:2eff:fe81:e1bc/64 scope link
valid_lft forever preferred_lft forever
6: vmbr1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
link/ether 00:01:2e:81:e1:bb brd ff:ff:ff:ff:ff:ff
inet6 fd65:3544:2664:1:201:2eff:fe81:e1bb/64 scope global dynamic mngtmpaddr
valid_lft 1814364sec preferred_lft 604764sec
inet6 xxx:xx:xxxx:xxxx:201:2eff:fe81:e1bb/64 scope global dynamic mngtmpaddr
valid_lft 604764sec preferred_lft 86364sec
inet6 fe80::201:2eff:fe81:e1bb/64 scope link
valid_lft forever preferred_lft forever

You have these two:

On my side I have these:

6: tap119i0: <BROADCAST,MULTICAST,PROMISC,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast master vmbr0 state UNKNOWN group default qlen 1000
link/ether 4a:fb:65:d0:5b:b7 brd ff:ff:ff:ff:ff:ff
7: tap119i1: <BROADCAST,MULTICAST,PROMISC,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast master vmbr1 state UNKNOWN group default qlen 1000
link/ether 56:f4:de:fd:b6:07 brd ff:ff:ff:ff:ff:ff
8: tap100i0: <BROADCAST,MULTICAST,PROMISC,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast master vmbr0 state UNKNOWN group default qlen 1000
link/ether 26:a6:5b:ae:47:b1 brd ff:ff:ff:ff:ff:ff
13: veth202i0@if12: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue master vmbr0 state UP group default qlen 1000
link/ether fe:52:27:16:fa:ac brd ff:ff:ff:ff:ff:ff link-netnsid 1
17: veth203i0@if16: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue master vmbr0 state UP group default qlen 1000
link/ether fe:cf:90:fc:8b:6e brd ff:ff:ff:ff:ff:ff link-netnsid 0

The Virtual Ethernet (veth) are missing, but these are for Linux Containers, so this isn’t the issue.
My Firewall (running with VMID 119) has two NICs configured:

6: tap119i0: <BROADCAST,MULTICAST,PROMISC,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast master vmbr0 state UNKNOWN group default qlen 1000
link/ether 4a:fb:65:d0:5b:b7 brd ff:ff:ff:ff:ff:ff
7: tap119i1: <BROADCAST,MULTICAST,PROMISC,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast master vmbr1 state UNKNOWN group default qlen 1000
link/ether 56:f4:de:fd:b6:07 brd ff:ff:ff:ff:ff:ff

On Proxmox in the folder /etc/pve/qemu-server/ you should find the config files of your VMs.
VMID.conf, eg 100.conf

Send the output of the config of your NethServer…

It should look like this (Mine is OPNsense…):

bootdisk: scsi0
cores: 2
ide2: none,media=cdrom
memory: 2048
name: ABHO-OPNsense
net0: virtio=56:16:61:C4:1B:34,bridge=vmbr0
net1: virtio=46:4C:E5:A9:0C:FE,bridge=vmbr1
numa: 0
onboot: 1
ostype: other
scsi0: local-lvm:vm-119-disk-1,size=32G
scsihw: virtio-scsi-pci
smbios1: uuid=093fee83-0ba9-41bc-a126-7a2f0e195ff0
sockets: 1
startup: order=1,up=120,down=120

Andy_Wismer · February 29, 2020, 10:36am

Basically, it appears to me that Proxmox is now running correctly, both NICs are up, the Basis for the Bridge, enp2s0 and enp3s0 are up and running, and both Bridges vmbr0 and vmbr1 are shown as up and running (now).

I think the configuration temporariily “lost” the correct allocation, as vmbr1 did not exist for a while…
I assume the config-file
(Is nano /etc/pve/qemu-server/100.conf the correct VM?)
lost the entry for vmbr1…

capote · February 29, 2020, 10:39am

agent: 1
bootdisk: scsi0
cores: 2
ide2: local:iso/nethserver-7.7.1908-x86_64.iso,media=cdrom
memory: 2048
name: v-nethserver
net0: virtio=4E:FA:17:F3:3F:74,bridge=vmbr1
numa: 0
ostype: l26
scsi0: local-lvm:vm-100-disk-0,size=35G
scsihw: virtio-scsi-pci
smbios1: uuid=3d6c4331-3b5d-4a7b-83f3-4ab5baaaf0dc
sockets: 1
vmgenid: 0136bb30-ef92-4449-aaad-9525a3949974

Andy_Wismer · February 29, 2020, 10:40am

Doesn’t look like 2 NICs to me…

What does the GUI show for that VM?

capote · February 29, 2020, 10:41am

Andy_Wismer · February 29, 2020, 10:43am

Also here, there is only NIC allocated.

Maybe one allocation got lost…

But with the concept as it is, your NethServer running as firewall, and that scenarion simply NEEDS two NICs (at least!).

capote · February 29, 2020, 10:43am

I added manually

Andy_Wismer · February 29, 2020, 10:44am

I’d use virtio, especially for the LAN connection…
That gives you internally 10GB/s, an Intel e1000 will only get 1 GB/s

Also: After the installation, remove the CD (Set to none)…

capote · February 29, 2020, 10:45am

voiala…

capote · February 29, 2020, 10:48am

done.

I have to a job for my wife now…

Andy_Wismer · February 29, 2020, 10:48am

Starting to look real good!

Now, you just need to allocate it as “RED”!

I need to hit the road, got a meeting 200 km away (train trip). I’ll be back in the evening, bus you can always send a mail…

capote · February 29, 2020, 10:49am

thanks a lot, have a good trip

Andy_Wismer · February 29, 2020, 10:50am

You’re Welcome!

Let me know how things are going.

For a NethServer firewall 2 GB RAM is OK.
For more, I’d allocate 4, 8 or even 16 GB RAM, if available…

capote · February 29, 2020, 12:01pm

soooo…my frieds.
My Nethserver as Gateway is up and running. He substitutes my dedicated Nethserver well.

Thanks to all who endured the news bombing, but especially to Andy who didn’t let up until I was sitting on the horse.

After the toil of the mountain comes the toil of the plain.

Andy_Wismer · February 29, 2020, 12:32pm

Great!

capote · February 29, 2020, 12:46pm

Resume

Use Case: Build a virtual Nethserver as gateway on top of a Proxmox-System
Reason for this use case: primarily to use the flexibility of virtual machines, especially backup, snapshots for better disaster management and faster recovery of entire network servers

Initial situation :

Internet-/DSL-Router: 192.168.2.1/24

currently connected via switch to my dedicated Nethserver on Server 1
later planned to connect to enp2s0 of my Server 2 (Promox)

Server 1 : dedicated NS-Gateway 192.168.3.1/24 (GREEN/LAN) with DHCP and DNS + 192.168.2.5 (RED/WAN)
Server 1 shout be substituted by Server 2 with Promox and a virtualised Nethserver as Gateway
all Clients (Diskstation, Workstations, Laptops, Mobile Phones…),connected via switch with Server 1 (dedicated NS): 192.168.3.2-192.168.3.254)
for Installation purposes Server 2 (Promox) with enps20 (not connected) and enp3s0 connected via switch to the gateway. This server is pingable from other clients and has internet connection.

My planned Architecture :

Internet-/DSL-Router: 192.168.2.1

connected to enp2s0 of my Server 2 (Promox)
connection to Server 1 disconnected

Server 2 :

enps2s0 with 192.168.2.2 connected to my DSL-Router
enp3s0 connected to my switch (LAN-side)
virtual NS-Gateway with DHCP and DNS

all Clients (Diskstation, Workstations, Laptops, Mobile Phones…),connected via switch with Server 1 (dedicated NS): 192.168.3.2-192.168.3.254)
Server 2 (Promox) with enps20 (connected to DSl-Router) and enp3s0 (connected to the switch).

Installation of Proxmox:

Connect the server with one NIC enp3s0 to the switch
Install the Proxmox-Server with a static IP inside the LAN-IP-Range
modify the apt sources nano /etc/apt/sources.list.d/pve-enterprise.list and disable the source
# deb https://enterprise.proxmox.com/debian/pve buster pve-enterprise
modify the apt sources again nano /etc/apt/sources.list by adding
# PVE pve-no-subscription repository provided by proxmox.com,
# NOT recommended for production use
deb http://download.proxmox.com/debian/pve buster pve-no-subscription
update the Proxmox server apt update & apt full-upgrade
install ifupdown2 to avoid reboots in case of network changes: apt install ifupdown2
reboot

Ensure a well defined NIC configuration

root@proxmox:~# nano /etc/network/interfaces
auto lo
iface lo inet loopback

auto enp3s0
iface enp3s0 inet manual
#Connected to Switch

auto enp2s0
iface enp2s0 inet manual
#disconnected: planned to DSL router

auto vmbr0
iface vmbr0 inet static
address 192.168.3.200
netmask 255.255.255.0
gateway 192.168.3.1
bridge-ports enp3s0
bridge-stp off
bridge-fd 0

auto vmbr1
iface vmbr1 inet manual
bridge-ports enp2s0
bridge-stp off
bridge-fd 0

root@proxmox:~# reboot

Ensure the right configuration inside the Proxmox-GUI

Installation of Nethserver inside a new created VM

Install a new VM
Install Nethserver
install QEMU-Guest-Agent: yum install qemu-guest-agent
Update the Nethserver
Configure the Network inside Nethserver
before:

image1485×256 38.6 KB

after:

image1328×302 23.9 KB

image724×630 27.2 KB

image671×622 24.3 KB
Connect the LAN cable from enp2s0 to the router and disconnect the old Server 1 (dedicated Nethserver) if relevant
Activate DHCP on eth1

image734×72 11.1 KB
have fun