Bitbull Tech Notes - home of free minds ...

Install Kimchi and Ginger on CentOS7 KVM host

This is a nice web-ui for single kvm virt-hosts

I tried a lot of UIs, but this is so far the best opensource kvm web-ui, which is using libvirt for management.
Why is libvirt important?
It keeps your host open to integrate/migrate from/to other virt solutions, it is the one and only standard so far.
If you do not use libvirt, you have the alternative to use pure-qemu to run your VMs, which isn't really an alternative.
Proxmox is using pure qemu comands with own management solution and this is the reason that no-one ever has ever built a Proxmox management integration.
Good for Proxmox business, bad for comunity, so better go the libvirt way.

I am missing the integration of multiple KVM hosts, but hopefully this is not the latest realease of kimchi !

Thanks Kimchi Team to build this web-ui !!!

CU

Chris

yum -y install epel-release deltarpm chronyd wget 
yum makecache
yum -y update
yum install libvirt-python libvirt libvirt-daemon-config-network qemu-kvm python-ethtool sos \
          python-ipaddr nfs-utils iscsi-initiator-utils pyparted python-libguestfs libguestfs-tools novnc \
          spice-html5 python-configobj python-magic python-paramiko python-pillow virt-top

systemctl enable chronyd
systemctl restart chronyd
# firefox https://github.com/kimchi-project/kimchi/releases/latest

yum -y install http://kimchi-project.github.io/gingerbase/downloads/latest/ginger-base.el7.centos.noarch.rpm \
          http://kimchi-project.github.io/ginger/downloads/latest/ginger.el7.centos.noarch.rpm \
          http://kimchi-project.github.io/wok/downloads/latest/wok.el7.centos.noarch.rpm \
          https://github.com/kimchi-project/kimchi/releases/download/2.5.0/kimchi-2.5.0-0.el7.centos.noarch.rpm

firewall-cmd --add-service kimchid --permanent

systemctl enable wockd nginx
systemctl restart wockd nginx firewalld

Now you can reach the KVM virt UI at:

https://kvm-host:8001

 

undefined

iptables config to isolate and route networks on KVM hosts for testing

here I have isolated and nated 3 networks: 192.168.{10,15.33}.x

 

# Generated by iptables-save v1.4.21 on Sat Nov 26 12:02:53 2016
*mangle
:PREROUTING ACCEPT [2855:275135]
:INPUT ACCEPT [2238:219430]
:FORWARD ACCEPT [0:0]
:OUTPUT ACCEPT [1484:256521]
:POSTROUTING ACCEPT [1484:256521]
-A POSTROUTING -o virbr2 -p udp -m udp --dport 68 -j CHECKSUM --checksum-fill
-A POSTROUTING -o virbr1 -p udp -m udp --dport 68 -j CHECKSUM --checksum-fill
-A POSTROUTING -o virbr0 -p udp -m udp --dport 68 -j CHECKSUM --checksum-fill
-A POSTROUTING -o virbr3 -p udp -m udp --dport 68 -j CHECKSUM --checksum-fill
COMMIT
# Completed on Sat Nov 26 12:02:53 2016
# Generated by iptables-save v1.4.21 on Sat Nov 26 12:02:53 2016
*nat
:PREROUTING ACCEPT [977:101237]
:INPUT ACCEPT [513:58953]
:OUTPUT ACCEPT [120:9090]
:POSTROUTING ACCEPT [0:0]
:DOCKER - [0:0]
-A PREROUTING -m addrtype --dst-type LOCAL -j DOCKER
-A OUTPUT ! -d 127.0.0.0/8 -m addrtype --dst-type LOCAL -j DOCKER
-A POSTROUTING -o enp5s0 -j MASQUERADE
-A POSTROUTING -s 192.168.10.0/24 -d 224.0.0.0/24 -j RETURN
-A POSTROUTING -s 192.168.10.0/24 -d 255.255.255.255/32 -j RETURN
-A POSTROUTING -s 192.168.10.0/24 ! -d 192.168.10.0/24 -p tcp -j MASQUERADE --to-ports 1024-65535
-A POSTROUTING -s 192.168.10.0/24 ! -d 192.168.10.0/24 -p udp -j MASQUERADE --to-ports 1024-65535
-A POSTROUTING -s 192.168.10.0/24 ! -d 192.168.10.0/24 -j MASQUERADE
-A POSTROUTING -s 192.168.33.0/24 -d 224.0.0.0/24 -j RETURN
-A POSTROUTING -s 192.168.33.0/24 -d 255.255.255.255/32 -j RETURN
-A POSTROUTING -s 192.168.33.0/24 ! -d 192.168.33.0/24 -p tcp -j MASQUERADE --to-ports 1024-65535
-A POSTROUTING -s 192.168.33.0/24 ! -d 192.168.33.0/24 -p udp -j MASQUERADE --to-ports 1024-65535
-A POSTROUTING -s 192.168.33.0/24 ! -d 192.168.33.0/24 -j MASQUERADE
-A POSTROUTING ! -s 192.168.33.0/24 -d 192.168.33.0/24 -j MASQUERADE
-A POSTROUTING -s 192.168.15.0/24 -d 224.0.0.0/24 -j RETURN
-A POSTROUTING -s 192.168.15.0/24 -d 255.255.255.255/32 -j RETURN
-A POSTROUTING -s 192.168.15.0/24 ! -d 192.168.15.0/24 -p tcp -j MASQUERADE --to-ports 1024-65535
-A POSTROUTING -s 192.168.15.0/24 ! -d 192.168.15.0/24 -p udp -j MASQUERADE --to-ports 1024-65535
-A POSTROUTING -s 192.168.15.0/24 ! -d 192.168.15.0/24 -j MASQUERADE
-A POSTROUTING -s 172.17.0.0/16 ! -o docker0 -j MASQUERADE
-A DOCKER -i docker0 -j RETURN
COMMIT
# Completed on Sat Nov 26 12:02:53 2016
# Generated by iptables-save v1.4.21 on Sat Nov 26 12:02:53 2016
*filter
:INPUT ACCEPT [2238:219430]
:FORWARD ACCEPT [0:0]
:OUTPUT ACCEPT [1484:256521]
:DOCKER - [0:0]
:DOCKER-ISOLATION - [0:0]
-A INPUT -i virbr2 -p udp -m udp --dport 53 -j ACCEPT
-A INPUT -i virbr2 -p tcp -m tcp --dport 53 -j ACCEPT
-A INPUT -i virbr2 -p udp -m udp --dport 67 -j ACCEPT
-A INPUT -i virbr2 -p tcp -m tcp --dport 67 -j ACCEPT
-A INPUT -i virbr1 -p udp -m udp --dport 53 -j ACCEPT
-A INPUT -i virbr1 -p tcp -m tcp --dport 53 -j ACCEPT
-A INPUT -i virbr1 -p udp -m udp --dport 67 -j ACCEPT
-A INPUT -i virbr1 -p tcp -m tcp --dport 67 -j ACCEPT
-A INPUT -i virbr0 -p udp -m udp --dport 53 -j ACCEPT
-A INPUT -i virbr0 -p tcp -m tcp --dport 53 -j ACCEPT
-A INPUT -i virbr0 -p udp -m udp --dport 67 -j ACCEPT
-A INPUT -i virbr0 -p tcp -m tcp --dport 67 -j ACCEPT
-A INPUT -i virbr3 -p udp -m udp --dport 53 -j ACCEPT
-A INPUT -i virbr3 -p tcp -m tcp --dport 53 -j ACCEPT
-A INPUT -i virbr3 -p udp -m udp --dport 67 -j ACCEPT
-A INPUT -i virbr3 -p tcp -m tcp --dport 67 -j ACCEPT
-A FORWARD -d 192.168.10.0/24 -o virbr2 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
-A FORWARD -s 192.168.10.0/24 -i virbr2 -j ACCEPT
-A FORWARD -i virbr2 -o virbr2 -j ACCEPT
-A FORWARD -o virbr2 -j REJECT --reject-with icmp-port-unreachable
-A FORWARD -i virbr2 -j REJECT --reject-with icmp-port-unreachable
-A FORWARD -d 192.168.33.0/24 -o virbr1 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
-A FORWARD -s 192.168.33.0/24 -i virbr1 -j ACCEPT
-A FORWARD -i virbr1 -o virbr1 -j ACCEPT
-A FORWARD -i virbr0 -o virbr0 -j ACCEPT
-A FORWARD -o virbr0 -j REJECT --reject-with icmp-port-unreachable
-A FORWARD -i virbr0 -j REJECT --reject-with icmp-port-unreachable
-A FORWARD -d 192.168.15.0/24 -o virbr3 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
-A FORWARD -s 192.168.15.0/24 -i virbr3 -j ACCEPT
-A FORWARD -i virbr3 -o virbr3 -j ACCEPT
-A FORWARD -o virbr3 -j REJECT --reject-with icmp-port-unreachable
-A FORWARD -i virbr3 -j REJECT --reject-with icmp-port-unreachable
-A FORWARD -j DOCKER-ISOLATION
-A FORWARD -o docker0 -j DOCKER
-A FORWARD -o docker0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
-A FORWARD -i docker0 ! -o docker0 -j ACCEPT
-A FORWARD -i docker0 -o docker0 -j ACCEPT
-A OUTPUT -o virbr2 -p udp -m udp --dport 68 -j ACCEPT
-A OUTPUT -o virbr1 -p udp -m udp --dport 68 -j ACCEPT
-A OUTPUT -o virbr0 -p udp -m udp --dport 68 -j ACCEPT
-A OUTPUT -o virbr3 -p udp -m udp --dport 68 -j ACCEPT
-A DOCKER-ISOLATION -j RETURN
COMMIT
# Completed on Sat Nov 26 12:02:53 2016

enable nested KVM for AMD cpus

Enable KVM nesting

[root@clue2 ~]# cat /sys/module/kvm_amd/parameters/nested
0

[root@clue2 ~]# echo "options kvm-amd nested=1" >> /etc/modprobe.d/dist.conf

[root@clue2 ~]# rmmod kvm-amd

[root@clue2 ~]# modprobe kvm-amd

[root@clue2 ~]# modinfo kvm_amd | grep -i nested
parm:           nested:int

[root@clue2 ~]# cat /sys/module/kvm_amd/parameters/nested
1

Configure VM to use it:

[root@clue2 ~]# virsh edit ov-compute1
---modify---
<cpu mode='host-passthrough'/>
---------
[root@clue2 ~]# virsh start ov-compute1

 

External qcow2 snapshots on CentOS7 KVM

I am using a glusterfs storage with qcow2 files on it for months.
VMs are running fast, live migration works also fine.
But I was missing real live snapshot.

Internal qcow2 snapshots which are also dumping guest memory are working fine,
but they suspend during backup and it can take from few second to few minutes.

So I was searching for something like external snapshots,
but there was no way to blockcommit the delta file back to its original after backing it up.

But with recent CentOS7 and oVirt kvm repo, now it is possible!!!

  • Guest freezes during backup for less than 2 seconds (depends on your storage and system)
  • Memory is not dumped, as it is a diskonly backup
  • Backup is now possible by simply copy the qcow2 file
  • Delta file can now be committed back into original
  • Delta file can not be deleted with virsh, but there is a simple workaround, see below

I have written a prototype of backup tool, which can do cron based backup (external), restore and snapshot(internal)
http://www.bitbull.ch/dl/scripts/virsh-qcow-backup.sh

cheers, chris

[root@clue1 ~]# virsh domblklist vm10
Target     Source
------------------------------------------------
vda        /srv/vm/images/vm10.qcow2

[root@clue1 ~]# ls -l /srv/vm/images/vm10*
-rw-r--r--. 1 qemu qemu 2908291072 Jun 25 12:37 /srv/vm/images/vm10.qcow2

[root@clue1 ~]# virsh snapshot-list vm10
 Name                 Creation Time             State
------------------------------------------------------------

[root@clue1 ~]# virsh snapshot-create-as --domain vm10 snap --disk-only --atomic
Domain snapshot snap created

[root@clue1 ~]# virsh snapshot-list vm10
 Name                 Creation Time             State
------------------------------------------------------------
 snap                 2015-06-25 12:38:12 +0200 disk-snapshot

[root@clue1 ~]# virsh domblklist vm10
Target     Source
------------------------------------------------
vda        /srv/vm/images/vm10.snap

[root@clue1 ~]# ls -l /srv/vm/images/vm10*
-rw-r--r--. 1 qemu qemu 2908291072 Jun 25 12:38 /srv/vm/images/vm10.qcow2
-rw-------. 1 qemu qemu     197632 Jun 25 12:38 /srv/vm/images/vm10.snap

[root@clue1 ~]# cp /srv/vm/images/vm10.qcow2 /srv/vm/images/vm10.qcow2.bak

[root@clue1 ~]# ssh vm10 mkdir /test

[root@clue1 ~]# ssh vm10 touch /test/$(date +%H_%M_%S)

[root@clue1 ~]# ssh vm10 ls /test/
12_40_19

[root@clue1 ~]# virsh blockcommit vm10 vda --active --pivot  --verbose
Block Commit: [100 %]
Successfully pivoted

[root@clue1 ~]# virsh domblklist vm10
Target     Source
------------------------------------------------
vda        /srv/vm/images/vm10.qcow2

[root@clue1 ~]# virsh snapshot-list vm10
 Name                 Creation Time             State
------------------------------------------------------------
 snap                 2015-06-25 12:38:12 +0200 disk-snapshot

[root@clue1 ~]# virsh snapshot-delete vm10 snap --metadata
Domain snapshot snap deleted

[root@clue1 ~]# virsh snapshot-list vm10
 Name                 Creation Time             State
------------------------------------------------------------

[root@clue1 ~]# ls -l /srv/vm/images/vm10*
-rw-r--r--. 1 qemu qemu 2908356608 Jun 25 12:42 /srv/vm/images/vm10.qcow2
-rw-r--r--. 1 root root 2908291072 Jun 25 12:39 /srv/vm/images/vm10.qcow2.bak
-rw-------. 1 qemu qemu    3080192 Jun 25 12:40 /srv/vm/images/vm10.snap

[root@clue1 ~]# rm -f /srv/vm/images/vm10.snap

[root@clue1 ~]# ssh vm10 sync

[root@clue1 ~]# virsh destroy vm10
Domain vm10 destroyed

[root@clue1 ~]# virsh start vm10
Domain vm10 started

[root@clue1 ~]# ssh vm10 ls /test/
12_40_19

[root@clue1 ~]# virsh destroy vm10
Domain vm10 destroyed

[root@clue1 ~]# mv /srv/vm/images/vm10.qcow2 /srv/vm/images/vm10.qcow2.current

[root@clue1 ~]# mv /srv/vm/images/vm10.qcow2.bak /srv/vm/images/vm10.qcow2

[root@clue1 ~]# virsh start vm10
Domain vm10 started

[root@clue1 ~]# ssh vm10 ls /test/
ls: cannot access /test/: No such file or directory

[root@clue1 ~]# virsh snapshot-list vm10
 Name                 Creation Time             State
------------------------------------------------------------

[root@clue1 ~]# ls -l /srv/vm/images/vm10*
-rw-r--r--. 1 qemu qemu 2908553216 Jun 25 12:45 /srv/vm/images/vm10.qcow2
-rw-r--r--. 1 root root 2908553216 Jun 25 12:44 /srv/vm/images/vm10.qcow2.current

[root@clue1 ~]# rpm -qa | grep kvm
libvirt-daemon-kvm-1.2.8-16.el7_1.3.x86_64
qemu-kvm-ev-2.1.2-23.el7_1.3.1.x86_64
qemu-kvm-common-ev-2.1.2-23.el7_1.3.1.x86_64

[root@clue1 ~]# ls -1 /etc/yum.repos.d/*
/etc/yum.repos.d/CentOS-Base.repo
/etc/yum.repos.d/CentOS-CR.repo
/etc/yum.repos.d/CentOS-Debuginfo.repo
/etc/yum.repos.d/CentOS-fasttrack.repo
/etc/yum.repos.d/CentOS-Sources.repo
/etc/yum.repos.d/CentOS-Vault.repo
/etc/yum.repos.d/glusterfs-epel.repo
/etc/yum.repos.d/ovirt-3.5-dependencies.repo
/etc/yum.repos.d/ovirt-3.5.repo
Home