Expect(err).NotTo(HaveOccurred(),"Failed to write file %q at remote path %q with ssh client %+v",p.fileName,temporaryRemotePath,sshClient)
gomega.Expect(err).NotTo(gomega.HaveOccurred(),"Failed to write file %q at remote path %q with ssh client %+v",p.fileName,temporaryRemotePath,sshClient)
It("each node by ordering unclean reboot and ensure they function upon restart",func(){
ginkgo.It("each node by ordering unclean reboot and ensure they function upon restart",func(){
// unclean shutdown and restart
// unclean shutdown and restart
// We sleep 10 seconds to give some time for ssh command to cleanly finish before the node is shutdown.
// We sleep 10 seconds to give some time for ssh command to cleanly finish before the node is shutdown.
testReboot(f.ClientSet,"nohup sh -c 'echo 1 | sudo tee /proc/sys/kernel/sysrq && sleep 10 && echo b | sudo tee /proc/sysrq-trigger' >/dev/null 2>&1 &",nil)
testReboot(f.ClientSet,"nohup sh -c 'echo 1 | sudo tee /proc/sys/kernel/sysrq && sleep 10 && echo b | sudo tee /proc/sysrq-trigger' >/dev/null 2>&1 &",nil)
})
})
It("each node by triggering kernel panic and ensure they function upon restart",func(){
ginkgo.It("each node by triggering kernel panic and ensure they function upon restart",func(){
// kernel panic
// kernel panic
// We sleep 10 seconds to give some time for ssh command to cleanly finish before kernel panic is triggered.
// We sleep 10 seconds to give some time for ssh command to cleanly finish before kernel panic is triggered.
testReboot(f.ClientSet,"nohup sh -c 'echo 1 | sudo tee /proc/sys/kernel/sysrq && sleep 10 && echo c | sudo tee /proc/sysrq-trigger' >/dev/null 2>&1 &",nil)
testReboot(f.ClientSet,"nohup sh -c 'echo 1 | sudo tee /proc/sys/kernel/sysrq && sleep 10 && echo c | sudo tee /proc/sysrq-trigger' >/dev/null 2>&1 &",nil)
})
})
It("each node by switching off the network interface and ensure they function upon switch on",func(){
ginkgo.It("each node by switching off the network interface and ensure they function upon switch on",func(){
// switch the network interface off for a while to simulate a network outage
// switch the network interface off for a while to simulate a network outage
// We sleep 10 seconds to give some time for ssh command to cleanly finish before network is down.
// We sleep 10 seconds to give some time for ssh command to cleanly finish before network is down.
testReboot(f.ClientSet,"nohup sh -c 'sleep 10 && sudo ip link set eth0 down && sleep 120 && sudo ip link set eth0 up && (sudo dhclient || true)' >/dev/null 2>&1 &",nil)
testReboot(f.ClientSet,"nohup sh -c 'sleep 10 && sudo ip link set eth0 down && sleep 120 && sudo ip link set eth0 up && (sudo dhclient || true)' >/dev/null 2>&1 &",nil)
})
})
It("each node by dropping all inbound packets for a while and ensure they function afterwards",func(){
ginkgo.It("each node by dropping all inbound packets for a while and ensure they function afterwards",func(){
// tell the firewall to drop all inbound packets for a while
// tell the firewall to drop all inbound packets for a while
// We sleep 10 seconds to give some time for ssh command to cleanly finish before starting dropping inbound packets.
// We sleep 10 seconds to give some time for ssh command to cleanly finish before starting dropping inbound packets.
// We still accept packages send from localhost to prevent monit from restarting kubelet.
// We still accept packages send from localhost to prevent monit from restarting kubelet.