#!/bin/bash
# Copyright (c) 2021 m-privacy GmbH
#
# Last change: 09/Mar/2021, ao

date

if test "$(hostname)" != "ceph1"
then
	echo "This is not host ceph1, exiting"
	exit
fi

SLOW="$(ceph -s|grep 'has slow ops'|grep 'osd'|cut -d ',' -f 3|cut -d ' ' -f 2|head -n 1|cut -d '.' -f 2)"
if test -z "$SLOW"
then
	SLOW="$(ceph -s|grep 'have slow ops'|grep 'osd'|cut -d '[' -f 2 |cut -d ']' -f 1|tr ',' ' '|sed -e 's|osd\.||g'|cut -d ' ' -f 1)"
fi

if test -z "$SLOW"
then
	echo "All OSD fine"
else
	BLOCKED="$(ceph -s|grep 'oldest one blocked for '|head -n 1|cut -d ',' -f 2|cut -d ' ' -f 6)"
	if test "$BLOCKED" -ge 90
	then
		echo "Too old: osd.$SLOW (or more) blocked ${BLOCKED}s >= 90s"
		for i in 1 2 3 4 5
		do
			grep -q "ceph$i" /etc/hosts || continue
			for o in $SLOW
			do
				echo "ssh root@ceph$i systemctl reset-failed ceph-osd@${o}.service"
				timeout 10 ssh root@ceph$i systemctl reset-failed ceph-osd@${o}.service 2>&1|grep -v "failed because the control\|for details.\|Failed to reset failed state"
				echo "ssh root@ceph$i systemctl restart ceph-osd@${o}.service"
				timeout 10 ssh root@ceph$i systemctl restart ceph-osd@${o}.service 2>&1|grep -v "failed because the control\|for details."
			done
		done
	else
		echo "Too young: osd.$SLOW (or more) blocked ${BLOCKED}s < 90s"
	fi
fi

MDS="$(ceph -s|grep 'MDSs report slow requests')"
if test -z "$MDS"
then
	echo "All MDS fine"
else
	ACTIVE="$(ceph mds stat|cut -d '=' -f 2)"
	if test -n "$ACTIVE"
	then
		BLOCKED="$(ssh $ACTIVE "grep 'oldest blocked for >' /var/log/ceph/ceph-mds.${ACTIVE}.log|tail -n 1|cut -d '>' -f 2|cut -d '.' -f 1|tr -d ' '")"
		if test "$BLOCKED" -ge 30
		then
			echo "Too old: mds.$ACTIVE blocked ${BLOCKED}s >= 30s"
			echo "ssh root@$ACTIVE systemctl reset-failed ceph-mds@${ACTIVE}.service"
			timeout 10 ssh root@$ACTIVE systemctl reset-failed ceph-mds@${ACTIVE}.service
			echo "ssh root@$ACTIVE systemctl stop ceph-mds@${ACTIVE}.service"
			timeout 10 ssh root@$ACTIVE systemctl stop ceph-mds@${ACTIVE}.service
			echo "sleep 10"
			sleep 10
			echo "ssh root@$ACTIVE systemctl start ceph-mds@${ACTIVE}.service"
			timeout 10 ssh root@$ACTIVE systemctl start ceph-mds@${ACTIVE}.service
		else
			echo "Too young: mds.$ACTIVE blocked ${BLOCKED}s < 30s"
		fi
	else
		echo "Cannot find active MDS"
	fi
fi

MON="$(ceph -s|grep 'has slow ops'|grep 'mon'|cut -d ',' -f 3|cut -d ' ' -f 2|head -n 1|cut -d '.' -f 2)"
if test -z "$MON"
then
	echo "All MON fine"
else
	BLOCKED="$(ceph -s|grep 'oldest one blocked for '|head -n 1|cut -d ',' -f 2|cut -d ' ' -f 6)"
	if test "$BLOCKED" -ge 90
	then
		echo "Too old: mon.$SLOW (or more) blocked ${BLOCKED}s >= 90s"
		for i in 1 2 3 4 5
		do
			grep -q "ceph$i" /etc/hosts || continue
			for o in $SLOW
			do
				echo "ssh root@ceph$i systemctl reset-failed ceph-mon@${o}.service"
				timeout 10 ssh root@ceph$i systemctl reset-failed ceph-mon@${o}.service 2>&1|grep -v "failed because the control\|for details.\|Failed to reset failed state"
				echo "ssh root@ceph$i systemctl restart ceph-mon@${o}.service"
				timeout 10 ssh root@ceph$i systemctl restart ceph-mon@${o}.service 2>&1|grep -v "failed because the control\|for details."
			done
		done
	else
		echo "Too young: osd.$SLOW (or more) blocked ${BLOCKED}s < 90s"
	fi
fi

exit 0
