Update cron example configs to match production (#860)

Author: mxsasha

documentation/example_configuration/opt_internetnl_etc/cron/batch_crontab

@@ -4,5 +4,9 @@
 # Batch scans come in huge numbers, so there is a need for more frequent restarts. Given tasks are acked late, and the
 # workers are stopped gracefully no tasks are losts doing this. Suppose we'll want to do 10.000 scans an hour, we'll
 # have to reset every 30 minutes or so.
-*/30 * * * *     /opt/internetnl/etc/cron/batch_restart_services.sh
+# However, in Jan 2023 we discovered that in production we only did this every 6 hours,
+# still exceeding the connection limit. The restart every 6 hrs was reflected here,
+# along with a new script that restarts based on number of Redis connections.
+0 */6 * * *     /opt/internetnl/etc/cron/batch_restart_services.sh
+*/3 * * * *     /opt/internetnl/etc/cron/restart_redis_gunicorn_on_too_many_connections.sh
 0 3 * * *        /opt/internetnl/etc/cron/restart_gunicorn.sh

documentation/example_configuration/opt_internetnl_etc/cron/restart_redis_gunicorn_on_too_many_connections.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+
+REDIS_CONS=$(netstat -antlp | grep :6379 | grep "CLOSE_WAIT" | wc -l)
+
+if [ ${REDIS_CONS} -gt 50000 ]; then
+    logger --tag redis-connections-limit-cron "Redis is having too much connections in CLOSED state. Restarting internetnl batch services"
+    for i in $(ls -1 /etc/systemd/system/internetnl-batch*.service); do /bin/systemctl restart `basename $i`; done
+fi