Allow use of multiple rqworkers (#161)

Author: p-rintz

.gitignore

@@ -4,3 +4,4 @@
 *.retry
 *.code-workspace
 *.sw?
+.venv

README.adoc

@@ -234,6 +234,15 @@ Use this syntax if your redis is installed with sentinet architecture (multiple
 the second set of variables if you wish to split your cache database from your
 webhooks database.
 
+[source,yaml]
+----
+netbox_rqworker_processes: 1
+----
+
+Specify how many request queue workers should be started by the systemd service.
+You can leave this at the default of 1, unless you have a large number of reports,
+scripts and other background tasks.
+
 [source,yaml]
 ----
 netbox_config:

defaults/main.yml

@@ -100,3 +100,4 @@ netbox_pip_constraints:
 
 netbox_keep_uwsgi_updated: false
 netbox_uwsgi_options: {}
+netbox_rqworker_processes: 1

handlers/main.yml

@@ -19,11 +19,7 @@
 
 - name: restart netbox-rqworker.service
   systemd:
-    name: netbox-rqworker.service
+    name: "netbox-rqworker@{{ item }}.service"
     state: restarted
     daemon_reload: true
-
-- name: reload netbox-rqworker.service
-  systemd:
-    name: netbox-rqworker.service
-    state: reloaded
+  with_sequence: count="{{ netbox_rqworker_processes }}"

molecule/default/converge.yml

@@ -27,6 +27,7 @@
         role_attr_flags: CREATEDB,NOSUPERUSER
     ## REDIS server install
     redis_bind: 127.0.0.1
+    netbox_rqworker_processes: 2
   roles:
     - geerlingguy.postgresql
     - davidwittman.redis

molecule/default/tests/test_default.py

@@ -10,7 +10,8 @@ def test_services(host):
     services = [
         "netbox.socket",
         "netbox.service",
-        "netbox-rqworker.service"
+        "netbox-rqworker@1.service"
+        "netbox-rqworker@2.service"
     ]
     for service in services:
         s = host.service(service)

tasks/deploy_netbox.yml

@@ -80,7 +80,7 @@
                py_compile.compile(f, c); os.remove(c)\""
   notify:
     - reload netbox.service
-    - reload netbox-rqworker.service
+    - restart netbox-rqworker.service
 
 - name: Generate LDAP configuration for NetBox if enabled
   template:

tasks/main.yml

@@ -106,7 +106,7 @@
 - name: Install NetBox-rqworker service unit file
   template:
     src: netbox-rqworker.service.j2
-    dest: /lib/systemd/system/netbox-rqworker.service
+    dest: /lib/systemd/system/netbox-rqworker@.service
   notify:
     - restart netbox-rqworker.service
 
@@ -121,9 +121,10 @@
 
 - name: Start and enable netbox-rqworker.service
   systemd:
-    name: netbox-rqworker.service
+    name: "netbox-rqworker@{{ item }}.service"
     state: started
     enabled: true
+  with_sequence: count="{{ netbox_rqworker_processes }}"
 
 - name: Restore the previous Ansible Python interpreter
   set_fact:

templates/netbox-rqworker.service.j2

@@ -1,25 +1,25 @@
 {{ ansible_managed | comment }}
 [Unit]
-Description=NetBox RQ-Worker
-Documentation=http://netbox.readthedocs.io/en/{{ 'latest' if netbox_git else 'stable' }}/installation/3-http-daemon/#supervisord-installation
-After=syslog.target
+Description=NetBox Request Queue Worker %i
+Documentation=https://docs.netbox.dev/
+After=network-online.target
+Wants=network-online.target
 
 [Service]
+Type=simple
+
 WorkingDirectory={{ netbox_shared_path }}
-ExecStart={{ netbox_virtualenv_path }}/bin/python \
-    {{ netbox_current_path }}/netbox/manage.py rqworker   
-ExecReload=/bin/kill -1 $MAINPID
-ExecStop=/bin/kill -2 $MAINPID
+ExecStart={{ netbox_virtualenv_path }}/bin/python {{ netbox_current_path }}/netbox/manage.py rqworker high default low
+
 StandardOutput=journal
 StandardError=journal
 User={{ netbox_user }}
 Group={{ netbox_group }}
 Restart=on-failure
-#SuccessExitStatus=15 17 29 30
-KillSignal=SIGQUIT
-StandardError=syslog
+RestartSec=30
+
 NotifyAccess=all
-PrivateTmp=yes
+PrivateTmp=true
 ProtectSystem=full
 DeviceAllow=/dev/null rw
 DeviceAllow=/dev/urandom r

tests/test.yml

@@ -13,7 +13,7 @@
           shell: cat /srv/netbox/shared/uwsgi.ini
           changed_when: false
         - name: NetBox rq-worker service status  # noqa 303 305
-          shell: "systemctl status netbox-rqworker.service"
+          shell: "systemctl status netbox-rqworker@1.service"
           changed_when: false
         - name: NetBox application log  # noqa 305
           shell: cat /srv/netbox/shared/application.log