Merge pull request #173 from lae/develop

release v1.0.5

Author: lae

.gitignore

@@ -4,3 +4,4 @@
 *.retry
 *.code-workspace
 *.sw?
+.venv

README.adoc

@@ -36,6 +36,7 @@ Provided you have Ansible installed and are using defaults:
 [source,bash,subs="attributes"]
 ----
 ansible-galaxy install geerlingguy.postgresql davidwittman.redis {role}
+ansible-galaxy collection install community.postgresql
 ansible-playbook -i your.server.fqdn, ~/.ansible/roles/{role}/examples/playbook_single_host_deploy.yml -K
 ----
 
@@ -47,6 +48,7 @@ You can also use Vagrant, if you prefer, to bring up NetBox at `localhost:8080`:
 [source,bash,subs="attributes"]
 ----
 ansible-galaxy install geerlingguy.postgresql davidwittman.redis {role}
+ansible-galaxy collection install community.postgresql
 cd ~/.ansible/roles/{role}/
 vagrant up
 ----
@@ -67,12 +69,23 @@ endif::[]
 
 === PostgreSQL
 
-This role does not setup a PostgreSQL server (but will create a database if
-needed), so you'll need to setup a PostgreSQL server and create a database user
-separate from this role. Take a look at the _Example Playbook_ section.
+This role does not setup a PostgreSQL server (but will create a database if needed), so you'll need to setup a PostgreSQL server and create a database user separate from this role.
+Take a look at the _Example Playbook_ section.
 
-WARNING: NetBox v2.2.0+ require PostgreSQL 9.4 at the minimum, which may not be
-available in your distribution's repos. You may want to use a role for this.
+In addition, for Ansible 2.10+, you may need to install the `community.postgresql` collection.
+It is recommended to specify this in your playbook's `requirements.yml` file.
+For example:
+
+[source,yaml]
+----
+---
+collections:
+  - name: community.postgresql
+    version: 3.4.0
+----
+
+WARNING: NetBox v2.2.0+ require PostgreSQL 9.4 at the minimum, which may not be available in your distribution's repos.
+You may want to use a role for this.
 
 === Redis
 
@@ -231,6 +244,15 @@ Use this syntax if your redis is installed with sentinet architecture (multiple
 the second set of variables if you wish to split your cache database from your
 webhooks database.
 
+[source,yaml]
+----
+netbox_rqworker_processes: 1
+----
+
+Specify how many request queue workers should be started by the systemd service.
+You can leave this at the default of 1, unless you have a large number of reports,
+scripts and other background tasks.
+
 [source,yaml]
 ----
 netbox_config:
@@ -422,6 +444,15 @@ netbox_uwsgi_options: {}
 Specify extra configuration options to insert into `uwsgi.ini` here. This is
 expected to be a dictionary of key/value pairs, e.g. `buffer-size: 65535`.
 
+[source,yaml]
+netbox_uwsgi_in_venv: false
+
+Toggle `netbox_uwsgi_in_venv` to `true` if you want `uwsgi` to be installed in the same virtual environment as NetBox.
+Otherwise, it will be installed system-wide into the library path of the python version used to created the virtual environment (normal/legacy behavior).
+
+WARNING: There's a possibility that this may become the default in a later version of this role (I think after further cross-platform testing).
+See https://github.com/lae/ansible-role-netbox/issues/144[issue #144] for further details.
+
 [source,yaml]
 netbox_install_epel: true
 

defaults/main.yml

@@ -99,4 +99,6 @@ netbox_pip_constraints:
   # - 'MarkupSafe<2.1.0'
 
 netbox_keep_uwsgi_updated: false
+netbox_uwsgi_in_venv: false
 netbox_uwsgi_options: {}
+netbox_rqworker_processes: 1

handlers/main.yml

@@ -19,11 +19,7 @@
 
 - name: restart netbox-rqworker.service
   systemd:
-    name: netbox-rqworker.service
+    name: "netbox-rqworker@{{ item }}.service"
     state: restarted
     daemon_reload: true
-
-- name: reload netbox-rqworker.service
-  systemd:
-    name: netbox-rqworker.service
-    state: reloaded
+  with_sequence: count="{{ netbox_rqworker_processes }}"

molecule/default/converge.yml

@@ -27,6 +27,7 @@
         role_attr_flags: CREATEDB,NOSUPERUSER
     ## REDIS server install
     redis_bind: 127.0.0.1
+    netbox_rqworker_processes: 2
   roles:
     - geerlingguy.postgresql
     - davidwittman.redis

molecule/default/tests/test_default.py

@@ -10,7 +10,8 @@ def test_services(host):
     services = [
         "netbox.socket",
         "netbox.service",
-        "netbox-rqworker.service"
+        "netbox-rqworker@1.service"
+        "netbox-rqworker@2.service"
     ]
     for service in services:
         s = host.service(service)

tasks/deploy_netbox.yml

@@ -80,7 +80,7 @@
                py_compile.compile(f, c); os.remove(c)\""
   notify:
     - reload netbox.service
-    - reload netbox-rqworker.service
+    - restart netbox-rqworker.service
 
 - name: Generate LDAP configuration for NetBox if enabled
   template:
@@ -204,7 +204,7 @@
         app_path: "{{ netbox_current_path }}/netbox"
         virtualenv: "{{ netbox_virtualenv_path }}"
 
-    - name: Clear cached data in NetBox
+    - name: Clear cached data in NetBox < 3
       django_manage:
         command: "invalidate all"
         app_path: "{{ netbox_current_path }}/netbox"
@@ -213,14 +213,16 @@
         - netbox_stable and netbox_stable_version is version('3.0.0', '<')
           or netbox_git and _netbox_git_contains_invalidate_removed.rc != 0
 
-    - name: Clear cached data in NetBox 3.2.3+
+    - name: Clear cached data in NetBox 3.2.3 to 3.6
       django_manage:
         command: "clearcache"
         app_path: "{{ netbox_current_path }}/netbox"
         virtualenv: "{{ netbox_virtualenv_path }}"
       when:
         - netbox_stable and netbox_stable_version is version('3.2.2', '>')
-          or netbox_git and _netbox_git_contains_add_clearcache.rc == 0
+          and netbox_stable_version is version('3.7.0', '<')
+          or netbox_git and ( _netbox_git_contains_add_clearcache.rc == 0
+            and _netbox_git_contains_remove_clearcache.rc != 0 )
 
   become: true
   become_user: "{{ netbox_user }}"

tasks/install_via_git.yml

@@ -15,77 +15,91 @@
     group: "{{ netbox_group }}"
     state: directory
 
-- name: Check existence of commit 1fb67b7, fixing issue netbox#2239
-  shell: 'set -o pipefail; git log --format=%H "{{ netbox_git_version }}" | grep ^1fb67b791f1a91c624dae4a1cd256e4cf3ddbb77'
-  args:
-    chdir: "{{ netbox_git_repo_path }}"
-    executable: /bin/bash
-  register: _netbox_git_contains_issue_2239_fix
-  changed_when: false
-  failed_when: "_netbox_git_contains_issue_2239_fix.rc not in [0, 1]"
+- name: Check for presence of various commits in git history
+  block:
+    - name: Check existence of commit 1fb67b7, fixing issue netbox#2239
+      shell: 'set -o pipefail; git log --format=%H "{{ netbox_git_version }}" | grep ^1fb67b791f1a91c624dae4a1cd256e4cf3ddbb77'
+      args:
+        chdir: "{{ netbox_git_repo_path }}"
+        executable: /bin/bash
+      register: _netbox_git_contains_issue_2239_fix
+      changed_when: false
+      failed_when: "_netbox_git_contains_issue_2239_fix.rc not in [0, 1]"
 
-- name: Check existence of commit 3590ed3, renaming webhooks to tasks
-  shell: 'set -o pipefail; git log --format=%H "{{ netbox_git_version }}" | grep ^3590ed378d161dd724fad2dc73ff56da746352f8'
-  args:
-    chdir: "{{ netbox_git_repo_path }}"
-    executable: /bin/bash
-  register: _netbox_git_contains_tasks_rename
-  changed_when: false
-  failed_when: "_netbox_git_contains_tasks_rename.rc not in [0, 1]"
+    - name: Check existence of commit 3590ed3, renaming webhooks to tasks
+      shell: 'set -o pipefail; git log --format=%H "{{ netbox_git_version }}" | grep ^3590ed378d161dd724fad2dc73ff56da746352f8'
+      args:
+        chdir: "{{ netbox_git_repo_path }}"
+        executable: /bin/bash
+      register: _netbox_git_contains_tasks_rename
+      changed_when: false
+      failed_when: "_netbox_git_contains_tasks_rename.rc not in [0, 1]"
 
-- name: Check existence of commit 90dbe9b, renaming DEFAULT_TIMEOUT to RQ_DEFAULT_TIMEOUT
-  shell: 'set -o pipefail; git log --format=%H "{{ netbox_git_version }}" | grep ^90dbe9bf60ab3c72be10fd070c65c26dca543ca5'
-  args:
-    chdir: "{{ netbox_git_repo_path }}"
-    executable: /bin/bash
-  register: _netbox_git_contains_rq_timeout
-  changed_when: False
-  failed_when: "_netbox_git_contains_rq_timeout.rc not in [0, 1]"
+    - name: Check existence of commit 90dbe9b, renaming DEFAULT_TIMEOUT to RQ_DEFAULT_TIMEOUT
+      shell: 'set -o pipefail; git log --format=%H "{{ netbox_git_version }}" | grep ^90dbe9bf60ab3c72be10fd070c65c26dca543ca5'
+      args:
+        chdir: "{{ netbox_git_repo_path }}"
+        executable: /bin/bash
+      register: _netbox_git_contains_rq_timeout
+      changed_when: False
+      failed_when: "_netbox_git_contains_rq_timeout.rc not in [0, 1]"
 
-- name: Check existence of commit f560693, introducing trace_paths management command
-  shell: 'set -o pipefail; git log --format=%H "{{ netbox_git_version }}" | grep ^f560693748d1f35e79ad9d006a8a9b75ef5ae37b'
-  args:
-    chdir: "{{ netbox_git_repo_path }}"
-    executable: /bin/bash
-  register: _netbox_git_contains_trace_paths
-  changed_when: False
-  failed_when: "_netbox_git_contains_trace_paths.rc not in [0, 1]"
+    - name: Check existence of commit f560693, introducing trace_paths management command
+      shell: 'set -o pipefail; git log --format=%H "{{ netbox_git_version }}" | grep ^f560693748d1f35e79ad9d006a8a9b75ef5ae37b'
+      args:
+        chdir: "{{ netbox_git_repo_path }}"
+        executable: /bin/bash
+      register: _netbox_git_contains_trace_paths
+      changed_when: False
+      failed_when: "_netbox_git_contains_trace_paths.rc not in [0, 1]"
 
-- name: Check existence of commit e165dca, introducing mkdocs command during update
-  shell: 'set -o pipefail; git log --format=%H "{{ netbox_git_version }}" | grep ^7058d6ca5ae2901383874bcea5fedad31e165dca'
-  args:
-    chdir: "{{ netbox_git_repo_path }}"
-    executable: /bin/bash
-  register: _netbox_git_contains_mkdocs
-  changed_when: False
-  failed_when: "_netbox_git_contains_mkdocs.rc not in [0, 1]"
+    - name: Check existence of commit e165dca, introducing mkdocs command during update
+      shell: 'set -o pipefail; git log --format=%H "{{ netbox_git_version }}" | grep ^7058d6ca5ae2901383874bcea5fedad31e165dca'
+      args:
+        chdir: "{{ netbox_git_repo_path }}"
+        executable: /bin/bash
+      register: _netbox_git_contains_mkdocs
+      changed_when: False
+      failed_when: "_netbox_git_contains_mkdocs.rc not in [0, 1]"
 
-- name: Check existence of commit d87ec82, introducing nightly housekeeping command
-  shell: 'set -o pipefail; git log --format=%H "{{ netbox_git_version }}" | grep ^d87ec82fe34d92a84ee6f2a97ba0a87a53eed015'
-  args:
-    chdir: "{{ netbox_git_repo_path }}"
-    executable: /bin/bash
-  register: _netbox_git_contains_housekeeping
-  changed_when: False
-  failed_when: "_netbox_git_contains_housekeeping.rc not in [0, 1]"
+    - name: Check existence of commit d87ec82, introducing nightly housekeeping command
+      shell: 'set -o pipefail; git log --format=%H "{{ netbox_git_version }}" | grep ^d87ec82fe34d92a84ee6f2a97ba0a87a53eed015'
+      args:
+        chdir: "{{ netbox_git_repo_path }}"
+        executable: /bin/bash
+      register: _netbox_git_contains_housekeeping
+      changed_when: False
+      failed_when: "_netbox_git_contains_housekeeping.rc not in [0, 1]"
 
-- name: Check existence of commit 028c876, removing the invalidate command
-  shell: 'set -o pipefail; git log --format=%H "{{ netbox_git_version }}" | grep ^028c876bcafbaede2731f191512bbebe3f1b6a9e'
-  args:
-    chdir: "{{ netbox_git_repo_path }}"
-    executable: /bin/bash
-  register: _netbox_git_contains_invalidate_removed
-  changed_when: False
-  failed_when: "_netbox_git_contains_invalidate_removed.rc not in [0, 1]"
+    - name: Check existence of commit 028c876, removing the invalidate command
+      shell: 'set -o pipefail; git log --format=%H "{{ netbox_git_version }}" | grep ^028c876bcafbaede2731f191512bbebe3f1b6a9e'
+      args:
+        chdir: "{{ netbox_git_repo_path }}"
+        executable: /bin/bash
+      register: _netbox_git_contains_invalidate_removed
+      changed_when: False
+      failed_when: "_netbox_git_contains_invalidate_removed.rc not in [0, 1]"
 
-- name: Check existence of commit b172ae6, adding the clearcache command
-  shell: 'set -o pipefail; git log --format=%H "{{ netbox_git_version }}" | grep ^b172ae65d24f6dc161c4b609477a4547bb62a50b'
-  args:
-    chdir: "{{ netbox_git_repo_path }}"
-    executable: /bin/bash
-  register: _netbox_git_contains_add_clearcache
-  changed_when: False
-  failed_when: "_netbox_git_contains_add_clearcache.rc not in [0, 1]"
+    - name: Check existence of commit b172ae6, adding the clearcache command
+      shell: 'set -o pipefail; git log --format=%H "{{ netbox_git_version }}" | grep ^b172ae65d24f6dc161c4b609477a4547bb62a50b'
+      args:
+        chdir: "{{ netbox_git_repo_path }}"
+        executable: /bin/bash
+      register: _netbox_git_contains_add_clearcache
+      changed_when: False
+      failed_when: "_netbox_git_contains_add_clearcache.rc not in [0, 1]"
+
+    - name: Check existence of commit 2d1f882, removing the clearcache command
+      shell: 'set -o pipefail; git log --format=%H "{{ netbox_git_version }}" | grep ^2d1f88272497ca72d2e1eca8e291c04538c6810e'
+      args:
+        chdir: "{{ netbox_git_repo_path }}"
+        executable: /bin/bash
+      register: _netbox_git_contains_remove_clearcache
+      changed_when: False
+      failed_when: "_netbox_git_contains_remove_clearcache.rc not in [0, 1]"
+
+  become: true
+  become_user: "{{ netbox_user }}"
 
 - name: Archive and extract snapshot of git repository
   shell: 'set -o pipefail; git archive "{{ netbox_git_version }}" | tar -x -C "{{ netbox_git_deploy_path }}"'

tasks/load_variables.yml

@@ -59,3 +59,8 @@
   set_fact:
     _netbox_global_python: "{{ ansible_python_interpreter }}"
   when: ansible_python_interpreter is defined
+
+- name: Set the execution uwsgi command
+  set_fact:
+    netbox_uwsgi_cmd: "/usr/bin/env uwsgi"
+  when: "not netbox_uwsgi_in_venv | bool"

tasks/main.yml

@@ -42,7 +42,7 @@
     home: "{{ netbox_home }}"
 
 - name: Ensure Postgres database exists (via socket)
-  postgresql_db:
+  community.postgresql.postgresql_db:
     name: "{{ netbox_database }}"
     login_user: "{{ netbox_database_user }}"
     login_unix_socket: "{{ netbox_database_socket }}"
@@ -54,7 +54,7 @@
     - netbox_database_host is not defined
 
 - name: Ensure Postgres database exists (via TCP)
-  postgresql_db:
+  community.postgresql.postgresql_db:
     name: "{{ netbox_database }}"
     login_host: "{{ netbox_database_host }}"
     port: "{{ netbox_database_port }}"
@@ -72,6 +72,7 @@
     name: uwsgi
     state: "{{ 'latest' if netbox_keep_uwsgi_updated else 'present' }}"
     umask: "0022"
+    virtualenv: "{{ omit if not netbox_uwsgi_in_venv else netbox_virtualenv_path }}"
   environment:
     PATH: "/usr/local/bin:{{ _path }}"
   notify:
@@ -106,7 +107,7 @@
 - name: Install NetBox-rqworker service unit file
   template:
     src: netbox-rqworker.service.j2
-    dest: /lib/systemd/system/netbox-rqworker.service
+    dest: /lib/systemd/system/netbox-rqworker@.service
   notify:
     - restart netbox-rqworker.service
 
@@ -121,9 +122,10 @@
 
 - name: Start and enable netbox-rqworker.service
   systemd:
-    name: netbox-rqworker.service
+    name: "netbox-rqworker@{{ item }}.service"
     state: started
     enabled: true
+  with_sequence: count="{{ netbox_rqworker_processes }}"
 
 - name: Restore the previous Ansible Python interpreter
   set_fact: