Merge pull request #9 from PaperMtn/develop

Release Version 3.0.0

Author: PaperMtn

.dockerignore

@@ -0,0 +1,25 @@
+__pycache__
+*.pyc
+*.pyo
+*.pyd
+.Python
+env
+venv
+dist
+signatures
+watchman-signatures
+pip-log.txt
+pip-delete-this-directory.txt
+.tox
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.log
+.git
+.mypy_cache
+.pytest_cache
+.hypothesis
+*.csv

.github/ISSUE_TEMPLATE/feature_request.md

@@ -17,8 +17,5 @@ A clear and concise description of what you want to happen.
 A clear and concise description of any alternative solutions or features you've considered.
 
 **Additional context**
-<<<<<<< HEAD
 Add any other context or screenshots about the feature request here.
-=======
-Add any other context or screenshots about the feature request here.
->>>>>>> 500caddadff50ff27d7100a970f2971515df6f4b
+

.github/workflows/dockerpublish.yml

@@ -1,37 +1,30 @@
-name: Docker Image Publish
+name: ci
 
 on:
-  release:
-    types: [published]
+  push:
+    branches: [ master ]
 
 jobs:
-  github-cache:
+  build:
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-      - name: Cache Docker layers
-        uses: actions/cache@v2
-        with:
-          path: /tmp/.buildx-cache
-          key: ${{ runner.os }}-buildx-${{ github.sha }}
-          restore-keys: |
-            ${{ runner.os }}-buildx-
-      - name: Login to DockerHub
-        uses: docker/login-action@v1
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+      -
+        name: Login to Docker Hub
+        uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: Build and push
-        uses: docker/build-push-action@v2
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      -
+        name: Build and push
+        uses: docker/build-push-action@v4
         with:
           context: .
           file: ./Dockerfile
           push: true
-          tags: papermountain/gitlab-watchman:latest
-          cache-from: type=local,src=/tmp/.buildx-cache
-          cache-to: type=local,dest=/tmp/.buildx-cache
+          tags: ${{ secrets.DOCKERHUB_USERNAME }}/gitlab-watchman:latest

.github/workflows/github_release.yml

@@ -0,0 +1,52 @@
+name: Create Release
+
+on:
+  push:
+    branches: [ master ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [ '3.10' ]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install flake8
+          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+      - name: Lint with flake8
+        run: |
+          # stop the build if there are Python syntax errors or undefined names
+          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+      - name: Test setup & install
+        run: |
+          pip install build twine
+          python3 -m build 
+          twine check dist/*.whl
+          python3 -m pip install dist/*.whl
+      - name: Extract release notes
+        id: extract-release-notes
+        uses: ffurrer2/extract-release-notes@v1
+        with:
+          changelog_file: CHANGELOG.md
+      - name: Retrieve version
+        run: |
+          echo "TAG_NAME=$(gitlab-watchman -v | awk '{match($0, /[0-9]+\.[0-9]+\.[0-9]+/); print substr($0, RSTART, RLENGTH)}')" >> $GITHUB_OUTPUT
+        id: version
+      - name: Release
+        uses: ncipollo/release-action@v1
+        with:
+          artifacts: "dist/*.tar.gz"
+          body: ${{ steps.extract-release-notes.outputs.release_notes }}
+          tag: ${{ steps.version.outputs.TAG_NAME }}
+          name: GitLab Watchman - ${{ steps.version.outputs.TAG_NAME }}

.github/workflows/pythonpackage.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ['3.7', '3.8', '3.9', '3.10']
+        python-version: ['3.10']
 
     steps:
     - uses: actions/checkout@v2
@@ -36,38 +36,10 @@ jobs:
         flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
     - name: Test setup & install
       run: |
-        pip install build
-        python3 -m build
+        pip install build twine
+        python3 -m build 
+        twine check dist/*.whl
         python3 -m pip install dist/*.whl
-    - name: Test rules
-      run: |
-        python3 -m unittest tests/test_signatures.py
-    - name: Test run
-      run: |
-        gitlab-watchman --version
-        gitlab-watchman --help
-
-  build-windows:
-
-    runs-on: windows-latest
-    strategy:
-      matrix:
-        python-version: ['3.7', '3.8', '3.9', '3.10']
-
-    steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install build
-    - name: Test setup & install
-      run: |
-        python -m build
-        pip install --find-links=dist\ gitlab-watchman
     - name: Test run
       run: |
         gitlab-watchman --version

.github/workflows/pythonpublish.yml

@@ -4,8 +4,8 @@
 name: Upload Python Package
 
 on:
-  release:
-    types: [published]
+  push:
+    branches: [ master ]
 
 jobs:
   deploy:

.gitignore

@@ -123,3 +123,5 @@ venv.bak/
 .mypy_cache/
 .dmypy.json
 dmypy.json
+watchman-signatures/
+*.csv

CHANGELOG.md

@@ -1,4 +1,23 @@
-## 2.0.0 - 2022-04-01
+## [3.0.0] - 2023-05-15
+This major version release brings multiple updates to GitLab Watchman in usability, functionality and behind the scenes improvements.
+### Added
+- Support for centralised signatures from the [Watchman Signatures repository](https://github.com/PaperMtn/watchman-signatures)
+  - This makes it much easier to keep the signature base for all Watchman applications up to date, and to add functionality to GitLab Watchman with new signatures. New signatures are downloaded, and updates to existing signatures are applied, at runtime, meaning GitLab Watchman will always be using the most up to date signatures.
+- Major UI overhaul
+  - A lot of feedback said GitLab Watchman was hard to read. This version introduces new terminal optimised logging as a logging option, as well as JSON formatting. This formatting is now the default when running with no output option selected, and is a lot easier for humans to read. Also, colours!
+- Enumeration options added
+  - GitLab Watchman now gathers more information from an instance. Useful if your use case is more red than blue...
+    - Instance metadata output to terminal 
+    - Information on the user you are authenticated as, and the token you are using, including what permissions it has.
+    - All instance users output to CSV
+    - All instance projects output to CSV
+    - All instance groups output to CSV
+- Option choose between verbose or succinct logging when using JSON output. Default is succinct.
+- Debug logging option
+### Removed
+- Local/custom signatures - Centralised signatures mean that user-created custom signatures can't be used with GitLab Watchman for Enterprise Grid anymore. If you have made a signature you think would be good for sharing with the community, feel free to add it to the Watchman Signatures repository, so it can be used in all Watchman applications
+
+## [2.0.0] - 2022-03-22
 ### Added:
 - New scopes for finding exposed data in:
   - notes
@@ -19,7 +38,7 @@
 - The --output flag is no longer required, and therefore not supported
 
 
-## 1.4.0 - 2020-12-24
+## [1.4.0] - 2020-12-24
 ### Added:
 - Refactor of rules into directories for easier management
 - Multiprocessing implemented for searching for matches. GitLab Watchman now splits regex filtering between the cores available on the device, meaning the more cores you have, the faster searching should run.
@@ -36,7 +55,7 @@
   - Microsoft NuGet keys
 
 
-## 1.3.0 - 2020-12-12
+## [1.3.0] - 2020-12-12
 ### Added:
 - Add more information about the namespaces a project is in to logs
 - Added owner details of that namespace, for groups and users
@@ -50,7 +69,7 @@
 ### Removed:
 - Enhanced logging that includes nested information, such as namespace owners, means that CSV logging is no longer practical. CSV logging has been removed and JSON via STDOUT is now the default option.
 
-## 1.2.0 - 2020-11-16
+## [1.2.0] - 2020-11-16
 ### Added:
 - More data on namespaces added to logs
 - Better search queries for existing rules to filter out false positives
@@ -61,12 +80,12 @@
 ### Fixed:
 - Bug on outputting match string for blobs/wiki-blobs
 
-## 1.1.0 - 2020-11-14
+## [1.1.0] - 2020-11-14
 ### Fixed
 - Retry added for occasional Requests HTTPSConnectionPool error
 
 ### Added
 - Exact regex string match added to output from message searches
 
-## 1.0.0 - 2020-10-01
+## [1.0.0] - 2020-10-01
 Release

CONTRIBUTING.md

@@ -6,15 +6,6 @@ The two main areas where you can contribute are:
 - Signature files
 - Additional functionality
 
-
-## Adding new signatures
-GitLab Watchman runs using YAML signature files that are stored in the `signature` directory. They define what to search Slack for, and are the heart of the application.
-
-Instructions on how to create your own signature files can be found in `docs\signatures.md`
-
-If you do write your own signatures, please contribute them to the project by creating a pull request.
-
-
 ## Additional functionality
 You can make recommendations for new functionality via raising issues using the feature request template. Even better, you could contribute the additional functionality yourself and create a pull request for the changes to be added to a future release.
 

Dockerfile

@@ -1,20 +1,13 @@
-FROM alpine/git AS initlayer
-WORKDIR /workdir
-RUN git clone https://github.com/PaperMtn/gitlab-watchman.git
-
-FROM python:buster
-RUN addgroup --gid 1000 gitlab-watchman
-RUN useradd -u 1000 -g 1000 gitlab-watchman
-RUN mkdir /home/gitlab-watchman
-COPY --from=initlayer /workdir/gitlab-watchman /home/gitlab-watchman
-RUN chown -R gitlab-watchman: /home/gitlab-watchman
-WORKDIR /home/gitlab-watchman
-
-RUN python3 -m pip install --upgrade pip
-RUN python3 -m pip install requests build PyYAML
-RUN python3 -m build
-RUN python3 -m pip install dist/*.whl
-
-USER gitlab-watchman
-
-ENTRYPOINT ["/usr/local/bin/gitlab-watchman"]
+# syntax=docker/dockerfile:1
+
+FROM python:3.10
+COPY . /opt/gitlab-watchman
+WORKDIR /opt/gitlab-watchman
+ENV PYTHONPATH=/opt/gitlab-watchman GITLAB_WATCHMAN_TOKEN="" GITLAB_WATCHMAN_URL=""
+RUN pip3 install -r requirements.txt build && \
+    chmod -R 700 . && \
+    python3 -m build && \
+    python3 -m pip install dist/*.whl
+STOPSIGNAL SIGINT
+WORKDIR /opt/gitlab-watchman
+ENTRYPOINT ["gitlab-watchman"]