diff --git a/.github/workflows/j2cli.yml b/.github/workflows/j2cli.yml new file mode 100644 index 0000000..3d742c8 --- /dev/null +++ b/.github/workflows/j2cli.yml @@ -0,0 +1,84 @@ +# Related documentation: +# https://jacobian.org/til/github-actions-poetry/ +# https://github.com/snok/install-poetry +name: m000/j2cli +on: + push: + branches: + - "**/*" + pull_request: +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true +jobs: + test: + strategy: + fail-fast: false + matrix: + os: [ "ubuntu-latest" ] + python-version: [ "3.10", "3.11" ] + runs-on: ${{ matrix.os }} +# env: +# TOXENV: py${{ matrix.python-version }}-pyyaml6 + steps: + - name: Repository checkout + uses: actions/checkout@v4 + - name: Python ${{ matrix.python-version }} setup + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Env-Inspect1 (py${{ matrix.python-version }}) + run: (echo "\n--- env"; env | sort; echo "\n--- tox"; which tox; echo "\n--- poetry"; which poetry) || true + - name: File-Inspect1 (py${{ matrix.python-version }}) + run: find . + + - name: Install Poetry + if: steps.cached-poetry.outputs.cache-hit != 'true' + uses: snok/install-poetry@v1 + with: + version: 1.7.0 + virtualenvs-create: true + virtualenvs-in-project: true # create .venv in test directory + - name: Install Poetry development dependencies # main dependencies are handled by tox + if: steps.cached-poetry.outputs.cache-hit != 'true' + run: poetry install --no-interaction --no-root --only=dev + + - name: Env-Inspect2 (py${{ matrix.python-version }}) + run: (echo "\n--- env"; env | sort; echo "\n--- tox"; which tox; echo "\n--- poetry"; which poetry) || true + - name: File-Inspect2 (py${{ matrix.python-version }}) + run: find . + + - name: Activate Poetry development dependencies # main dependencies are handled by tox + if: steps.cached-poetry.outputs.cache-hit == 'true' + run: source .venv/bin/activate + + - name: Env-Inspect3 (py${{ matrix.python-version }}) + run: (echo "\n--- env"; env | sort; echo "\n--- tox"; which tox; echo "\n--- poetry"; which poetry) || true + - name: File-Inspect3 (py${{ matrix.python-version }}) + run: find . + + - name: Load cached tox environment + id: cached-toxenv + uses: actions/cache@v3 + with: + path: .tox/py${{ matrix.python-version }}-** + key: toxenv-py${{ matrix.python-version }}-${{ hashFiles('poetry.lock', 'tox.ini') }} + + - name: Tox tests (py${{ matrix.python-version }}) + run: poetry run tox + + - name: Send coverage reports to Codecov + uses: codecov/codecov-action@v3 + with: + token: ${{ secrets.CODECOV_TOKEN }} + #files: ./coverage1.xml,./coverage2.xml # optional + #flags: unittests # optional + #name: codecov-umbrella # optional + fail_ci_if_error: false + verbose: true + + - name: Env-Inspect4 (py${{ matrix.python-version }}) + run: (echo "\n--- env"; env | sort; echo "\n--- tox"; which tox; echo "\n--- poetry"; which poetry) || true + - name: File-Inspect4 (py${{ matrix.python-version }}) + run: find . diff --git a/.gitignore b/.gitignore index 896d753..aa1e303 100644 --- a/.gitignore +++ b/.gitignore @@ -1,13 +1,21 @@ -# ===[ APP ]=== # - -# ===[ PYTHON PACKAGE ]=== # +# Python Packaging +*.egg/ +*.egg-info/ +pyenv/ /build/ /dist/ /MANIFEST -/*.egg/ -/*.egg-info/ -# ===[ OTHER ]=== # +# Python Caches +__pycache__/ +*.py[cod] +*.pot +*.mo + +# Python Utils +.coverage +/htmlcov/ +/.tox/ # IDE Projects .idea @@ -24,21 +32,14 @@ *.DS_Store Thumbs.db -# Utils -/.tox/ -.sass-cache/ -.coverage - -# Generated -__pycache__ -*.py[cod] -*.pot -*.mo - # Runtime -/*.log -/*.pid +*.log +*.pid + +# Locals +local/ +*.local -# ===[ EXCLUDES ]=== # +# Forced Exclusions !.gitkeep !.htaccess diff --git a/.python-version b/.python-version deleted file mode 100644 index ec22fb4..0000000 --- a/.python-version +++ /dev/null @@ -1,6 +0,0 @@ -j2cli -2.7.16 -3.4.9 -3.5.6 -3.6.8 -3.7.2 diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index b0725a2..0000000 --- a/.travis.yml +++ /dev/null @@ -1,31 +0,0 @@ -os: linux -sudo: false -language: python - -matrix: - include: - - python: 2.7 - env: TOXENV=py27 - - python: 3.4 - env: TOXENV=py34 - - python: 3.5 - env: TOXENV=py35 - - python: 3.6 - env: TOXENV=py36 - - python: 3.7-dev - env: TOXENV=py37 - - python: pypy - env: TOXENV=pypy - - python: pypy3 - env: TOXENV=pypy - - {python: 3.6, env: TOXENV=py36-pyyaml5.1} - - {python: 3.6, env: TOXENV=py36-pyyaml3.13} - - {python: 3.6, env: TOXENV=py36-pyyaml3.12} - - {python: 3.6, env: TOXENV=py36-pyyaml3.11} - - {python: 3.6, env: TOXENV=py36-pyyaml3.10} -install: - - pip install tox -cache: - - pip -script: - - tox diff --git a/CHANGELOG.md b/CHANGELOG.md index ab80cc1..f662507 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,22 @@ +## 0.5.0 (2021-05-31) +* Project spun-off, renamed to jj2cli. Version bumped-up to 0.5.0 to + show parity plus a few extra features. Future changes to version will + not have any significance regarding to the parent j2cli project. +* New: Support for multiple input files, squashed in a single context prior + to rendering. +* New: Support for using *data-specs* to describe input files. + - Allows mixing/matching different input formats. + - Allows attaching input format in a specific location of the context. + - Support for list-formatted inputs via aforementioned attaching. +* New: `--ignore-missing`/`-I` flag, for ignoring non-existing input files. +* Change: `--undefined`/`-U` now allows you to set the behaviour of Jinja2 + for undefined variables. +* Change: `--fallback-format`/`-f` now sets the *fallback* format, rather than + forcing the use of a specific format. Forcing a specific format can be + achieved via *data-specs*. +* Change: Currently only Python 3.8 and 3.9 are supported. The goal is to + eventually support Python >=3.6 and Python 2.7. + ## 0.3.12 (2019-08-18) * Fix: use `env` format from stdin diff --git a/LICENSE b/LICENSE index 7a916f8..01a0786 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,7 @@ -Copyright (c) 2014, Mark Vartanyan +Copyright (c) 2019-2023, Manolis Stamatogiannakis +Copyright (c) 2014-2019, Mark Vartanyan +Copyright (c) 2012-2013, Matt Robenolt + All rights reserved. Redistribution and use in source and binary forms, with or without modification, diff --git a/Makefile b/Makefile index e4b3601..671770d 100644 --- a/Makefile +++ b/Makefile @@ -1,17 +1,24 @@ -all: +PYPACKAGE=jj2cli +TOX_QUICKTEST=py3.10-pyyaml6 +TOX_LINTTEST=lint + +.PHONY: help all clean test test-lint test-tox test-tox-quick test-pytest test-pytest-cover + +help: ##- Show this help. + @sed -e '/#\{2\}-/!d; s/\\$$//; s/:[^#\t]*/:/; s/#\{2\}- *//' $(MAKEFILE_LIST) -SHELL := /bin/bash +all: + @echo no default action # Package -.PHONY: clean clean: @rm -rf build/ dist/ *.egg-info/ README.md README.rst @pip install -e . # have to reinstall because we are using self -README.md: $(shell find j2cli/) $(wildcard misc/_doc/**) +README.md: $(shell find src/) $(wildcard misc/_doc/**) @python misc/_doc/README.py | python j2cli/__init__.py -f json -o $@ misc/_doc/README.md.j2 -.PHONY: build publish-test publish +.PHONY: build check-pyenv install-pyenv publish-test publish build: README.md @./setup.py build sdist bdist_wheel publish-test: README.md @@ -19,9 +26,34 @@ publish-test: README.md publish: README.md @twine upload dist/* +check-pyenv: +ifeq ($(VIRTUAL_ENV),) + $(error Not in a virtualenv) +else + @printf "Installing in %s.\n" "$(VIRTUAL_ENV)" +endif + +test-lint: ##- Run configured linters. + prospector + +test-tox: ##- Run all Tox tests. + tox + +test-tox-quick: ##- Run test only for the TOX_QUICKTEST profile. + tox -e $(TOX_QUICKTEST) + +test-tox-lint: ##- Run configured linters via Tox. + tox -e $(TOX_LINTTEST) + +test-pytest: + pytest + +#test-nose-cover: ## Use pytest to produce a test coverage report. + #nosetests --with-coverage --cover-package $(PYPACKAGE) + #75 $(COVERAGE_XML): .coveragerc + #76 pytest --cov-report xml:$(@) --cov=. + +test: test-lint test-tox test-nose ##- Run all linters and tests. -.PHONY: test test-tox -test: - @nosetests -test-tox: - @tox +install-pyenv: check-pyenv ##- Install to the current virtualenv. + pip install -e . diff --git a/README.md b/README.md index b34a436..4d6a5ac 100644 --- a/README.md +++ b/README.md @@ -1,138 +1,116 @@ -[![Build Status](https://travis-ci.org/kolypto/j2cli.svg)](https://travis-ci.org/kolypto/j2cli) -[![Pythons](https://img.shields.io/badge/python-2.6%20%7C%202.7%20%7C%203.4%E2%80%933.7%20%7C%20pypy-blue.svg)](.travis.yml) - -j2cli - Jinja2 Command-Line Tool -================================ - -`j2cli` is a command-line tool for templating in shell-scripts, -leveraging the [Jinja2](http://jinja.pocoo.org/docs/) library. - -Features: - -* Jinja2 templating -* INI, YAML, JSON data sources supported -* Allows the use of environment variables in templates! Hello [Docker](http://www.docker.com/) :) - -Inspired by [mattrobenolt/jinja2-cli](https://github.com/mattrobenolt/jinja2-cli) - -## Installation - -``` -pip install j2cli -``` - -To enable the YAML support with [pyyaml](http://pyyaml.org/): - -``` -pip install j2cli[yaml] -``` - -## Tutorial - -Suppose, you want to have an nginx configuration file template, `nginx.j2`: - -```jinja2 -server { - listen 80; - server_name {{ nginx.hostname }}; - - root {{ nginx.webroot }}; - index index.htm; -} -``` - -And you have a JSON file with the data, `nginx.json`: - -```json -{ - "nginx":{ - "hostname": "localhost", - "webroot": "/var/www/project" - } -} -``` - -This is how you render it into a working configuration file: - -```bash -$ j2 -f json nginx.j2 nginx.json > nginx.conf -``` - -The output is saved to `nginx.conf`: - -``` -server { - listen 80; - server_name localhost; - - root /var/www/project; - index index.htm; -} -``` - -Alternatively, you can use the `-o nginx.conf` option. - -## Tutorial with environment variables - -Suppose, you have a very simple template, `person.xml`: - -```jinja2 -{{ name }}{{ age }} -``` - -What is the easiest way to use j2 here? -Use environment variables in your bash script: - -```bash -$ export name=Andrew -$ export age=31 -$ j2 /tmp/person.xml -Andrew31 -``` - -## Using environment variables - -Even when you use yaml or json as the data source, you can always access environment variables -using the `env()` function: - -```jinja2 -Username: {{ login }} -Password: {{ env("APP_PASSWORD") }} -``` - - -## Usage - -Compile a template using INI-file data source: - - $ j2 config.j2 data.ini - -Compile using JSON data source: - - $ j2 config.j2 data.json - -Compile using YAML data source (requires PyYAML): - - $ j2 config.j2 data.yaml - -Compile using JSON data on stdin: - - $ curl http://example.com/service.json | j2 --format=json config.j2 - -Compile using environment variables (hello Docker!): - - $ j2 config.j2 - -Or even read environment variables from a file: - - $ j2 --format=env config.j2 data.env - -Or pipe it: (note that you'll have to use the "-" in this particular case): - - $ j2 --format=env config.j2 - < data.env - - -# Reference +[![Build Status](https://travis-ci.com/m000/j2cli.svg?branch=heresy-refactor)](https://travis-ci.com/m000/j2cli/tree/heresy-refactor) +[![Pythons](https://img.shields.io/badge/python%7B3.8%2C%203.9%7D-blue.svg)](.travis.yml) +# jj2cli - Juiced Jinja2 command-line tool + +jj2cli (previously `j2cli`) is a command-line tool for templating in +shell-scripts, leveraging the [Jinja2](http://jinja.pocoo.org/docs/) +library. +It supports [several formats](#supported-formats) for loading the context +used for rendering the Jinja2 templates. Loading environment variables as +rendering context is also supported. + +> **Warning** +> This branch is WIP, towards completely spinning-off the tool from +> the [upstream][j2cli]. The aim is to keep the HEAD of the branch +> usable. However, until the spin-off is complete you should expect: +> - frequent history rewrites +> - cli option changes +> - breakage if you use an older Python (<3.10) +> +> Having said that, you are welcome to use this branch and start an +> issue if you encounter any problems or have feedback. + +## jj2cli features and roadmap + +The following planned/implemented features differentiate jj2cli from +its upstreams. + +- [ ] Focus on modern Python, initially ≥3.10. This is to allow modernizing + the codebase. Support for Python ≥3.8 may be considered later, if there + are appealing reasons for that. +- [ ] Switch to more modern tooling. + * [x] [pytest][pytest] (to replace [nose][nose]) + * [ ] [ruff][ruff] (to replace [prospector][prospector]) +- [ ] Rendering of multiple templates using the same context in one go. + Rendering a couple of dozens template one-by-one is fairly slow. + This should make the tool snappier to use, but also means that + the command line interface will need to change. +- [ ] Template dependency analysis to allow better integration with tools + like [make][make]. Such tools are otherwise oblivious to Jinja2 template + inheritance/inclusion. +- [ ] Extended library of Jinja2 filters. This should allow using jj2cli + out of the box in a wider range of use cases. +- [x] Support of *context squashing* (see [below](#context-squashing)), + to eliminate the need to preprocess context data with external tools. + +## Getting started + +### Installation +```sh +# simple install +$ pip install jj2cli +# install with yaml support +$ pip install jj2cli yaml] +``` + +### Basic usage +```sh +# render config from template config.j2 +$ j2 config.j2 config.json -o config + +# render using yaml data from stdin +$ wget -O - http://example.com/config.yml | j2 --format=yml config.j2 +``` + +For an extensive list of examples, see [docs/examples.md](docs/examples.md). + +## Context data + +### Supported formats +jj2cli supports importing context from several different sources: + + * [JSON][json]: A language-independent data-serialization format, originally derived + from JavaScript. + * [YAML][yaml]: A data-serialization language, designed to be human-readable. + * [INI][ini]: Windows-style configuration files. + * env: Simple [unix-style][ini] environment variable assignments. + +For examples with each supported format, see [docs/formats.md](docs/formats.md). + +### Context squashing +One of the main strengths of jj2cli is that it is not limited to using a single +data file as context. Several data files—perhaps in different formats—can be +used to construct the rendering context. +As the contents of the data files may "overlap", jj2cli *recursively squashes* +their contents to produce the context that will be used for rendering. +The order of squashing is *from left to right*. I.e. the contents of a data file +may be overriden by any data files specified *after it* on the command line. + +Here is a simple example illustrating how context squashing works: + + * `a.json` contents: + ```json + {"a": 1, "c": {"x": 2, "y": 3}} + ``` + * `b.json` contents: + ```json + {"b": 2, "c": {"y": 4}} + ``` + * effective context when rendering with `a.json` and `b.json` (in that order): + ```json + {"a": 1, "b": 2, "c": {"x": 2, "y": 4}} + ``` + +### Loading data as a context subtree +By default, loaded data are squashed with the top-level context. However, this +may not always be desired, especially when . E.g., when you load all the environment variables +from the shell, the variables may overwri + +For this, jj2cli supports attaching the data from a +source under a variable of the top-level context. + + +## Reference `j2` accepts the following arguments: * `template`: Jinja2 template file to render @@ -146,13 +124,13 @@ Options: To import environment variables into the global scope, give it an empty string: `--import-env=`. (This will overwrite any existing variables!) * `-o outfile`: Write rendered template to a file -* `--undefined`: Allow undefined variables to be used in templates (no error will be raised) - +* `--undefined={strict, normal, debug}`: Specify the behaviour of jj2 for undefined + variables. Refer to [Jinja2 docs][jinja2-undefined] for details. * `--filters filters.py`: Load custom Jinja2 filters and tests from a Python file. Will load all top-level functions and register them as filters. This option can be used multiple times to import several files. * `--tests tests.py`: Load custom Jinja2 filters and tests from a Python file. -* `--customize custom.py`: A Python file that implements hooks to fine-tune the j2cli behavior. +* `--customize custom.py`: A Python file that implements hooks to fine-tune the jj2cli behavior. This is fairly advanced stuff, use it only if you really need to customize the way Jinja2 is initialized. See [Customization](#customization) for more info. @@ -161,247 +139,39 @@ There is some special behavior with environment variables: * When `data` is not provided (data is `-`), `--format` defaults to `env` and thus reads environment variables * When `--format=env`, it can read a special "environment variables" file made like this: `env > /tmp/file.env` -## Formats - - -### env -Data input from environment variables. - -Render directly from the current environment variable values: - - $ j2 config.j2 - -Or alternatively, read the values from a dotenv file: - -``` -NGINX_HOSTNAME=localhost -NGINX_WEBROOT=/var/www/project -NGINX_LOGS=/var/log/nginx/ -``` - -And render with: - - $ j2 config.j2 data.env - $ env | j2 --format=env config.j2 - -If you're going to pipe a dotenv file into `j2`, you'll need to use "-" as the second argument to explicitly: - - $ j2 config.j2 - < data.env - -### ini -INI data input format. - -data.ini: - -``` -[nginx] -hostname=localhost -webroot=/var/www/project -logs=/var/log/nginx/ -``` - -Usage: - - $ j2 config.j2 data.ini - $ cat data.ini | j2 --format=ini config.j2 - -### json -JSON data input format - -data.json: - -``` -{ - "nginx":{ - "hostname": "localhost", - "webroot": "/var/www/project", - "logs": "/var/log/nginx/" - } -} -``` - -Usage: - - $ j2 config.j2 data.json - $ cat data.json | j2 --format=ini config.j2 - -### yaml -YAML data input format. - -data.yaml: - -``` -nginx: - hostname: localhost - webroot: /var/www/project - logs: /var/log/nginx -``` - -Usage: - - $ j2 config.j2 data.yml - $ cat data.yml | j2 --format=yaml config.j2 - - - - -Extras -====== - -## Filters - - -### `docker_link(value, format='{addr}:{port}')` -Given a Docker Link environment variable value, format it into something else. - -This first parses a Docker Link value like this: - - DB_PORT=tcp://172.17.0.5:5432 - -Into a dict: - -```python -{ - 'proto': 'tcp', - 'addr': '172.17.0.5', - 'port': '5432' -} -``` - -And then uses `format` to format it, where the default format is '{addr}:{port}'. - -More info here: [Docker Links](https://docs.docker.com/userguide/dockerlinks/) - -### `env(varname, default=None)` -Use an environment variable's value inside your template. - -This filter is available even when your data source is something other that the environment. - -Example: - -```jinja2 -User: {{ user_login }} -Pass: {{ "USER_PASSWORD"|env }} -``` - -You can provide the default value: - -```jinja2 -Pass: {{ "USER_PASSWORD"|env("-none-") }} -``` - -For your convenience, it's also available as a function: - -```jinja2 -User: {{ user_login }} -Pass: {{ env("USER_PASSWORD") }} -``` - -Notice that there must be quotes around the environment variable name - - - - -Customization -============= - -j2cli now allows you to customize the way the application is initialized: - -* Pass additional keywords to Jinja2 environment -* Modify the context before it's used for rendering -* Register custom filters and tests - -This is done through *hooks* that you implement in a customization file in Python language. -Just plain functions at the module level. - -The following hooks are available: - -* `j2_environment_params() -> dict`: returns a `dict` of additional parameters for - [Jinja2 Environment](http://jinja.pocoo.org/docs/2.10/api/#jinja2.Environment). -* `j2_environment(env: Environment) -> Environment`: lets you customize the `Environment` object. -* `alter_context(context: dict) -> dict`: lets you modify the context variables that are going to be - used for template rendering. You can do all sorts of pre-processing here. -* `extra_filters() -> dict`: returns a `dict` with extra filters for Jinja2 -* `extra_tests() -> dict`: returns a `dict` with extra tests for Jinja2 - -All of them are optional. - -The example customization.py file for your reference: - -```python -# -# Example customize.py file for j2cli -# Contains potional hooks that modify the way j2cli is initialized - - -def j2_environment_params(): - """ Extra parameters for the Jinja2 Environment """ - # Jinja2 Environment configuration - # http://jinja.pocoo.org/docs/2.10/api/#jinja2.Environment - return dict( - # Just some examples - - # Change block start/end strings - block_start_string='<%', - block_end_string='%>', - # Change variable strings - variable_start_string='<<', - variable_end_string='>>', - # Remove whitespace around blocks - trim_blocks=True, - lstrip_blocks=True, - # Enable line statements: - # http://jinja.pocoo.org/docs/2.10/templates/#line-statements - line_statement_prefix='#', - # Keep \n at the end of a file - keep_trailing_newline=True, - # Enable custom extensions - # http://jinja.pocoo.org/docs/2.10/extensions/#jinja-extensions - extensions=('jinja2.ext.i18n',), - ) - - -def j2_environment(env): - """ Modify Jinja2 environment - - :param env: jinja2.environment.Environment - :rtype: jinja2.environment.Environment - """ - env.globals.update( - my_function=lambda v: 'my function says "{}"'.format(v) - ) - return env - - -def alter_context(context): - """ Modify the context and return it """ - # An extra variable - context['ADD'] = '127' - return context - - -def extra_filters(): - """ Declare some custom filters. - - Returns: dict(name = function) - """ - return dict( - # Example: {{ var | parentheses }} - parentheses=lambda t: '(' + t + ')', - ) - - -def extra_tests(): - """ Declare some custom tests - - Returns: dict(name = function) - """ - return dict( - # Example: {% if a|int is custom_odd %}odd{% endif %} - custom_odd=lambda n: True if (n % 2) else False - ) - -# - -``` - +## Extras + +### Filters +For convenience, jj2cli offers several additional Jinja2 filters that can be used +in your templates. These filters should help you avoid having to implement an +[advanced customization module](#advanced-customization) for many use cases. + +See [docs/filters.md](docs/filters.md) for details on the available filters. + +### Advanced customization +jj2cli offers several *hooks* that allow for more advanced customization of its +operation. This includes: + + * passing additional keywords to Jinja2 environment + * modifying the context before it's used for rendering + * registering custom filters and tests + +See [docs/advanced.md](docs/advanced.md) for details on advanced customization. + +## Credits +jj2cli is inspired by and builds on [kolypto/j2cli][j2cli] and +[mattrobenolt/jinja2-cli][jinja2-cli] tools. + +[docker]: http://www.docker.com/ +[env]: https://en.wikipedia.org/wiki/Environment_variable#Unix +[ini]: https://en.wikipedia.org/wiki/INI_file +[j2cli]: https://github.com/kolypto/j2cli +[jinja2-cli]: https://github.com/mattrobenolt/jinja2-cli +[jinja2-undefined]: https://jinja.palletsprojects.com/en/2.10.x/api/#undefined-types +[json]: https://en.wikipedia.org/wiki/JSON +[make]: https://www.gnu.org/software/make/ +[nose]: https://nose.readthedocs.io/ +[prospector]: https://prospector.landscape.io/en/master/ +[pytest]: https://docs.pytest.org/ +[ruff]: https://docs.astral.sh/ruff/ +[yaml]: https://en.wikipedia.org/wiki/YAML diff --git a/__etup.py b/__etup.py new file mode 100755 index 0000000..283cd06 --- /dev/null +++ b/__etup.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python +""" # jj2cli - Juiced Jinja2 command-line tool + +`jj2cli` (previously `j2cli`) is a command-line tool for templating in +shell-scripts, leveraging the [Jinja2](http://jinja.pocoo.org/docs/) +library. + +Features: + +* Jinja2 templating with support +* Support for data sources in various formats (ini, yaml, json, env) +* Mixing and matching data sources +* Template dependency analysis + +Inspired by [kolypto/j2cli](https://github.com/kolypto/j2cli) and +[mattrobenolt/jinja2-cli](https://github.com/mattrobenolt/jinja2-cli). +""" + +from setuptools import setup, find_packages +from sys import version_info as PYVER + + +setup( + long_description=__doc__, + long_description_content_type='text/markdown', + + packages=find_packages('src'), + package_dir={'': 'src'}, + #py_modules=[splitext(basename(path))[0] for path in glob('src/*.py')], + include_package_data=True, + + scripts=[], + entry_points={ + 'console_scripts': [ + 'j2 = jj2cli:render', # temporarily keep the old entry point + 'jj2 = jj2cli:render', + 'jj2dep = jj2cli:dependencies', + ] + }, + extras_require=dict(packages_extra), + zip_safe=False, + platforms='any', +) diff --git a/docs/advanced.md b/docs/advanced.md new file mode 100644 index 0000000..04a0e4c --- /dev/null +++ b/docs/advanced.md @@ -0,0 +1,103 @@ +Customization +============= + +j2cli now allows you to customize the way the application is initialized: + +* Pass additional keywords to Jinja2 environment +* Modify the context before it's used for rendering +* Register custom filters and tests + +This is done through *hooks* that you implement in a customization file in Python language. +Just plain functions at the module level. + +The following hooks are available: + +* `j2_environment_params() -> dict`: returns a `dict` of additional parameters for + [Jinja2 Environment](http://jinja.pocoo.org/docs/2.10/api/#jinja2.Environment). +* `j2_environment(env: Environment) -> Environment`: lets you customize the `Environment` object. +* `alter_context(context: dict) -> dict`: lets you modify the context variables that are going to be + used for template rendering. You can do all sorts of pre-processing here. +* `extra_filters() -> dict`: returns a `dict` with extra filters for Jinja2 +* `extra_tests() -> dict`: returns a `dict` with extra tests for Jinja2 + +All of them are optional. + +The example customization.py file for your reference: + +```python +# +# Example customize.py file for j2cli +# Contains potional hooks that modify the way j2cli is initialized + + +def j2_environment_params(): + """ Extra parameters for the Jinja2 Environment """ + # Jinja2 Environment configuration + # http://jinja.pocoo.org/docs/2.10/api/#jinja2.Environment + return dict( + # Just some examples + + # Change block start/end strings + block_start_string='<%', + block_end_string='%>', + # Change variable strings + variable_start_string='<<', + variable_end_string='>>', + # Remove whitespace around blocks + trim_blocks=True, + lstrip_blocks=True, + # Enable line statements: + # http://jinja.pocoo.org/docs/2.10/templates/#line-statements + line_statement_prefix='#', + # Keep \n at the end of a file + keep_trailing_newline=True, + # Enable custom extensions + # http://jinja.pocoo.org/docs/2.10/extensions/#jinja-extensions + extensions=('jinja2.ext.i18n',), + ) + + +def j2_environment(env): + """ Modify Jinja2 environment + + :param env: jinja2.environment.Environment + :rtype: jinja2.environment.Environment + """ + env.globals.update( + my_function=lambda v: 'my function says "{}"'.format(v) + ) + return env + + +def alter_context(context): + """ Modify the context and return it """ + # An extra variable + context['ADD'] = '127' + return context + + +def extra_filters(): + """ Declare some custom filters. + + Returns: dict(name = function) + """ + return dict( + # Example: {{ var | parentheses }} + parentheses=lambda t: '(' + t + ')', + ) + + +def extra_tests(): + """ Declare some custom tests + + Returns: dict(name = function) + """ + return dict( + # Example: {% if a|int is custom_odd %}odd{% endif %} + custom_odd=lambda n: True if (n % 2) else False + ) + +# + +``` + diff --git a/docs/examples.md b/docs/examples.md new file mode 100644 index 0000000..a5ca363 --- /dev/null +++ b/docs/examples.md @@ -0,0 +1,104 @@ +## Tutorial + +Suppose, you want to have an nginx configuration file template, `nginx.j2`: + +```jinja2 +server { + listen 80; + server_name {{ nginx.hostname }}; + + root {{ nginx.webroot }}; + index index.htm; +} +``` + +And you have a JSON file with the data, `nginx.json`: + +```json +{ + "nginx":{ + "hostname": "localhost", + "webroot": "/var/www/project" + } +} +``` + +This is how you render it into a working configuration file: + +```bash +$ j2 -f json nginx.j2 nginx.json > nginx.conf +``` + +The output is saved to `nginx.conf`: + +``` +server { + listen 80; + server_name localhost; + + root /var/www/project; + index index.htm; +} +``` + +Alternatively, you can use the `-o nginx.conf` option. + +## Tutorial with environment variables + +Suppose, you have a very simple template, `person.xml`: + +```jinja2 +{{ name }}{{ age }} +``` + +What is the easiest way to use j2 here? +Use environment variables in your bash script: + +```bash +$ export name=Andrew +$ export age=31 +$ j2 /tmp/person.xml +Andrew31 +``` + +## Using environment variables + +Even when you use yaml or json as the data source, you can always access environment variables +using the `env()` function: + +```jinja2 +Username: {{ login }} +Password: {{ env("APP_PASSWORD") }} +``` + + +## Usage + +Compile a template using INI-file data source: + + $ j2 config.j2 data.ini + +Compile using JSON data source: + + $ j2 config.j2 data.json + +Compile using YAML data source (requires PyYAML): + + $ j2 config.j2 data.yaml + +Compile using JSON data on stdin: + + $ curl http://example.com/service.json | j2 --format=json config.j2 + +Compile using environment variables (hello Docker!): + + $ j2 config.j2 + +Or even read environment variables from a file: + + $ j2 --format=env config.j2 data.env + +Or pipe it: (note that you'll have to use the "-" in this particular case): + + $ j2 --format=env config.j2 - < data.env + diff --git a/docs/filters.md b/docs/filters.md new file mode 100644 index 0000000..a5748e9 --- /dev/null +++ b/docs/filters.md @@ -0,0 +1,52 @@ +## Filters + + +### `docker_link(value, format='{addr}:{port}')` +Given a Docker Link environment variable value, format it into something else. + +This first parses a Docker Link value like this: + + DB_PORT=tcp://172.17.0.5:5432 + +Into a dict: + +```python +{ + 'proto': 'tcp', + 'addr': '172.17.0.5', + 'port': '5432' +} +``` + +And then uses `format` to format it, where the default format is '{addr}:{port}'. + +More info here: [Docker Links](https://docs.docker.com/userguide/dockerlinks/) + +### `env(varname, default=None)` +Use an environment variable's value inside your template. + +This filter is available even when your data source is something other that the environment. + +Example: + +```jinja2 +User: {{ user_login }} +Pass: {{ "USER_PASSWORD"|env }} +``` + +You can provide the default value: + +```jinja2 +Pass: {{ "USER_PASSWORD"|env("-none-") }} +``` + +For your convenience, it's also available as a function: + +```jinja2 +User: {{ user_login }} +Pass: {{ env("USER_PASSWORD") }} +``` + +Notice that there must be quotes around the environment variable name + + diff --git a/docs/formats.md b/docs/formats.md new file mode 100644 index 0000000..cbfe164 --- /dev/null +++ b/docs/formats.md @@ -0,0 +1,90 @@ +# Supported Formats + +Following, we show how to use the different data file formats supported +bu j2cli to render an nginx configuration file template, `nginx.j2`: + +```jinja2 +server { + listen 80; + server_name {{ nginx.hostname }}; + + root {{ nginx.webroot }}; + index index.htm; +} +``` + +## JSON + +Data file contents: +```json +{ + "nginx":{ + "hostname": "localhost", + "webroot": "/var/www/project", + "logs": "/var/log/nginx/" + } +} +``` + +Usage: + + $ j2 config.j2 data.json -o config + $ j2 -f json config.j2 - < data.json > config + + +## YAML + +Data file contents: +```yaml +nginx: + hostname: localhost + webroot: /var/www/project + logs: /var/log/nginx +``` + +Usage: + + $ j2 config.j2 data.yaml -o config + $ j2 -f yaml config.j2 - < data.yaml > config + + +## INI + +Data file contents: +```ini +[nginx] +hostname=localhost +webroot=/var/www/project +logs=/var/log/nginx/ +``` + +Usage: + + $ j2 config.j2 data.ini -o config + $ j2 -f ini config.j2 - < data.ini > config + + +## env + +### From file +Data file contents: +```sh +NGINX_HOSTNAME=localhost +NGINX_WEBROOT=/var/www/project +NGINX_LOGS=/var/log/nginx/ +``` + +Usage: + + $ j2 config.j2 data.env -o config + $ j2 -f env config.j2 - < data.env > config + + +### From shell environment variables +Render directly from the current environment variable values: + +Usage: + + $ export NGINX_HOSTNAME=localhost NGINX_WEBROOT=/var/www/project NGINX_LOGS=/var/log/nginx/ + $ env | j2 -f env config.j2 - > config + diff --git a/j2cli/__init__.py b/j2cli/__init__.py deleted file mode 100644 index bc39e44..0000000 --- a/j2cli/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -#! /usr/bin/env python - -""" j2cli main file """ -import pkg_resources - -__author__ = "Mark Vartanyan" -__email__ = "kolypto@gmail.com" -__version__ = pkg_resources.get_distribution('j2cli').version - -from j2cli.cli import main - -if __name__ == '__main__': - main() diff --git a/j2cli/cli.py b/j2cli/cli.py deleted file mode 100644 index f900627..0000000 --- a/j2cli/cli.py +++ /dev/null @@ -1,220 +0,0 @@ -import io, os, sys -import argparse -import logging -from functools import reduce - -import jinja2 -import jinja2.loaders -from . import __version__ - -import imp, inspect - -from .context import FORMATS -from .context import parse_data_spec, read_context_data2, dict_update_deep -from .extras import filters -from .extras.customize import CustomizationModule - -# available log levels, adjusted with -v at command line -LOGLEVELS = [logging.ERROR, logging.WARNING, logging.INFO, logging.DEBUG] - -# format to use for logging -LOGFORMAT = '%(levelname)s: %(message)s' - -class FilePathLoader(jinja2.BaseLoader): - """ Custom Jinja2 template loader which just loads a single template file """ - - def __init__(self, cwd, encoding='utf-8'): - self.cwd = cwd - self.encoding = encoding - - def get_source(self, environment, template): - # Path - filename = os.path.join(self.cwd, template) - - # Read - try: - with io.open(template, 'rt', encoding=self.encoding) as f: - contents = f.read() - except IOError: - raise jinja2.TemplateNotFound(template) - - # Finish - uptodate = lambda: False - return contents, filename, uptodate - - -class Jinja2TemplateRenderer(object): - """ Template renderer """ - - ENABLED_EXTENSIONS=( - 'jinja2.ext.i18n', - 'jinja2.ext.do', - 'jinja2.ext.loopcontrols', - ) - - def __init__(self, cwd, allow_undefined, no_compact=False, j2_env_params={}): - # Custom env params - j2_env_params.setdefault('keep_trailing_newline', True) - j2_env_params.setdefault('undefined', jinja2.Undefined if allow_undefined else jinja2.StrictUndefined) - j2_env_params.setdefault('trim_blocks', not no_compact) - j2_env_params.setdefault('lstrip_blocks', not no_compact) - j2_env_params.setdefault('extensions', self.ENABLED_EXTENSIONS) - j2_env_params.setdefault('loader', FilePathLoader(cwd)) - - # Environment - self._env = jinja2.Environment(**j2_env_params) - self._env.globals.update(dict( - env=filters.env - )) - - def register_filters(self, filters): - self._env.filters.update(filters) - - def register_tests(self, tests): - self._env.tests.update(tests) - - def import_filters(self, filename): - self.register_filters(self._import_functions(filename)) - - def import_tests(self, filename): - self.register_tests(self._import_functions(filename)) - - def _import_functions(self, filename): - m = imp.load_source('imported-funcs', filename) - return dict((name, func) for name, func in inspect.getmembers(m) if inspect.isfunction(func)) - - def render(self, template_path, context): - """ Render a template - :param template_path: Path to the template file - :type template_path: basestring - :param context: Template data - :type context: dict - :return: Rendered template - :rtype: basestring - """ - return self._env \ - .get_template(template_path) \ - .render(context) \ - .encode('utf-8') - - -def render_command(argv): - """ Pure render command - :param argv: Command-line arguments - :type argv: list - :return: Rendered template - :rtype: basestring - """ - formats_names = list(FORMATS.keys()) - parser = argparse.ArgumentParser( - description='Command-line interface to Jinja2 for templating in shell scripts.', - epilog='', - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument('-V', '--version', action='version', - version='j2cli {0}, Jinja2 {1}'.format(__version__, jinja2.__version__)) - parser.add_argument('-v', '--verbose', action='count', default=0, - help='Increase verbosity.') - parser.add_argument('-f', '--fallback-format', default='ini', choices=formats_names, - help='Specify fallback data format. ' - 'Used for data with no specified format and no appropriate extension.') - parser.add_argument('--filters', nargs='+', default=[], metavar='python-file', dest='filters', - help='Load top-level functions from the specified file(s) as Jinja2 filters.') - parser.add_argument('--tests', nargs='+', default=[], metavar='python-file', dest='tests', - help='Load top-level functions from the specified file(s) as Jinja2 tests.') - parser.add_argument('--customize', default=None, metavar='python-file', dest='customize', - help='Load custom j2cli behavior from a Python file.') - parser.add_argument('--no-compact', action='store_true', dest='no_compact', - help='Do not compact space around Jinja2 blocks.') - parser.add_argument('-U', '--undefined', action='store_true', dest='undefined', - help='Allow undefined variables to be used in templates (no error will be raised.)') - parser.add_argument('-o', metavar='outfile', dest='output_file', help="Output to a file instead of stdout.") - parser.add_argument('template', help='Template file to process.') - parser.add_argument('data', nargs='+', default=[], - help='Input data specification. Multiple sources in different formats can be specified. ' - 'The different sources will be squashed into a singled dict. ' - 'The format is ::. ' - 'Parts of the specification that are not needed can be ommitted. ' - 'See examples at the end of the help.') - args = parser.parse_args(argv[1:]) - logging.basicConfig(format=LOGFORMAT, level=LOGLEVELS[args.verbose % len(LOGLEVELS)]) - logging.debug("Parsed arguments: %s", args) - - # Parse data specifications - dspecs = [parse_data_spec(d, fallback_format=args.fallback_format) for d in args.data] - - # Customization - if args.customize is not None: - customize = CustomizationModule( - imp.load_source('customize-module', args.customize) - ) - else: - customize = CustomizationModule(None) - - # Read data based on specs - data = [read_context_data2(*dspec) for dspec in dspecs] - - # Squash data into a single context - context = reduce(dict_update_deep, data, {}) - - # Apply final customizations - context = customize.alter_context(context) - - # Renderer - renderer = Jinja2TemplateRenderer(os.getcwd(), args.undefined, args.no_compact, j2_env_params=customize.j2_environment_params()) - customize.j2_environment(renderer._env) - - # Filters, Tests - renderer.register_filters({ - 'docker_link': filters.docker_link, - 'env': filters.env, - }) - for fname in args.filters: - renderer.import_filters(fname) - for fname in args.tests: - renderer.import_tests(fname) - - renderer.register_filters(customize.extra_filters()) - renderer.register_tests(customize.extra_tests()) - - # Render - try: - result = renderer.render(args.template, context) - except jinja2.exceptions.UndefinedError as e: - # When there's data at stdin, tell the user they should use '-' - try: - stdin_has_data = stdin is not None and not stdin.isatty() - if args.format == 'env' and args.data == None and stdin_has_data: - extra_info = ( - "\n\n" - "If you're trying to pipe a .env file, please run me with a '-' as the data file name:\n" - "$ {cmd} {argv} -".format(cmd=os.path.basename(sys.argv[0]), argv=' '.join(sys.argv[1:])) - ) - e.args = (e.args[0] + extra_info,) + e.args[1:] - except: - # The above code is so optional that any, ANY, error, is ignored - pass - - # Proceed - raise - - # -o - if args.output_file: - with io.open(args.output_file, 'wt', encoding='utf-8') as f: - f.write(result.decode('utf-8')) - f.close() - return b'' - - # Finish - return result - - - -def main(): - """ CLI Entry point """ - try: - output = render_command(sys.argv) - except SystemExit: - return 1 - outstream = getattr(sys.stdout, 'buffer', sys.stdout) - outstream.write(output) diff --git a/j2cli/context.py b/j2cli/context.py deleted file mode 100644 index 40760c5..0000000 --- a/j2cli/context.py +++ /dev/null @@ -1,335 +0,0 @@ -import six -import os -import sys -import six -import re -import logging -import platform -import collections - -# Adjust for aliases removed in python 3.8 -try: - collectionsAbc = collections.abc -except AttributeError: - collectionsAbc = collections - -#region Parsers - -def _parse_ini(data_string): - """ INI data input format. - - data.ini: - - ``` - [nginx] - hostname=localhost - webroot=/var/www/project - logs=/var/log/nginx/ - ``` - - Usage: - - $ j2 config.j2 data.ini - $ cat data.ini | j2 --format=ini config.j2 - """ - from io import StringIO - - # Override - class MyConfigParser(ConfigParser.ConfigParser): - def as_dict(self): - """ Export as dict - :rtype: dict - """ - d = dict(self._sections) - for k in d: - d[k] = dict(self._defaults, **d[k]) - d[k].pop('__name__', None) - return d - - # Parse - ini = MyConfigParser() - ini.readfp(ini_file_io(data_string)) - - # Export - return ini.as_dict() - -def _parse_json(data_string): - """ JSON data input format - - data.json: - - ``` - { - "nginx":{ - "hostname": "localhost", - "webroot": "/var/www/project", - "logs": "/var/log/nginx/" - } - } - ``` - - Usage: - - $ j2 config.j2 data.json - $ cat data.json | j2 --format=ini config.j2 - """ - return json.loads(data_string) - -def _parse_yaml(data_string): - """ YAML data input format. - - data.yaml: - - ``` - nginx: - hostname: localhost - webroot: /var/www/project - logs: /var/log/nginx - ``` - - Usage: - - $ j2 config.j2 data.yml - $ cat data.yml | j2 --format=yaml config.j2 - """ - # Loader - try: - # PyYAML 5.1 supports FullLoader - Loader = yaml.FullLoader - except AttributeError: - # Have to use SafeLoader for older versions - Loader = yaml.SafeLoader - # Done - return yaml.load(data_string, Loader=Loader) - -def _parse_env(data_string): - """ Data input from environment variables. - - Render directly from the current environment variable values: - - $ j2 config.j2 - - Or alternatively, read the values from a dotenv file: - - ``` - NGINX_HOSTNAME=localhost - NGINX_WEBROOT=/var/www/project - NGINX_LOGS=/var/log/nginx/ - ``` - - And render with: - - $ j2 config.j2 data.env - $ env | j2 --format=env config.j2 - - If you're going to pipe a dotenv file into `j2`, you'll need to use "-" as the second argument to explicitly: - - $ j2 config.j2 - < data.env - """ - # Parse - if isinstance(data_string, six.string_types): - data = filter( - lambda l: len(l) == 2 , - ( - list(map( - str.strip, - line.split('=', 1) - )) - for line in data_string.split("\n")) - ) - else: - data = data_string - - # Finish - return data - - -FORMATS = { - 'ini': _parse_ini, - 'json': _parse_json, - 'yaml': _parse_yaml, - 'env': _parse_env -} - -FORMATS_ALIASES = dict(zip(FORMATS.keys(), FORMATS.keys())) -FORMATS_ALIASES.update({ - 'yml': 'yaml', -}) - -#endregion - - - -#region Imports - -# JSON: simplejson | json -try: - import simplejson as json -except ImportError: - try: - import json - except ImportError: - del FORMATS['json'] - -# INI: Python 2 | Python 3 -try: - import ConfigParser - from io import BytesIO as ini_file_io -except ImportError: - import configparser as ConfigParser - from io import StringIO as ini_file_io - -# YAML -try: - import yaml -except ImportError: - del FORMATS['yaml'] - -#endregion - -def dict_update_deep(d, u): - """ Performs a deep update of d with data from u. - :param d: Dictionary to be updated. - :type dict: dict - :param u: Dictionary with updates to be applied. - :type dict: dict - :return: Updated version of d. - :rtype: dict - """ - for k, v in six.iteritems(u): - dv = d.get(k, {}) - if not isinstance(dv, collectionsAbc.Mapping): - d[k] = v - elif isinstance(v, collectionsAbc.Mapping): - d[k] = dict_update_deep(dv, v) - else: - d[k] = v - return d - -def parse_data_spec(dspec, fallback_format='ini'): - """ Parse a data file specification. - :param dspec: Data file specification in format [:][:]. - :type dspec: str - :param fallback_format: Format to fallback to if no format is set/guessed. - :type fallback_format: str - :return: (location, ctx_dest, format) - :rtype: tuple - """ - source = ctx_dst = fmt = None - - ### set fmt ########################################### - # manually specified format - if fmt is None: - left, delim, right = dspec.rpartition(':') - if left != '' and right in FORMATS_ALIASES: - source = left - fmt = FORMATS_ALIASES[right] - # guess format by extension - if fmt is None or right == '?': - left, delim, right = dspec.rpartition('.') - if left != '' and right in FORMATS_ALIASES: - source = dspec - fmt = FORMATS_ALIASES[right] - # use fallback format - if fmt is None: - source = dspec - fmt = FORMATS_ALIASES[fallback_format] - - ### set ctx_dst ####################################### - left, delim, right = source.rpartition(':') - if platform.system() == 'Windows' and re.match(r'^[a-z]$', left, re.I): - # windows path (e.g. 'c:\foo.json') -- ignore split - pass - elif left != '' and right != '': - # normal case (e.g. '/data/foo.json:dst') - source = left - ctx_dst = right - elif left != '' and right == '': - # empty ctx_dst (e.g. '/data/foo:1.json:) -- used when source contains ':' - source = left - else: - # no ctx_dst specified - pass - - ### return ############################################ - return (source, ctx_dst, fmt) - -def read_context_data2(source, ctx_dst, fmt): - """ Read context data into a dictionary - :param source: Source file to read from. - Use '-' for stdin, None to read environment (requires fmt == 'env'.) - :type source: str|None - :param ctx_dst: Variable name that will contain the loaded data in the returned dict. - If None, data are loaded to the top-level of the dict. - :type ctx_dst: str|None - :param fmt: Data format of the loaded data. - :type fmt: str - :return: Dictionary with the context data. - :rtype: dict - """ - logging.debug("Reading data: source=%s, ctx_dst=%s, fmt=%s", source, ctx_dst, fmt) - - # Special case: environment variables - if source == '-': - # read data from stdin - data = sys.stdin.read() - elif source is not None: - # read data from file - with open(source, 'r') as sourcef: - data = sourcef.read() - else: - data = None - - if data is None and fmt == env: - # load environment to context dict - if sys.version_info[0] > 2: - context = os.environ.copy() - else: - # python2: encode environment variables as unicode - context = dict((k.decode('utf-8'), v.decode('utf-8')) for k, v in os.environ.items()) - elif data is not None: - # parse data to context dict - context = FORMATS[fmt](data) - else: - # this shouldn't have happened - logging.error("Can't read data in %s format from %s.", fmt, source) - sys.exit(1) - - if ctx_dst is None: - return context - else: - return {ctx_dst: context} - -def read_context_data(format, f, environ, import_env=None): - """ Read context data into a dictionary - :param format: Data format - :type format: str - :param f: Data file stream, or None (for env) - :type f: file|None - :param import_env: Variable name, if any, that will contain environment variables of the template. - :type import_env: bool|None - :return: Dictionary with the context data - :rtype: dict - """ - - # Special case: environment variables - if format == 'env' and f is None: - return _parse_env(environ) - - # Read data string stream - data_string = f.read() - - # Parse it - if format not in FORMATS: - raise ValueError('{0} format unavailable'.format(format)) - context = FORMATS[format](data_string) - - # Import environment - if import_env is not None: - if import_env == '': - context.update(environ) - else: - context[import_env] = environ - - # Done - return context diff --git a/j2cli/extras/__init__.py b/j2cli/extras/__init__.py deleted file mode 100644 index 8b2e53c..0000000 --- a/j2cli/extras/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from . import filters diff --git a/j2cli/extras/filters.py b/j2cli/extras/filters.py deleted file mode 100644 index 2890ec4..0000000 --- a/j2cli/extras/filters.py +++ /dev/null @@ -1,79 +0,0 @@ -""" Custom Jinja2 filters """ -import os - -from jinja2 import is_undefined -import re - - -def docker_link(value, format='{addr}:{port}'): - """ Given a Docker Link environment variable value, format it into something else. - - This first parses a Docker Link value like this: - - DB_PORT=tcp://172.17.0.5:5432 - - Into a dict: - - ```python - { - 'proto': 'tcp', - 'addr': '172.17.0.5', - 'port': '5432' - } - ``` - - And then uses `format` to format it, where the default format is '{addr}:{port}'. - - More info here: [Docker Links](https://docs.docker.com/userguide/dockerlinks/) - - :param value: Docker link (from an environment variable) - :param format: The format to apply. Supported placeholders: `{proto}`, `{addr}`, `{port}` - :return: Formatted string - """ - # pass undefined values on down the pipeline - if is_undefined(value): - return value - - # Parse the value - m = re.match(r'(?P.+)://' r'(?P.+):' r'(?P.+)$', value) - if not m: - raise ValueError('The provided value does not seems to be a Docker link: {0}'.format(value)) - d = m.groupdict() - - # Format - return format.format(**d) - - -def env(varname, default=None): - """ Use an environment variable's value inside your template. - - This filter is available even when your data source is something other that the environment. - - Example: - - ```jinja2 - User: {{ user_login }} - Pass: {{ "USER_PASSWORD"|env }} - ``` - - You can provide the default value: - - ```jinja2 - Pass: {{ "USER_PASSWORD"|env("-none-") }} - ``` - - For your convenience, it's also available as a function: - - ```jinja2 - User: {{ user_login }} - Pass: {{ env("USER_PASSWORD") }} - ``` - - Notice that there must be quotes around the environment variable name - """ - if default is not None: - # With the default, there's never an error - return os.getenv(varname, default) - else: - # Raise KeyError when not provided - return os.environ[varname] diff --git a/misc/_doc/README.md.j2 b/misc/_doc/README.md.j2 deleted file mode 100644 index d068d7f..0000000 --- a/misc/_doc/README.md.j2 +++ /dev/null @@ -1,214 +0,0 @@ -[![Build Status](https://travis-ci.org/kolypto/j2cli.svg)](https://travis-ci.org/kolypto/j2cli) -[![Pythons](https://img.shields.io/badge/python-2.6%20%7C%202.7%20%7C%203.4%E2%80%933.7%20%7C%20pypy-blue.svg)](.travis.yml) - -j2cli - Jinja2 Command-Line Tool -================================ - -`j2cli` is a command-line tool for templating in shell-scripts, -leveraging the [Jinja2](http://jinja.pocoo.org/docs/) library. - -Features: - -* Jinja2 templating -* INI, YAML, JSON data sources supported -* Allows the use of environment variables in templates! Hello [Docker](http://www.docker.com/) :) - -Inspired by [mattrobenolt/jinja2-cli](https://github.com/mattrobenolt/jinja2-cli) - -## Installation - -``` -pip install j2cli -``` - -To enable the YAML support with [pyyaml](http://pyyaml.org/): - -``` -pip install j2cli[yaml] -``` - -## Tutorial - -Suppose, you want to have an nginx configuration file template, `nginx.j2`: - -{% raw %}```jinja2 -server { - listen 80; - server_name {{ nginx.hostname }}; - - root {{ nginx.webroot }}; - index index.htm; -} -```{% endraw %} - -And you have a JSON file with the data, `nginx.json`: - -```json -{ - "nginx":{ - "hostname": "localhost", - "webroot": "/var/www/project" - } -} -``` - -This is how you render it into a working configuration file: - -```bash -$ j2 -f json nginx.j2 nginx.json > nginx.conf -``` - -The output is saved to `nginx.conf`: - -``` -server { - listen 80; - server_name localhost; - - root /var/www/project; - index index.htm; -} -``` - -Alternatively, you can use the `-o nginx.conf` option. - -## Tutorial with environment variables - -Suppose, you have a very simple template, `person.xml`: - -{% raw %}```jinja2 -{{ name }}{{ age }} -```{% endraw %} - -What is the easiest way to use j2 here? -Use environment variables in your bash script: - -```bash -$ export name=Andrew -$ export age=31 -$ j2 /tmp/person.xml -Andrew31 -``` - -## Using environment variables - -Even when you use yaml or json as the data source, you can always access environment variables -using the `env()` function: - -{% raw %}```jinja2 -Username: {{ login }} -Password: {{ env("APP_PASSWORD") }} -```{% endraw %} - - -## Usage - -Compile a template using INI-file data source: - - $ j2 config.j2 data.ini - -Compile using JSON data source: - - $ j2 config.j2 data.json - -Compile using YAML data source (requires PyYAML): - - $ j2 config.j2 data.yaml - -Compile using JSON data on stdin: - - $ curl http://example.com/service.json | j2 --format=json config.j2 - -Compile using environment variables (hello Docker!): - - $ j2 config.j2 - -Or even read environment variables from a file: - - $ j2 --format=env config.j2 data.env - -Or pipe it: (note that you'll have to use the "-" in this particular case): - - $ j2 --format=env config.j2 - < data.env - - -# Reference -`j2` accepts the following arguments: - -* `template`: Jinja2 template file to render -* `data`: (optional) path to the data used for rendering. - The default is `-`: use stdin. Specify it explicitly when using env! - -Options: - -* `--format, -f`: format for the data file. The default is `?`: guess from file extension. -* `--import-env VAR, -e EVAR`: import all environment variables into the template as `VAR`. - To import environment variables into the global scope, give it an empty string: `--import-env=`. - (This will overwrite any existing variables!) -* `-o outfile`: Write rendered template to a file -* `--undefined`: Allow undefined variables to be used in templates (no error will be raised) - -* `--filters filters.py`: Load custom Jinja2 filters and tests from a Python file. - Will load all top-level functions and register them as filters. - This option can be used multiple times to import several files. -* `--tests tests.py`: Load custom Jinja2 filters and tests from a Python file. -* `--customize custom.py`: A Python file that implements hooks to fine-tune the j2cli behavior. - This is fairly advanced stuff, use it only if you really need to customize the way Jinja2 is initialized. - See [Customization](#customization) for more info. - -There is some special behavior with environment variables: - -* When `data` is not provided (data is `-`), `--format` defaults to `env` and thus reads environment variables -* When `--format=env`, it can read a special "environment variables" file made like this: `env > /tmp/file.env` - -## Formats - -{% for name, format in formats|dictsort() %} -### {{ name }} -{{ format.doc }} -{% endfor %} - - - -Extras -====== - -## Filters - -{% for name, filter in extras.filters|dictsort() %} -### `{{ filter.qsignature }}` -{{ filter.doc }} -{% endfor %} - - - -Customization -============= - -j2cli now allows you to customize the way the application is initialized: - -* Pass additional keywords to Jinja2 environment -* Modify the context before it's used for rendering -* Register custom filters and tests - -This is done through *hooks* that you implement in a customization file in Python language. -Just plain functions at the module level. - -The following hooks are available: - -* `j2_environment_params() -> dict`: returns a `dict` of additional parameters for - [Jinja2 Environment](http://jinja.pocoo.org/docs/2.10/api/#jinja2.Environment). -* `j2_environment(env: Environment) -> Environment`: lets you customize the `Environment` object. -* `alter_context(context: dict) -> dict`: lets you modify the context variables that are going to be - used for template rendering. You can do all sorts of pre-processing here. -* `extra_filters() -> dict`: returns a `dict` with extra filters for Jinja2 -* `extra_tests() -> dict`: returns a `dict` with extra tests for Jinja2 - -All of them are optional. - -The example customization.py file for your reference: - -```python -{% include "tests/resources/customize.py" %} -``` - diff --git a/misc/_doc/README.py b/misc/_doc/README.py deleted file mode 100755 index c81d7d2..0000000 --- a/misc/_doc/README.py +++ /dev/null @@ -1,26 +0,0 @@ -#! /usr/bin/env python - -import json -import inspect -from exdoc import doc, getmembers - -import j2cli -import j2cli.context -import j2cli.extras.filters - - -README = { - 'formats': { - name: doc(f) - for name, f in j2cli.context.FORMATS.items() - }, - 'extras': { - 'filters': {k: doc(v) - for k, v in getmembers(j2cli.extras.filters) - if inspect.isfunction(v) and inspect.getmodule(v) is j2cli.extras.filters} - } -} - -assert 'yaml' in README['formats'], 'Looks like the YAML library is not installed!' - -print(json.dumps(README)) diff --git a/poetry.lock b/poetry.lock new file mode 100644 index 0000000..d1731d8 --- /dev/null +++ b/poetry.lock @@ -0,0 +1,376 @@ +# This file is automatically @generated by Poetry 1.7.0 and should not be changed by hand. + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "chardet" +version = "5.2.0" +description = "Universal encoding detector for Python 3" +optional = false +python-versions = ">=3.7" +files = [ + {file = "chardet-5.2.0-py3-none-any.whl", hash = "sha256:e1cf59446890a00105fe7b7912492ea04b6e6f06d4b742b2c788469e34c82970"}, + {file = "chardet-5.2.0.tar.gz", hash = "sha256:1b3b6ff479a8c414bc3fa2c0852995695c4a026dcd6d0633b2dd092ca39c1cf7"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "distlib" +version = "0.3.7" +description = "Distribution utilities" +optional = false +python-versions = "*" +files = [ + {file = "distlib-0.3.7-py2.py3-none-any.whl", hash = "sha256:2e24928bc811348f0feb63014e97aaae3037f2cf48712d51ae61df7fd6075057"}, + {file = "distlib-0.3.7.tar.gz", hash = "sha256:9dafe54b34a028eafd95039d5e5d4851a13734540f1331060d31c9916e7147a8"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.1.3" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.1.3-py3-none-any.whl", hash = "sha256:343280667a4585d195ca1cf9cef84a4e178c4b6cf2274caef9859782b567d5e3"}, + {file = "exceptiongroup-1.1.3.tar.gz", hash = "sha256:097acd85d473d75af5bb98e41b61ff7fe35efe6675e4f9370ec6ec5126d160e9"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "filelock" +version = "3.13.1" +description = "A platform independent file lock." +optional = false +python-versions = ">=3.8" +files = [ + {file = "filelock-3.13.1-py3-none-any.whl", hash = "sha256:57dbda9b35157b05fb3e58ee91448612eb674172fab98ee235ccb0b5bee19a1c"}, + {file = "filelock-3.13.1.tar.gz", hash = "sha256:521f5f56c50f8426f5e03ad3b281b490a87ef15bc6c526f168290f0c7148d44e"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.24)"] +testing = ["covdefaults (>=2.3)", "coverage (>=7.3.2)", "diff-cover (>=8)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)", "pytest-timeout (>=2.2)"] +typing = ["typing-extensions (>=4.8)"] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "jinja2" +version = "3.1.2" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, + {file = "Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "markupsafe" +version = "2.1.3" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cd0f502fe016460680cd20aaa5a76d241d6f35a1c3350c474bac1273803893fa"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e09031c87a1e51556fdcb46e5bd4f59dfb743061cf93c4d6831bf894f125eb57"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68e78619a61ecf91e76aa3e6e8e33fc4894a2bebe93410754bd28fce0a8a4f9f"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:65c1a9bcdadc6c28eecee2c119465aebff8f7a584dd719facdd9e825ec61ab52"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:525808b8019e36eb524b8c68acdd63a37e75714eac50e988180b169d64480a00"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:962f82a3086483f5e5f64dbad880d31038b698494799b097bc59c2edf392fce6"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:aa7bd130efab1c280bed0f45501b7c8795f9fdbeb02e965371bbef3523627779"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c9c804664ebe8f83a211cace637506669e7890fec1b4195b505c214e50dd4eb7"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-win32.whl", hash = "sha256:10bbfe99883db80bdbaff2dcf681dfc6533a614f700da1287707e8a5d78a8431"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-win_amd64.whl", hash = "sha256:1577735524cdad32f9f694208aa75e422adba74f1baee7551620e43a3141f559"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ad9e82fb8f09ade1c3e1b996a6337afac2b8b9e365f926f5a61aacc71adc5b3c"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3c0fae6c3be832a0a0473ac912810b2877c8cb9d76ca48de1ed31e1c68386575"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b076b6226fb84157e3f7c971a47ff3a679d837cf338547532ab866c57930dbee"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bfce63a9e7834b12b87c64d6b155fdd9b3b96191b6bd334bf37db7ff1fe457f2"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:338ae27d6b8745585f87218a3f23f1512dbf52c26c28e322dbe54bcede54ccb9"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e4dd52d80b8c83fdce44e12478ad2e85c64ea965e75d66dbeafb0a3e77308fcc"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:df0be2b576a7abbf737b1575f048c23fb1d769f267ec4358296f31c2479db8f9"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-win32.whl", hash = "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca379055a47383d02a5400cb0d110cef0a776fc644cda797db0c5696cfd7e18e"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:b7ff0f54cb4ff66dd38bebd335a38e2c22c41a8ee45aa608efc890ac3e3931bc"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c011a4149cfbcf9f03994ec2edffcb8b1dc2d2aede7ca243746df97a5d41ce48"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:56d9f2ecac662ca1611d183feb03a3fa4406469dafe241673d521dd5ae92a155"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-win32.whl", hash = "sha256:8758846a7e80910096950b67071243da3e5a20ed2546e6392603c096778d48e0"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-win_amd64.whl", hash = "sha256:787003c0ddb00500e49a10f2844fac87aa6ce977b90b0feaaf9de23c22508b24"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:2ef12179d3a291be237280175b542c07a36e7f60718296278d8593d21ca937d4"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2c1b19b3aaacc6e57b7e25710ff571c24d6c3613a45e905b1fde04d691b98ee0"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8afafd99945ead6e075b973fefa56379c5b5c53fd8937dad92c662da5d8fd5ee"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8c41976a29d078bb235fea9b2ecd3da465df42a562910f9022f1a03107bd02be"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d080e0a5eb2529460b30190fcfcc4199bd7f827663f858a226a81bc27beaa97e"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:69c0f17e9f5a7afdf2cc9fb2d1ce6aabdb3bafb7f38017c0b77862bcec2bbad8"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:504b320cd4b7eff6f968eddf81127112db685e81f7e36e75f9f84f0df46041c3"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:42de32b22b6b804f42c5d98be4f7e5e977ecdd9ee9b660fda1a3edf03b11792d"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-win32.whl", hash = "sha256:ceb01949af7121f9fc39f7d27f91be8546f3fb112c608bc4029aef0bab86a2a5"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-win_amd64.whl", hash = "sha256:1b40069d487e7edb2676d3fbdb2b0829ffa2cd63a2ec26c4938b2d34391b4ecc"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:8023faf4e01efadfa183e863fefde0046de576c6f14659e8782065bcece22198"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6b2b56950d93e41f33b4223ead100ea0fe11f8e6ee5f641eb753ce4b77a7042b"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9dcdfd0eaf283af041973bff14a2e143b8bd64e069f4c383416ecd79a81aab58"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:05fb21170423db021895e1ea1e1f3ab3adb85d1c2333cbc2310f2a26bc77272e"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282c2cb35b5b673bbcadb33a585408104df04f14b2d9b01d4c345a3b92861c2c"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ab4a0df41e7c16a1392727727e7998a467472d0ad65f3ad5e6e765015df08636"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7ef3cb2ebbf91e330e3bb937efada0edd9003683db6b57bb108c4001f37a02ea"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:0a4e4a1aff6c7ac4cd55792abf96c915634c2b97e3cc1c7129578aa68ebd754e"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-win32.whl", hash = "sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-win_amd64.whl", hash = "sha256:3fd4abcb888d15a94f32b75d8fd18ee162ca0c064f35b11134be77050296d6ba"}, + {file = "MarkupSafe-2.1.3.tar.gz", hash = "sha256:af598ed32d6ae86f1b747b82783958b1a4ab8f617b06fe68795c7f026abbdcad"}, +] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "platformdirs" +version = "3.11.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.7" +files = [ + {file = "platformdirs-3.11.0-py3-none-any.whl", hash = "sha256:e9d171d00af68be50e9202731309c4e658fd8bc76f55c11c7dd760d023bda68e"}, + {file = "platformdirs-3.11.0.tar.gz", hash = "sha256:cf8ee52a3afdb965072dcc652433e0c7e3e40cf5ea1477cd4b3b1d2eb75495b3"}, +] + +[package.extras] +docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.1)", "sphinx-autodoc-typehints (>=1.24)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4)", "pytest-cov (>=4.1)", "pytest-mock (>=3.11.1)"] + +[[package]] +name = "pluggy" +version = "1.3.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.3.0-py3-none-any.whl", hash = "sha256:d89c696a773f8bd377d18e5ecda92b7a3793cbe66c87060a6fb58c7b6e1061f7"}, + {file = "pluggy-1.3.0.tar.gz", hash = "sha256:cf61ae8f126ac6f7c451172cf30e3e43d3ca77615509771b3a984a0730651e12"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "pyproject-api" +version = "1.6.1" +description = "API to interact with the python pyproject.toml based projects" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyproject_api-1.6.1-py3-none-any.whl", hash = "sha256:4c0116d60476b0786c88692cf4e325a9814965e2469c5998b830bba16b183675"}, + {file = "pyproject_api-1.6.1.tar.gz", hash = "sha256:1817dc018adc0d1ff9ca1ed8c60e1623d5aaca40814b953af14a9cf9a5cae538"}, +] + +[package.dependencies] +packaging = ">=23.1" +tomli = {version = ">=2.0.1", markers = "python_version < \"3.11\""} + +[package.extras] +docs = ["furo (>=2023.8.19)", "sphinx (<7.2)", "sphinx-autodoc-typehints (>=1.24)"] +testing = ["covdefaults (>=2.3)", "pytest (>=7.4)", "pytest-cov (>=4.1)", "pytest-mock (>=3.11.1)", "setuptools (>=68.1.2)", "wheel (>=0.41.2)"] + +[[package]] +name = "pytest" +version = "7.4.3" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pytest-7.4.3-py3-none-any.whl", hash = "sha256:0d009c083ea859a71b76adf7c1d502e4bc170b80a8ef002da5806527b9591fac"}, + {file = "pytest-7.4.3.tar.gz", hash = "sha256:d989d136982de4e3b29dabcc838ad581c64e8ed52c11fbe86ddebd9da0818cd5"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} + +[package.extras] +testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] + +[[package]] +name = "tox" +version = "4.11.3" +description = "tox is a generic virtualenv management and test command line tool" +optional = false +python-versions = ">=3.8" +files = [ + {file = "tox-4.11.3-py3-none-any.whl", hash = "sha256:599af5e5bb0cad0148ac1558a0b66f8fff219ef88363483b8d92a81e4246f28f"}, + {file = "tox-4.11.3.tar.gz", hash = "sha256:5039f68276461fae6a9452a3b2c7295798f00a0e92edcd9a3b78ba1a73577951"}, +] + +[package.dependencies] +cachetools = ">=5.3.1" +chardet = ">=5.2" +colorama = ">=0.4.6" +filelock = ">=3.12.3" +packaging = ">=23.1" +platformdirs = ">=3.10" +pluggy = ">=1.3" +pyproject-api = ">=1.6.1" +tomli = {version = ">=2.0.1", markers = "python_version < \"3.11\""} +virtualenv = ">=20.24.3" + +[package.extras] +docs = ["furo (>=2023.8.19)", "sphinx (>=7.2.4)", "sphinx-argparse-cli (>=1.11.1)", "sphinx-autodoc-typehints (>=1.24)", "sphinx-copybutton (>=0.5.2)", "sphinx-inline-tabs (>=2023.4.21)", "sphinxcontrib-towncrier (>=0.2.1a0)", "towncrier (>=23.6)"] +testing = ["build[virtualenv] (>=0.10)", "covdefaults (>=2.3)", "detect-test-pollution (>=1.1.1)", "devpi-process (>=1)", "diff-cover (>=7.7)", "distlib (>=0.3.7)", "flaky (>=3.7)", "hatch-vcs (>=0.3)", "hatchling (>=1.18)", "psutil (>=5.9.5)", "pytest (>=7.4)", "pytest-cov (>=4.1)", "pytest-mock (>=3.11.1)", "pytest-xdist (>=3.3.1)", "re-assert (>=1.1)", "time-machine (>=2.12)", "wheel (>=0.41.2)"] + +[[package]] +name = "virtualenv" +version = "20.24.6" +description = "Virtual Python Environment builder" +optional = false +python-versions = ">=3.7" +files = [ + {file = "virtualenv-20.24.6-py3-none-any.whl", hash = "sha256:520d056652454c5098a00c0f073611ccbea4c79089331f60bf9d7ba247bb7381"}, + {file = "virtualenv-20.24.6.tar.gz", hash = "sha256:02ece4f56fbf939dbbc33c0715159951d6bf14aaf5457b092e4548e1382455af"}, +] + +[package.dependencies] +distlib = ">=0.3.7,<1" +filelock = ">=3.12.2,<4" +platformdirs = ">=3.9.1,<4" + +[package.extras] +docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.2)", "sphinx-argparse (>=0.4)", "sphinxcontrib-towncrier (>=0.2.1a0)", "towncrier (>=23.6)"] +test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=23.1)", "pytest (>=7.4)", "pytest-env (>=0.8.2)", "pytest-freezer (>=0.4.8)", "pytest-mock (>=3.11.1)", "pytest-randomly (>=3.12)", "pytest-timeout (>=2.1)", "setuptools (>=68)", "time-machine (>=2.10)"] + +[metadata] +lock-version = "2.0" +python-versions = "^3.10" +content-hash = "d92c295edbd95dee79e1497dd9a79b266dadfae7ffcae67d106ae84b56503bbc" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..5f24295 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,119 @@ +# package ########################################################### +[build-system] +requires = ["poetry-core>=1.2.0", "pytest>=7.4", "tox>=4.11"] +build-backend = "poetry.core.masonry.api" + +# pytest ############################################################ +[tool.pytest.ini_options] +pythonpath = [ + "./src", +] + +# ruff ############################################################## +[tool.ruff] +target-version = "py310" +line-length = 100 +select = ["C4", "C90", "E", "F", "PL", "PT", "Q", "W"] +ignore = [ + "E731", # Lambdas are our friends. + "Q000", # We don't mind double quotes. For now... + "TRY003", +] +exclude = [ + ".vscode", # needed ??? + "#*.py", # Convention. Unversioned files we don't want to lint or delete. +] + +[tool.ruff.per-file-ignores] + #"__init__.py" = ["F401"] + #"*/settings/*.py" = ["E266", "F401", "F403" , "F405"] + #"**/tests/**/*.py" = ["PLR0913"] + +[tool.ruff.mccabe] +max-complexity = 15 + +# Legend: +# A -> flake8-builtins +# B* -> flake8-bugbear +# C4 -> flake8-comprehensions +# C90 -> mccabe +# DJ -> flake8-django +# E -> pycodestyle error +# F -> pyflakes +# I -> isort +# PT* -> flake8-pytest-style +# PTH* -> flake8-pathlib +# Q -> flake8-quotes +# SIM* -> flake8-simplify +# TRY* -> tryceratops +# W -> pycodestyle warning +# E241 -> Multiple spaces after ','. +# E731 -> Do not assign a lambda expression, use a def. +# F401 -> Module imported but unused. +# F403 -> 'from module import *' used; unable to detect undefined names. +# F405 -> Name may be undefined, or defined from star imports: module. +# PLR0913 -> Too many arguments to function call. +# TRY003 -> Avoid specifying long messages outside the exception class. +# Q000 -> Single quotes found but double quotes preferred. +# +# * -> Considered to be enabled in the future. + +# prospector - mirror in ruff +#strictness: medium +#test-warnings: true + +#ignore-paths: + #- misc + #- docs +#ignore-patterns: + #- pyenv* + +#pylint: + #disable: + #- cyclic-import # doesn't seem to work properly + +#vulture: + # turn on locally to spot unused code + #run: false + +# poetry ############################################################ +[tool.poetry] +name = "jj2cli" +version = "0.6.0" +description = "Juiced Jinja2 command-line tool" +license = "BSD-2-Clause" +authors = [ + "Manolis Stamatogiannakis ", +] +readme = "README.md" +homepage = "https://github.com/m000/j2cli" +repository = "https://github.com/m000/j2cli" +keywords = ['Jinja2', 'templating', 'command-line', 'CLI'] +classifiers=[ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "Operating System :: OS Independent", + "Topic :: Software Development", + "Natural Language :: English", + "Programming Language :: Python :: 3", +] + +[tool.poetry.scripts] +j2 = "jj2cli:render" # temporarily keep the old entry point +jj2 = "jj2cli:render" +jj2dep = "jj2cli:dependencies" + +[tool.poetry.dependencies] +python = "^3.10" +jinja2 = "^3.1" + +[tool.poetry.group.dev.dependencies] +pytest = "^7.0" +tox = "^4.0" + +[tool.poetry.group.yaml] +optional = true + +[tool.poetry.group.yaml.dependencies] +pyyaml = "^6.0" diff --git a/requirements-dev.txt b/requirements-dev.txt deleted file mode 100644 index f837c52..0000000 --- a/requirements-dev.txt +++ /dev/null @@ -1,4 +0,0 @@ -wheel -nose -exdoc -six diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index ed8a958..0000000 --- a/setup.cfg +++ /dev/null @@ -1,5 +0,0 @@ -[bdist_wheel] -universal = 1 - -[metadata] -license_file = LICENSE diff --git a/setup.py b/setup.py deleted file mode 100755 index 5733ee5..0000000 --- a/setup.py +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env python -""" j2cli - Jinja2 Command-Line Tool -================================ - -`j2cli` is a command-line tool for templating in shell-scripts, -leveraging the [Jinja2](http://jinja.pocoo.org/docs/) library. - -Features: - -* Jinja2 templating -* INI, YAML, JSON data sources supported -* Allows the use of environment variables in templates! Hello [Docker](http://www.docker.com/) :) - -Inspired by [mattrobenolt/jinja2-cli](https://github.com/mattrobenolt/jinja2-cli) -""" - -from setuptools import setup, find_packages -import sys - -# PyYAML 3.11 was the last to support Python 2.6 -# This code limits pyyaml version for older pythons -pyyaml_version = 'pyyaml >= 3.10' # fresh -if sys.version_info[:2] == (2, 6): - pyyaml_version = 'pyyaml<=3.11' - - -setup( - name='j2cli', - version='0.3.12b', - author='Mark Vartanyan', - author_email='kolypto@gmail.com', - - url='https://github.com/kolypto/j2cli', - license='BSD', - description='Command-line interface to Jinja2 for templating in shell scripts.', - long_description=__doc__, # can't do open('README.md').read() because we're describing self - long_description_content_type='text/markdown', - keywords=['Jinja2', 'templating', 'command-line', 'CLI'], - - packages=find_packages(), - scripts=[], - entry_points={ - 'console_scripts': [ - 'j2 = j2cli:main', - ] - }, - - install_requires=[ - 'jinja2 >= 2.7.2', - 'six >= 1.10', - ], - extras_require={ - 'yaml': [pyyaml_version,] - }, - include_package_data=True, - zip_safe=False, - test_suite='nose.collector', - - platforms='any', - classifiers=[ - 'Development Status :: 5 - Production/Stable', - 'Intended Audience :: Developers', - 'Intended Audience :: System Administrators', - 'Operating System :: OS Independent', - 'Topic :: Software Development', - 'Natural Language :: English', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 3', - ], -) diff --git a/src/jj2cli/__init__.py b/src/jj2cli/__init__.py new file mode 100644 index 0000000..0a20deb --- /dev/null +++ b/src/jj2cli/__init__.py @@ -0,0 +1,12 @@ +#! /usr/bin/env python +""" j2cli main file """ +import importlib.metadata + +__author__ = "Manolis Stamatogiannakis" +__email__ = "mstamat@gmail.com" +__version__ = importlib.metadata.version('jj2cli') + +from jj2cli.cli import render + +if __name__ == '__main__': + render() diff --git a/src/jj2cli/cli.py b/src/jj2cli/cli.py new file mode 100644 index 0000000..c66b60e --- /dev/null +++ b/src/jj2cli/cli.py @@ -0,0 +1,164 @@ +import argparse +import io +import logging +import os +import sys +from functools import reduce +from importlib.machinery import SourceFileLoader + +import jinja2 +import jinja2.loaders +import jinja2.meta + +from . import __version__, filters, parsers +from .customize import CustomizationModule +from .defaults import CONTEXT_FORMATS +from .render import Jinja2TemplateRenderer + +# available log levels, adjusted with -v at command line +LOGLEVELS = [logging.ERROR, logging.WARNING, logging.INFO, logging.DEBUG] + +# format to use for logging +LOGFORMAT = '%(levelname)s: %(message)s' + + +def render_command(argv): + """ Pure render command + :param argv: Command-line arguments + :type argv: list + :return: Rendered template + :rtype: basestring + """ + version_info = (__version__, jinja2.__version__) + parser = argparse.ArgumentParser( + description='Renders Jinja2 templates from the command line.', + epilog='', + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + p_input = parser.add_argument_group('input options') + p_output = parser.add_argument_group('output options') + p_custom = parser.add_argument_group('customization options') + + ### optional arguments ########################################## + parser.add_argument('-V', '--version', action='version', + version='jj2cli {0}, Jinja2 {1}'.format(*version_info)) + parser.add_argument('-v', '--verbose', action='count', default=0, + help='Increase verbosity.') + ### input options ############################################### + p_input.add_argument('template', help='Template file to process.') + p_input.add_argument('data', nargs='+', default=[], type=parsers.InputDataType(), + help='Input data specification. ' + 'Multiple sources in different formats can be specified. ' + 'The different sources will be squashed into a singled dict. ' + 'The format is [:[:]]. ' + 'Parts of the specification may be left empty. ' + 'See examples at the end of the help.') + p_input.add_argument('-U', '--undefined', default='strict', + dest='undefined', choices=Jinja2TemplateRenderer.UNDEFINED, + help='Set the Jinja2 beahaviour for undefined variables.)') + p_input.add_argument('-I', '--ignore-missing', action='store_true', + help='Ignore any missing data files.') + p_input.add_argument('-f', '--fallback-format', + default='ini', choices=CONTEXT_FORMATS, + help='Specify fallback data format. ' + 'Used for data with no specified format and no appropriate extension.') + ### output options ############################################## + p_output.add_argument('-o', metavar='outfile', dest='output_file', + help="Output to a file instead of stdout.") + p_output.add_argument('--no-compact', action='store_true', dest='no_compact', + help='Do not compact space around Jinja2 blocks.') + ### customization ############################################### + p_custom.add_argument('--filters', nargs='+', default=[], + metavar='python-file', dest='filters', + help='Load the top-level functions from the specified file(s) as Jinja2 filters.') + p_custom.add_argument('--tests', nargs='+', default=[], + metavar='python-file', dest='tests', + help='Load the top-level functions from the specified file(s) as Jinja2 tests.') + p_custom.add_argument('--customize', default=None, + metavar='python-file', dest='customize', + help='Load custom jj2cli behavior from a Python file.') + + args = parser.parse_args(argv[1:]) + logging.basicConfig(format=LOGFORMAT, level=LOGLEVELS[min(args.verbose, len(LOGLEVELS)-1)]) + logging.debug("PARSED_ARGS render_command: %s", args) + + # Customization + if args.customize is not None: + customize = CustomizationModule( + SourceFileLoader('customize-module', args.customize).load_module() + ) + else: + customize = CustomizationModule(None) + + # Read data based on specs. + data = [d.parse(ignore_missing=args.ignore_missing, fallback_format=args.fallback_format) for d in args.data] + + # Squash data into a single context. + context = reduce(parsers.dict_squash, data, {}) + + # Apply final customizations. + context = customize.alter_context(context) + + # Renderer + renderer = Jinja2TemplateRenderer(os.getcwd(), args.undefined, args.no_compact, j2_env_params=customize.j2_environment_params()) + customize.j2_environment(renderer._env) # pylint: disable=protected-access + + # Filters, Tests + renderer.register_filters(filters.EXTRA_FILTERS) + for fname in args.filters: + renderer.import_filters(fname) + for fname in args.tests: + renderer.import_tests(fname) + + renderer.register_filters(customize.extra_filters()) + renderer.register_tests(customize.extra_tests()) + result = renderer.render(args.template, context) + + # -o + if args.output_file: + with io.open(args.output_file, 'wt', encoding='utf-8') as f: + f.write(result.decode('utf-8')) + f.close() + return b'' + + # Finish + return result + + +def render(): + """ CLI entry point for rendering templates. """ + try: + output = render_command(sys.argv) + except SystemExit: + return 1 + outstream = getattr(sys.stdout, 'buffer', sys.stdout) + outstream.write(output) + return 0 + + +def dependencies(): + """ CLI entry point for analyzing template dependencies. """ + #version_info = (__version__, jinja2.__version__) + parser = argparse.ArgumentParser( + description='Analyze Jinja2 templates for dependencies.', + epilog='', + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + p_input = parser.add_argument_group('input options') + p_output = parser.add_argument_group('output options') + p_output_mode = p_output.add_mutually_exclusive_group() + ### input options ############################################### + p_input.add_argument('templates', metavar='TEMPLATE', nargs='+', + type=argparse.FileType('r', encoding='utf-8')) + ### output options ############################################## + p_output.add_argument('-f', '--format', + default='make', choices=sorted(DEPENDENCIES_OUTPUT_FORMATS), + help='Specify output format for dependencies.') + p_output_mode.add_argument('-o', metavar='outfile', dest='output_file', + help="Output to a file instead of stdout.") + p_output_mode.add_argument('--per-file', action='store_true', dest='per_file', + help='Produce one output file per input file.') + + args = parser.parse_args() + print(args) + raise NotImplementedError("jj2dep has not yet been implemented.") diff --git a/j2cli/extras/customize.py b/src/jj2cli/customize.py similarity index 87% rename from j2cli/extras/customize.py rename to src/jj2cli/customize.py index 366321f..d759f21 100644 --- a/j2cli/extras/customize.py +++ b/src/jj2cli/customize.py @@ -1,13 +1,9 @@ -class CustomizationModule(object): +class CustomizationModule: """ The interface for customization functions, defined as module-level functions """ def __init__(self, module=None): if module is not None: - def howcall(*args): - print(args) - exit(1) - # Import every module function as a method on ourselves for name in self._IMPORTED_METHOD_NAMES: try: diff --git a/src/jj2cli/defaults.py b/src/jj2cli/defaults.py new file mode 100644 index 0000000..69ec48b --- /dev/null +++ b/src/jj2cli/defaults.py @@ -0,0 +1,40 @@ +import itertools + +# Jinja2 extensions loaded by jj2cli. +JINJA2_ENABLED_EXTENSIONS = ( + 'jinja2.ext.i18n', + 'jinja2.ext.do', + 'jinja2.ext.loopcontrols', +) + +# Set yaml_loader for parsers. +try: + import yaml + try: + _yaml_loader = yaml.FullLoader + except AttributeError: + _yaml_loader = yaml.SafeLoader + yaml_load = lambda iostr: yaml.load(iostr, Loader=_yaml_loader) +except ImportError: + yaml_load = None + +# Supported context formats. +CONTEXT_FORMATS = ['env', 'ENV', 'ini', 'json', 'yaml'] +if yaml_load is None: + CONTEXT_FORMATS.remove('yaml') + +# Format aliases dictionary. +# COMPAT: Chaining used instead of unpacking (*) for backwards compatibility. +CONTEXT_FORMATS_ALIASES = dict(itertools.chain( + zip(CONTEXT_FORMATS, CONTEXT_FORMATS), + filter(lambda t: t[1] in CONTEXT_FORMATS, [('yml', 'yaml')]) +)) + +# Variables for parsing dataspecs. +DATASPEC_SEP = ':' +DATASPEC_COMPONENTS_MAX = 3 + +# Supported formats for outputting template dependencies. +DEPENDENCIES_OUTPUT_FORMATS = ['make', 'json', 'yaml', 'delim'] +if yaml_load is None: + DEPENDENCIES_OUTPUT_FORMATS.remove('yaml') diff --git a/src/jj2cli/filters.py b/src/jj2cli/filters.py new file mode 100644 index 0000000..edd1be9 --- /dev/null +++ b/src/jj2cli/filters.py @@ -0,0 +1,171 @@ +""" Additional Jinja2 filters """ +import os +import re +import sys +from jinja2 import is_undefined, pass_context + +if sys.version_info >= (3,0): + from shutil import which # pylint: disable=import-error +elif sys.version_info >= (2,5): + from shutilwhich import which # pylint: disable=import-error +else: + assert False, "Unsupported Python version: %s" % sys.version_info + +if sys.version_info >= (3,3): + from shlex import quote as sh_quote +elif sys.version_info >= (2,7): + from pipes import quote as sh_quote +else: + assert False, "Unsupported Python version: %s" % sys.version_info + +def docker_link(value, fmt='{addr}:{port}'): + """ Given a Docker Link environment variable value, format it into something else. + XXX: The name of the filter is not very informative. This is actually a partial URI parser. + + This first parses a Docker Link value like this: + + DB_PORT=tcp://172.17.0.5:5432 + + Into a dict: + + ```python + { + 'proto': 'tcp', + 'addr': '172.17.0.5', + 'port': '5432' + } + ``` + + And then uses `fmt` to format it, where the default format is '{addr}:{port}'. + + More info here: [Docker Links](https://docs.docker.com/userguide/dockerlinks/) + + :param value: Docker link (from an environment variable) + :param fmt: The format to apply. Supported placeholders: `{proto}`, `{addr}`, `{port}` + :return: Formatted string + """ + # pass undefined values on down the pipeline + if is_undefined(value): + return value + + # Parse the value + m = re.match(r'(?P.+)://' r'(?P.+):' r'(?P.+)$', value) + if not m: + raise ValueError('The provided value does not seems to be a Docker link: {0}'.format(value)) + d = m.groupdict() + + # Format + return fmt.format(**d) + + +def env(varname, default=None): + """ Use an environment variable's value inside your template. + + This filter is available even when your data source is something other that the environment. + + Example: + + ```jinja2 + User: {{ user_login }} + Pass: {{ "USER_PASSWORD"|env }} + ``` + + You can provide the default value: + + ```jinja2 + Pass: {{ "USER_PASSWORD"|env("-none-") }} + ``` + + For your convenience, it's also available as a function: + + ```jinja2 + User: {{ user_login }} + Pass: {{ env("USER_PASSWORD") }} + ``` + + Notice that there must be quotes around the environment variable name + """ + if default is None: + # Raise KeyError when not provided + return os.environ[varname] + + # With the default, there's never an error + return os.getenv(varname, default) + + +def align_suffix(text, delim, column=None, spaces_after_delim=1): + """ Align the suffixes of lines in text, starting from the specified delim. + + Example: XXX + """ + s='' + + if column is None or column == 'auto': + column = max(map(lambda ln: ln.find(delim), text.splitlines())) + elif column == 'previous': + column = align_suffix.column_previous + + for ln in map(lambda s: s.split(delim, 1), text.splitlines()): + if len(ln) < 2: + # no delimiter occurs + s += ln[0].rstrip() + os.linesep + elif ln[0].strip() == '': + # no content before delimiter - leave as-is + s += ln[0] + delim + ln[1] + os.linesep + else: + # align + s += ln[0].rstrip().ljust(column) + delim + spaces_after_delim*' ' + ln[1].strip() + os.linesep + + align_suffix.column_previous = column + return s + + +align_suffix.column_previous = None + + +@pass_context +def ctxlookup(context, key): + """ Lookup the value of a key in the template context. + """ + v = context + try: + for k in key.split('.'): + v = v[k] + return v + except KeyError: + return context.environment.undefined(name=key) + +def sh_opt(text, name, delim=" ", quote=False): + """ Format text as a command line option. + """ + if not text: + return '' + if quote: + text = sh_quote(text) + return '%s%s%s' % (name, delim, text) + + +def sh_optq(text, name, delim=" "): + """ Quote text and format as a command line option. + """ + return sh_opt(text, name, delim, quote=True) + + +# Filters to be loaded +EXTRA_FILTERS = { + 'sh_quote': sh_quote, + 'sh_which': which, + 'sh_expand': lambda s: os.path.expandvars(os.path.expanduser(s)), + 'sh_expanduser': os.path.expanduser, + 'sh_expandvars': os.path.expandvars, + 'sh_realpath': os.path.realpath, + 'sh_opt': sh_opt, + 'sh_optq': sh_optq, + 'ifelse': lambda t, truev, falsev: truev if t else falsev, + 'onoff': lambda t: 'on' if t else 'off', + 'yesno': lambda t: 'yes' if t else 'no', + 'docker_link': docker_link, + 'env': env, + 'align_suffix': align_suffix, + 'ctxlookup': ctxlookup, +} diff --git a/src/jj2cli/parsers.py b/src/jj2cli/parsers.py new file mode 100644 index 0000000..d64071d --- /dev/null +++ b/src/jj2cli/parsers.py @@ -0,0 +1,192 @@ +import configparser +import json +import logging +import os +import platform +import re +import sys +from pathlib import Path +from typing import Mapping + +from .defaults import (CONTEXT_FORMATS, CONTEXT_FORMATS_ALIASES, + DATASPEC_COMPONENTS_MAX, DATASPEC_SEP, yaml_load) + + +class InputDataType: + """Factory for creating jj2cli input data types. + + Instances of InputDataType are typically passed as type= arguments to the + ArgumentParser add_argument() method. + + Keyword Arguments: + - mode -- A string indicating how the file is to be opened. Accepts the + same values as the builtin open() function. + - bufsize -- The file's desired buffer size. Accepts the same values as + the builtin open() function. + - encoding -- The file's encoding. Accepts the same values as the + builtin open() function. + - errors -- A string indicating how encoding and decoding errors are to + be handled. Accepts the same value as the builtin open() function. + """ + def __init__(self, mode='r', bufsize=-1, encoding=None, errors=None): + self._mode = mode + self._bufsize = bufsize + self._encoding = encoding + self._errors = errors + + def __call__(self, dspec): + # detect windows-style paths + # NB: In Windows, filenames matching ^[a-z]$ always require a directory + # if the data spec has >1 components. + if DATASPEC_SEP == ':' and platform.system() == 'Windows': + m = re.match(r'^[a-z]:[^:]+', dspec, re.I) + else: + m = None + + # parse supplied components + if m is None: + # normal case + dspec = dspec.rsplit(DATASPEC_SEP, DATASPEC_COMPONENTS_MAX-1) + else: + # matched windows drive at the start of the data spec + dspec = dspec[m.span()[1] + 1:] + dspec = [m.group(0)] + dspec.rsplit(DATASPEC_SEP, DATASPEC_COMPONENTS_MAX-2) + + # pad missing components + dspec += (DATASPEC_COMPONENTS_MAX - len(dspec))*[None] + + # post-process parsed components + path, fmt, ctx_dst = dspec + path = Path(path) if path not in ['', '-'] else None + if fmt in CONTEXT_FORMATS_ALIASES: + # forced format is case-sensitive + fmt = CONTEXT_FORMATS_ALIASES[fmt] + elif fmt in ['', '?', None] and path is not None and path.suffix[1:] in CONTEXT_FORMATS_ALIASES: + # file extensions are case-insensitive + fmt = CONTEXT_FORMATS_ALIASES[path.suffix[1:].lower()] + else: + fmt = None + ctx_dst = None if ctx_dst in ['', None] else None + + # check for formats that don't use file input + if fmt == 'ENV' and path is not None: + logging.warning("Ignoring source for %s format: %s", fmt, path) + path = None + + # open stream and return InputData object + if path is None: + if fmt == 'ENV': + iostr = None + elif 'r' in self._mode: + iostr = sys.stdin + elif 'w' in self._mode: + # XXX: Is there a use-case we could use this? + iostr = sys.stdout + else: + raise ValueError("Invalid mode %r for std streams." % self._mode) + else: + try: + iostr = path.open(self._mode, self._bufsize, self._encoding, self._errors) + except FileNotFoundError as e: + # FileNotFoundError will be reraised later by InputData.parse(), + # depending on # whether the -I flag has been specified. + iostr = e + + return InputData(iostr, fmt, ctx_dst) + + +class InputData: + def __init__(self, iostr, fmt=None, ctx_dst=None): + self._iostr = iostr + self._fmt = fmt + self._ctx_dst = ctx_dst + + def __repr__(self): + ioinfo = (self._iostr + if self._iostr is None or isinstance(self._iostr, FileNotFoundError) + else '%s:%s:%s' % (self._iostr.name, self._iostr.mode, self._iostr.encoding)) + return '%s(%s, %s, %s)' % (type(self).__name__, ioinfo, self.fmt, self._ctx_dst) + + @property + def fmt(self): + return self._fmt + + @fmt.setter + def set_fmt(self, v): + if v in CONTEXT_FORMATS: + self._fmt = v + else: + raise ValueError("Invalid format %s." % v) + + def parse(self, ignore_missing=False, fallback_format='ini'): + """Parses the data from the data stream of the object. + If ignore_missing is set, missing files will produce and empty dict. + If no format is set for the object, fallback_format is used. + """ + fmt = self._fmt if self._fmt is not None else fallback_format + if isinstance(self._iostr, FileNotFoundError): + if ignore_missing is True: + return {} + raise self._iostr + return getattr(self, '_parse_%s' % fmt)() + + def _parse_ENV(self): + """Loads data from shell environment. + """ + return os.environ.copy() + + def _parse_env(self): + """Parses an env-like format. + XXX + """ + normalize = lambda t: (t[0].strip(), t[1].strip()) + return dict([ + normalize(ln.split('=', 1)) + for ln in self._iostr + if '=' in ln + ]) + + def _parse_ini(self): + """Parses windows-style ini files. + """ + class MyConfigParser(configparser.ConfigParser): + def as_dict(self): + """ Export as dict + :rtype: dict + """ + d = dict(self._sections) + for k in d: + d[k] = dict(self._defaults, **d[k]) + d[k].pop('__name__', None) + return d + ini = MyConfigParser() + ini.read_file(self._iostr) + return ini.as_dict() + + def _parse_json(self): + return json.load(self._iostr) + + def _parse_yaml(self): + if yaml_load is None: + raise RuntimeError("YAML data parser invoked, but no YAML support is present.") + return yaml_load(self._iostr) + + +def dict_squash(d, u): + """ Squashes contents of u on d. + :param d: Dictionary to be updated. + :type dict: dict + :param u: Dictionary with updates to be applied. + :type dict: dict + :return: Updated version of d. + :rtype: dict + """ + for k, v in u.items(): + dv = d.get(k, {}) + if not isinstance(dv, Mapping): + d[k] = v + elif isinstance(v, Mapping): + d[k] = dict_squash(dv, v) + else: + d[k] = v + return d diff --git a/src/jj2cli/render.py b/src/jj2cli/render.py new file mode 100644 index 0000000..7bf3fbe --- /dev/null +++ b/src/jj2cli/render.py @@ -0,0 +1,92 @@ +import inspect +import io +import logging +import os +from importlib.machinery import SourceFileLoader + +import jinja2 +import jinja2.loaders +import jinja2.meta + +from . import filters +from .defaults import JINJA2_ENABLED_EXTENSIONS + + +class FilePathLoader(jinja2.BaseLoader): + """ Custom Jinja2 template loader which just loads a single template file """ + + def __init__(self, cwd, encoding='utf-8'): + self.cwd = cwd + self.encoding = encoding + + def get_source(self, environment, template): + # Path + filename = os.path.join(self.cwd, template) + logging.debug("TEMPLATE_PATH %s", filename) + + # Read + try: + with io.open(template, 'rt', encoding=self.encoding) as f: + contents = f.read() + except IOError: + raise jinja2.TemplateNotFound(template) + + # Finish + uptodate = lambda: False + return contents, filename, uptodate + + +class Jinja2TemplateRenderer: + """ Template renderer """ + + UNDEFINED = { + 'strict': jinja2.StrictUndefined, # raises errors for undefined variables + 'normal': jinja2.Undefined, # can be printed/iterated - error on other operations + 'debug': jinja2.DebugUndefined, # return the debug info when printed + } + + def __init__(self, cwd, undefined='strict', no_compact=False, j2_env_params=None): + # Custom env params + j2_env_params = j2_env_params if j2_env_params is not None else {} + j2_env_params.setdefault('keep_trailing_newline', True) + j2_env_params.setdefault('undefined', self.UNDEFINED[undefined]) + j2_env_params.setdefault('trim_blocks', not no_compact) + j2_env_params.setdefault('lstrip_blocks', not no_compact) + j2_env_params.setdefault('extensions', JINJA2_ENABLED_EXTENSIONS) + j2_env_params.setdefault('loader', FilePathLoader(cwd)) + + # Environment + self._env = jinja2.Environment(**j2_env_params) + self._env.globals.update(dict( + env=filters.env + )) + + def register_filters(self, filters): + self._env.filters.update(filters) + + def register_tests(self, tests): + self._env.tests.update(tests) + + def import_filters(self, filename): + self.register_filters(self._import_functions(filename)) + + def import_tests(self, filename): + self.register_tests(self._import_functions(filename)) + + def _import_functions(self, filename): + m = SourceFileLoader('imported-funcs', filename).load_module() + return dict((name, func) for name, func in inspect.getmembers(m) if inspect.isfunction(func)) + + def render(self, template_path, context): + """ Render a template + :param template_path: Path to the template file + :type template_path: basestring + :param context: Template data + :type context: dict + :return: Rendered template + :rtype: basestring + """ + return self._env \ + .get_template(template_path) \ + .render(context) \ + .encode('utf-8') diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/render-test.py b/tests/render-test.py deleted file mode 100644 index 791764a..0000000 --- a/tests/render-test.py +++ /dev/null @@ -1,240 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import unicode_literals - -import unittest -import os, sys, io, os.path, tempfile -from copy import copy -from contextlib import contextmanager -from jinja2.exceptions import UndefinedError - -from j2cli.cli import render_command - -@contextmanager -def mktemp(contents): - """ Create a temporary file with the given contents, and yield its path """ - _, path = tempfile.mkstemp() - fp = io.open(path, 'wt+', encoding='utf-8') - fp.write(contents) - fp.flush() - try: - yield path - finally: - fp.close() - os.unlink(path) - - -@contextmanager -def mock_environ(new_env): - old_env = copy(os.environ) - os.environ.update(new_env) - yield - os.environ.clear() - os.environ.update(old_env) - - -class RenderTest(unittest.TestCase): - def setUp(self): - os.chdir( - os.path.dirname(__file__) - ) - - def _testme(self, argv, expected_output, stdin=None, env=None): - """ Helper test shortcut """ - with mock_environ(env or {}): - result = render_command(os.getcwd(), env or {}, stdin, argv) - if isinstance(result, bytes): - result = result.decode('utf-8') - self.assertEqual(result, expected_output) - - #: The expected output - expected_output = """server { - listen 80; - server_name localhost; - - root /var/www/project; - index index.htm; - - access_log /var/log/nginx//http.access.log combined; - error_log /var/log/nginx//http.error.log; -} -""" - - def _testme_std(self, argv, stdin=None, env=None): - self._testme(argv, self.expected_output, stdin, env) - - def test_ini(self): - # Filename - self._testme_std(['resources/nginx.j2', 'resources/data.ini']) - # Format - self._testme_std(['--format=ini', 'resources/nginx.j2', 'resources/data.ini']) - # Stdin - self._testme_std(['--format=ini', 'resources/nginx.j2'], stdin=open('resources/data.ini')) - self._testme_std(['--format=ini', 'resources/nginx.j2', '-'], stdin=open('resources/data.ini')) - - def test_json(self): - # Filename - self._testme_std(['resources/nginx.j2', 'resources/data.json']) - # Format - self._testme_std(['--format=json', 'resources/nginx.j2', 'resources/data.json']) - # Stdin - self._testme_std(['--format=json', 'resources/nginx.j2'], stdin=open('resources/data.json')) - self._testme_std(['--format=json', 'resources/nginx.j2', '-'], stdin=open('resources/data.json')) - - def test_yaml(self): - try: - import yaml - except ImportError: - raise unittest.SkipTest('Yaml lib not installed') - - # Filename - self._testme_std(['resources/nginx.j2', 'resources/data.yml']) - self._testme_std(['resources/nginx.j2', 'resources/data.yaml']) - # Format - self._testme_std(['--format=yaml', 'resources/nginx.j2', 'resources/data.yml']) - # Stdin - self._testme_std(['--format=yaml', 'resources/nginx.j2'], stdin=open('resources/data.yml')) - self._testme_std(['--format=yaml', 'resources/nginx.j2', '-'], stdin=open('resources/data.yml')) - - def test_env(self): - # Filename - self._testme_std(['--format=env', 'resources/nginx-env.j2', 'resources/data.env']) - self._testme_std([ 'resources/nginx-env.j2', 'resources/data.env']) - # Format - self._testme_std(['--format=env', 'resources/nginx-env.j2', 'resources/data.env']) - self._testme_std([ 'resources/nginx-env.j2', 'resources/data.env']) - # Stdin - self._testme_std(['--format=env', 'resources/nginx-env.j2', '-'], stdin=open('resources/data.env')) - self._testme_std([ 'resources/nginx-env.j2', '-'], stdin=open('resources/data.env')) - - # Environment! - # In this case, it's not explicitly provided, but implicitly gotten from the environment - env = dict(NGINX_HOSTNAME='localhost', NGINX_WEBROOT='/var/www/project', NGINX_LOGS='/var/log/nginx/') - self._testme_std(['--format=env', 'resources/nginx-env.j2'], env=env) - self._testme_std([ 'resources/nginx-env.j2'], env=env) - - def test_import_env(self): - # Import environment into a variable - with mktemp('{{ a }}/{{ env.B }}') as template: - with mktemp('{"a":1}') as context: - self._testme(['--format=json', '--import-env=env', template, context], '1/2', env=dict(B='2')) - # Import environment into global scope - with mktemp('{{ a }}/{{ B }}') as template: - with mktemp('{"a":1,"B":1}') as context: - self._testme(['--format=json', '--import-env=', template, context], '1/2', env=dict(B='2')) - - def test_env_file__equals_sign_in_value(self): - # Test whether environment variables with "=" in the value are parsed correctly - with mktemp('{{ A|default('') }}/{{ B }}/{{ C }}') as template: - with mktemp('A\nB=1\nC=val=1\n') as context: - self._testme(['--format=env', template, context], '/1/val=1') - - def test_unicode(self): - # Test how unicode is handled - # I'm using Russian language for unicode :) - with mktemp('Проверка {{ a }} связи!') as template: - with mktemp('{"a": "широкополосной"}') as context: - self._testme(['--format=json', template, context], 'Проверка широкополосной связи!') - - # Test case from issue #17: unicode environment variables - if sys.version_info[0] == 2: - # Python 2: environment variables are bytes - self._testme(['resources/name.j2'], u'Hello Jürgen!\n', env=dict(name=b'J\xc3\xbcrgen')) - else: - # Python 3: environment variables are unicode strings - self._testme(['resources/name.j2'], u'Hello Jürgen!\n', env=dict(name=u'Jürgen')) - - def test_filters__env(self): - with mktemp('user_login: kolypto') as yml_file: - with mktemp('{{ user_login }}:{{ "USER_PASS"|env }}') as template: - # Test: template with an env variable - self._testme(['--format=yaml', template, yml_file], 'kolypto:qwerty123', env=dict(USER_PASS='qwerty123')) - - # environment cleaned up - assert 'USER_PASS' not in os.environ - - # Test: KeyError - with self.assertRaises(KeyError): - self._testme(['--format=yaml', template, yml_file], 'kolypto:qwerty123', env=dict()) - - # Test: default - with mktemp('{{ user_login }}:{{ "USER_PASS"|env("-none-") }}') as template: - self._testme(['--format=yaml', template, yml_file], 'kolypto:-none-', env=dict()) - - # Test: using as a function - with mktemp('{{ user_login }}:{{ env("USER_PASS") }}') as template: - self._testme(['--format=yaml', template, yml_file], 'kolypto:qwerty123', env=dict(USER_PASS='qwerty123')) - - with self.assertRaises(KeyError): - # Variable not set - self._testme(['--format=yaml', template, yml_file], '', env=dict()) - - # Test: using as a function, with a default - with mktemp('{{ user_login }}:{{ env("USER_PASS", "-none-") }}') as template: - self._testme(['--format=yaml', template, yml_file], 'kolypto:qwerty123', env=dict(USER_PASS='qwerty123')) - self._testme(['--format=yaml', template, yml_file], 'kolypto:-none-', env=dict()) - - - def test_custom_filters(self): - with mktemp('{{ a|parentheses }}') as template: - self._testme(['--format=env', '--filters=resources/custom_filters.py', template], '(1)', env=dict(a='1')) - - def test_custom_tests(self): - with mktemp('{% if a|int is custom_odd %}odd{% endif %}') as template: - self._testme(['--format=env', '--tests=resources/custom_tests.py', template], 'odd', env=dict(a='1')) - - def test_output_file(self): - with mktemp('{{ a }}') as template: - try: - self._testme(['-o', '/tmp/j2-out', template], '', env=dict(a='123')) - self.assertEqual('123', io.open('/tmp/j2-out', 'r').read()) - finally: - os.unlink('/tmp/j2-out') - - def test_undefined(self): - """ Test --undefined """ - # `name` undefined: error - self.assertRaises(UndefinedError, self._testme, ['resources/name.j2'], u'Hello !\n', env=dict()) - # `name` undefined: no error - self._testme(['--undefined', 'resources/name.j2'], u'Hello !\n', env=dict()) - - def test_jinja2_extensions(self): - """ Test that an extension is enabled """ - with mktemp('{% do [] %}') as template: - # `do` tag is an extension - self._testme([template], '') - - - def test_customize(self): - """ Test --customize """ - # Test: j2_environment_params() - # Custom tag start/end - with mktemp('<% if 1 %>1<% else %>2<% endif %>') as template: - self._testme(['--customize=resources/customize.py', template], '1') - - # Test: j2_environment() - # custom function: my_function - with mktemp('<< my_function("hey") >>') as template: - self._testme(['--customize=resources/customize.py', template], 'my function says "hey"') - - # Test: alter_context() - # Extra variable: ADD=127 - with mktemp('<< ADD >>') as template: - self._testme(['--customize=resources/customize.py', template], '127') - - # Test: extra_filters() - with mktemp('<< ADD|parentheses >>') as template: - self._testme(['--customize=resources/customize.py', template], '(127)') - - # Test: extra_tests() - with mktemp('<% if ADD|int is custom_odd %>odd<% endif %>') as template: - self._testme(['--customize=resources/customize.py', template], 'odd') - - # reset - # otherwise it will load the same module even though its name has changed - del sys.modules['customize-module'] - - # Test: no hooks in a file - # Got to restore to the original configuration and use {% %} again - with mktemp('{% if 1 %}1{% endif %}') as template: - self._testme(['--customize=render-test.py', template], '1') diff --git a/tests/resources/custom_tests.py b/tests/resources/custom_tests.py index 7292712..c3e6ef8 100644 --- a/tests/resources/custom_tests.py +++ b/tests/resources/custom_tests.py @@ -1,3 +1,3 @@ def custom_odd(n): - return True if (n % 2) else False + return bool(n % 2) diff --git a/tests/resources/customize.py b/tests/resources/customize.py index 21beab0..993a169 100644 --- a/tests/resources/customize.py +++ b/tests/resources/customize.py @@ -37,7 +37,7 @@ def j2_environment(env): :rtype: jinja2.environment.Environment """ env.globals.update( - my_function=lambda v: 'my function says "{}"'.format(v) + my_function='my function says "{}"'.format ) return env @@ -67,7 +67,7 @@ def extra_tests(): """ return dict( # Example: {% if a|int is custom_odd %}odd{% endif %} - custom_odd=lambda n: True if (n % 2) else False + custom_odd=lambda n: bool(n % 2) ) # {% endraw %} diff --git a/tests/resources/data.yml b/tests/resources/data.yml deleted file mode 100644 index efd4cee..0000000 --- a/tests/resources/data.yml +++ /dev/null @@ -1,4 +0,0 @@ -nginx: - hostname: localhost - webroot: /var/www/project - logs: /var/log/nginx/ diff --git a/tests/resources/data/badext_nginx_data_env.json b/tests/resources/data/badext_nginx_data_env.json new file mode 120000 index 0000000..46700b2 --- /dev/null +++ b/tests/resources/data/badext_nginx_data_env.json @@ -0,0 +1 @@ +nginx_data.env \ No newline at end of file diff --git a/tests/resources/data/badext_nginx_data_ini.json b/tests/resources/data/badext_nginx_data_ini.json new file mode 120000 index 0000000..433c79d --- /dev/null +++ b/tests/resources/data/badext_nginx_data_ini.json @@ -0,0 +1 @@ +nginx_data.ini \ No newline at end of file diff --git a/tests/resources/data/badext_nginx_data_json.ini b/tests/resources/data/badext_nginx_data_json.ini new file mode 120000 index 0000000..94052c7 --- /dev/null +++ b/tests/resources/data/badext_nginx_data_json.ini @@ -0,0 +1 @@ +nginx_data.json \ No newline at end of file diff --git a/tests/resources/data/badext_nginx_data_yaml.json b/tests/resources/data/badext_nginx_data_yaml.json new file mode 120000 index 0000000..6f587f5 --- /dev/null +++ b/tests/resources/data/badext_nginx_data_yaml.json @@ -0,0 +1 @@ +nginx_data.yaml \ No newline at end of file diff --git a/tests/resources/data.env b/tests/resources/data/nginx_data.env similarity index 100% rename from tests/resources/data.env rename to tests/resources/data/nginx_data.env diff --git a/tests/resources/data.ini b/tests/resources/data/nginx_data.ini similarity index 100% rename from tests/resources/data.ini rename to tests/resources/data/nginx_data.ini diff --git a/tests/resources/data.json b/tests/resources/data/nginx_data.json similarity index 100% rename from tests/resources/data.json rename to tests/resources/data/nginx_data.json diff --git a/tests/resources/data.yaml b/tests/resources/data/nginx_data.yaml similarity index 100% rename from tests/resources/data.yaml rename to tests/resources/data/nginx_data.yaml diff --git a/tests/resources/data/nginx_data.yml b/tests/resources/data/nginx_data.yml new file mode 120000 index 0000000..6f587f5 --- /dev/null +++ b/tests/resources/data/nginx_data.yml @@ -0,0 +1 @@ +nginx_data.yaml \ No newline at end of file diff --git a/tests/resources/data/nginx_data_env b/tests/resources/data/nginx_data_env new file mode 120000 index 0000000..46700b2 --- /dev/null +++ b/tests/resources/data/nginx_data_env @@ -0,0 +1 @@ +nginx_data.env \ No newline at end of file diff --git a/tests/resources/data/nginx_data_ini b/tests/resources/data/nginx_data_ini new file mode 120000 index 0000000..433c79d --- /dev/null +++ b/tests/resources/data/nginx_data_ini @@ -0,0 +1 @@ +nginx_data.ini \ No newline at end of file diff --git a/tests/resources/data/nginx_data_json b/tests/resources/data/nginx_data_json new file mode 120000 index 0000000..94052c7 --- /dev/null +++ b/tests/resources/data/nginx_data_json @@ -0,0 +1 @@ +nginx_data.json \ No newline at end of file diff --git a/tests/resources/data/nginx_data_yaml b/tests/resources/data/nginx_data_yaml new file mode 120000 index 0000000..6f587f5 --- /dev/null +++ b/tests/resources/data/nginx_data_yaml @@ -0,0 +1 @@ +nginx_data.yaml \ No newline at end of file diff --git a/tests/resources/name.j2 b/tests/resources/name.j2 deleted file mode 100644 index 6cb905f..0000000 --- a/tests/resources/name.j2 +++ /dev/null @@ -1 +0,0 @@ -Hello {{name}}! diff --git a/tests/resources/out/nginx-env.conf b/tests/resources/out/nginx-env.conf new file mode 120000 index 0000000..4ed253b --- /dev/null +++ b/tests/resources/out/nginx-env.conf @@ -0,0 +1 @@ +nginx.conf \ No newline at end of file diff --git a/tests/resources/out/nginx.conf b/tests/resources/out/nginx.conf new file mode 100644 index 0000000..00547e1 --- /dev/null +++ b/tests/resources/out/nginx.conf @@ -0,0 +1,10 @@ +server { + listen 80; + server_name localhost; + + root /var/www/project; + index index.htm; + + access_log /var/log/nginx//http.access.log combined; + error_log /var/log/nginx//http.error.log; +} diff --git a/tests/resources/nginx-env.j2 b/tests/resources/tpl/nginx-env.conf.j2 similarity index 100% rename from tests/resources/nginx-env.j2 rename to tests/resources/tpl/nginx-env.conf.j2 diff --git a/tests/resources/nginx.j2 b/tests/resources/tpl/nginx.conf.j2 similarity index 100% rename from tests/resources/nginx.j2 rename to tests/resources/tpl/nginx.conf.j2 diff --git a/tests/tba b/tests/tba new file mode 100644 index 0000000..a00a6fc --- /dev/null +++ b/tests/tba @@ -0,0 +1,81 @@ + def rest_filters__env(self): + pass + #with mktemp('user_login: kolypto') as yml_file: + #with mktemp('{{ user_login }}:{{ "USER_PASS"|env }}') as template: + # Test: template with an env variable + #self._testme(['--format=yaml', template, yml_file], 'kolypto:qwerty123', env=dict(USER_PASS='qwerty123')) + + # environment cleaned up + #assert 'USER_PASS' not in os.environ + + # Test: KeyError + #with self.assertRaises(KeyError): + #self._testme(['--format=yaml', template, yml_file], 'kolypto:qwerty123', env=dict()) + + # Test: default + #with mktemp('{{ user_login }}:{{ "USER_PASS"|env("-none-") }}') as template: + #self._testme(['--format=yaml', template, yml_file], 'kolypto:-none-', env=dict()) + + # Test: using as a function + #with mktemp('{{ user_login }}:{{ env("USER_PASS") }}') as template: + #self._testme(['--format=yaml', template, yml_file], 'kolypto:qwerty123', env=dict(USER_PASS='qwerty123')) + + #with self.assertRaises(KeyError): + # Variable not set + #self._testme(['--format=yaml', template, yml_file], '', env=dict()) + + # Test: using as a function, with a default + #with mktemp('{{ user_login }}:{{ env("USER_PASS", "-none-") }}') as template: + #self._testme(['--format=yaml', template, yml_file], 'kolypto:qwerty123', env=dict(USER_PASS='qwerty123')) + #self._testme(['--format=yaml', template, yml_file], 'kolypto:-none-', env=dict()) + + + def rest_custom_filters(self): + pass + #with mktemp('{{ a|parentheses }}') as template: + #self._testme(['--format=env', '--filters=resources/custom_filters.py', template], '(1)', env=dict(a='1')) + + #def rest_custom_tests(self): + #with mktemp('{% if a|int is custom_odd %}odd{% endif %}') as template: + #self._testme(['--format=env', '--tests=resources/custom_tests.py', template], 'odd', env=dict(a='1')) + + def rest_jinja2_extensions(self): + """ Test that an extension is enabled """ + #with mktemp('{% do [] %}') as template: + ## `do` tag is an extension + #self._testme([template], '') + + + def rest_customize(self): + """ Test --customize """ + # Test: j2_environment_params() + # Custom tag start/end + #with mktemp('<% if 1 %>1<% else %>2<% endif %>') as template: + #self._testme(['--customize=resources/customize.py', template], '1') + + # Test: j2_environment() + # custom function: my_function + #with mktemp('<< my_function("hey") >>') as template: + #self._testme(['--customize=resources/customize.py', template], 'my function says "hey"') + + # Test: alter_context() + # Extra variable: ADD=127 + #with mktemp('<< ADD >>') as template: + #self._testme(['--customize=resources/customize.py', template], '127') + + # Test: extra_filters() + #with mktemp('<< ADD|parentheses >>') as template: + #self._testme(['--customize=resources/customize.py', template], '(127)') + + # Test: extra_tests() + #with mktemp('<% if ADD|int is custom_odd %>odd<% endif %>') as template: + #self._testme(['--customize=resources/customize.py', template], 'odd') + + # reset + # otherwise it will load the same module even though its name has changed + #del sys.modules['customize-module'] + + # Test: no hooks in a file + # Got to restore to the original configuration and use {% %} again + #with mktemp('{% if 1 %}1{% endif %}') as template: + #self._testme(['--customize=render-test.py', template], '1') diff --git a/tests/tba2 b/tests/tba2 new file mode 100644 index 0000000..7aa69f5 --- /dev/null +++ b/tests/tba2 @@ -0,0 +1,33 @@ + def rest_import_env(self): + pass + # Import environment into a variable + #with mktemp('{{ a }}/{{ env.B }}') as template: + #with mktemp('{"a":1}') as context: + #self._testme(['--format=json', '--import-env=env', template, context], '1/2', env=dict(B='2')) + # Import environment into global scope + #with mktemp('{{ a }}/{{ B }}') as template: + #with mktemp('{"a":1,"B":1}') as context: + #self._testme(['--format=json', '--import-env=', template, context], '1/2', env=dict(B='2')) + + def rest_env_file__equals_sign_in_value(self): + pass + # Test whether environment variables with "=" in the value are parsed correctly + #with mktemp('{{ A|default('') }}/{{ B }}/{{ C }}') as template: + #with mktemp('A\nB=1\nC=val=1\n') as context: + #self._testme(['--format=env', template, context], '/1/val=1') + + def rest_unicode(self): + pass + # Test how unicode is handled + # I'm using Russian language for unicode :) + #with mktemp('Проверка {{ a }} связи!') as template: + #with mktemp('{"a": "широкополосной"}') as context: + #self._testme(['--format=json', template, context], 'Проверка широкополосной связи!') + + # Test case from issue #17: unicode environment variables + #if sys.version_info[0] == 2: + # Python 2: environment variables are bytes + #self._testme(['resources/tpl/name.j2'], u'Hello Jürgen!\n', env=dict(name=b'J\xc3\xbcrgen')) + #else: + # Python 3: environment variables are unicode strings + #self._testme(['resources/tpl/name.j2'], u'Hello Jürgen!\n', env=dict(name=u'Jürgen')) diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..7222a89 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,281 @@ +# -*- coding: utf-8 -*- +import importlib +import logging +import os +import shlex +import sys +import tempfile +import unittest +from contextlib import contextmanager +from pathlib import Path +from typing import Iterable + +from jj2cli.cli import render_command +from jj2cli.parsers import DATASPEC_SEP + + +@contextmanager +def stdin_from(fin, *args, **kwargs): + """Use the contents of the specified file as stdin, and yield the original stdin.""" + if not isinstance(fin, Path): + fin = Path(fin) + stdin_bak = sys.stdin + sys.stdin = fin.open(*args, **kwargs) + logging.debug("STDIN from: %s", fin) + try: + yield + finally: + sys.stdin.close() + sys.stdin = stdin_bak + +@contextmanager +def temp_file(contents, suffix=None, text=True): + """Create a temporary file with the specified contents, and yield its path.""" + fd, f = tempfile.mkstemp(suffix=suffix, text=text) + fp = os.fdopen(fd, 'w') + fp.write(contents) + fp.close() + f = Path(f) + logging.debug("TEMP created: %s", f) + try: + yield f + finally: + f.unlink() + +@contextmanager +def temp_files(specs): + """Create multiple temporary files and yield their paths.""" + tempfiles = [] + for spec in specs: + if len(spec) == 3: + contents, suffix, text = spec + elif len(spec) == 2: + contents, suffix, text = spec + (True,) + else: + raise ValueError("Bad spec for temp file: %s" % repr(spec)) + fd, f = tempfile.mkstemp(suffix=suffix, text=text) + fp = os.fdopen(fd, 'w') + fp.write(contents) + fp.close() + f = Path(f) + tempfiles.append(f) + logging.debug("TEMP created: %s", f) + try: + yield tuple(tempfiles) + finally: + map(Path.unlink, tempfiles) + +@contextmanager +def environment(env): + """Temporarily set values from env in the environment.""" + env_bak = os.environ + os.environ = env_bak.copy() + os.environ.update(env) + try: + yield + finally: + os.environ = env_bak + +class RenderTest(unittest.TestCase): + WORKDIR = Path(__file__).parent + TPLDIR = Path('resources') / 'tpl' + DATADIR = Path('resources') / 'data' + OUTDIR = Path('resources') / 'out' + + def setUp(self): + os.chdir(self.WORKDIR) + + def _render_prep(self, tpl, data, expected_output, extra_args): + """ Helper for processing common options for test runners. + + data paths are expected to be relative + """ + tpl = self.TPLDIR / tpl + _data = data if isinstance(data, Iterable) and not isinstance(data, str) else [data] + data = [] + for dspec in _data: + dspec = str(dspec) if isinstance(dspec, Path) else dspec + p, sep, modifiers = dspec.partition(DATASPEC_SEP) + p = str(self.DATADIR / p) if (self.DATADIR / p).is_file() else p + data.append('%s%s%s' % (p, sep, modifiers)) + expected_output = (Path(expected_output) + if expected_output is not None + else self.OUTDIR / tpl.stem).read_text() + extra_args = [] if not extra_args else shlex.split(extra_args) + + return (str(tpl), data, expected_output, extra_args) + + def _class_for_name(self, fullcname): + """ Helper for getting a class object from its string representation. + """ + mname, _, cname = fullcname.rpartition('.') + try: + m = importlib.import_module(mname if mname else 'builtins') + c = getattr(m, cname) + return c + except (ImportError, AttributeError): + return None + + # pylint: disable=too-many-arguments + def _render_test(self, tpl, data=None, expected_output=None, + extra_args=None, exception=None, exception_msg=None): + """ Helper for rendering `tpl` using `data` and checking the results + against `expected_results`. Rendering is expected to succeed + without errors. + """ + tpl, data, expected_output, extra_args = self._render_prep( + tpl, data, expected_output, extra_args) + argv = ['dummy_command_name', *extra_args, tpl, *data] + logging.debug("PASSED_ARGS render_command: %s", argv) + + if exception is None: + result = render_command(argv) + if isinstance(result, bytes): + # XXX: maybe render_command() should just return utf-8? + result = result.decode('utf-8') + self.assertEqual(result, expected_output) + elif exception_msg is None: + c = self._class_for_name(exception) + self.assertRaises(c, render_command, argv) + else: + c = self._class_for_name(exception) + self.assertRaisesRegex(c, exception_msg, render_command, argv) + # pylint: enable=too-many-arguments + + def test_ENV(self): + """ Tests rendering with environment variables. + """ + with environment({"MYVAR": "test"}), temp_files(( + ("XXX{{ MYVAR }}XXX", ".j2"), + ("MYVAR=bad", ".env"), + ("XXXtestXXX", ".out"), + )) as (tpl, in_ignored, out_normal): + self._render_test(tpl, ":ENV", out_normal) + self._render_test(tpl, "-:ENV", out_normal, extra_args='--') + self._render_test(tpl, "%s:ENV" % (in_ignored), out_normal) + + def test_env(self): + """ Tests rendering with a single data file in env format. + """ + # simple render + self._render_test("nginx-env.conf.j2", "nginx_data.env") + # file + fallback format + self._render_test("nginx-env.conf.j2", "nginx_data_env", extra_args='--fallback-format=env') + # file + format override + self._render_test("nginx-env.conf.j2", "badext_nginx_data_env.json:env") + # stdin + fallback format + with stdin_from(self.DATADIR / "nginx_data_env"): + self._render_test("nginx-env.conf.j2", "-", extra_args='--fallback-format=env') + # stdin + format override + with stdin_from(self.DATADIR / "nginx_data_env"): + self._render_test("nginx-env.conf.j2", ":env") + with stdin_from(self.DATADIR / "nginx_data_env"): + self._render_test("nginx-env.conf.j2", "-:env", extra_args='--') + # file + default fallback format - failure + self._render_test("nginx.conf.j2", "nginx_data_env", + exception='configparser.MissingSectionHeaderError', + exception_msg='no section headers') + + def test_ini(self): + """ Tests rendering with a single data file in ini format. + """ + # simple render + self._render_test("nginx.conf.j2", "nginx_data.ini") + # file + fallback format + self._render_test("nginx.conf.j2", "nginx_data_ini", extra_args='--fallback-format=ini') + # file + format override + self._render_test("nginx.conf.j2", "badext_nginx_data_ini.json:ini") + # stdin + fallback format + with stdin_from(self.DATADIR / "nginx_data_ini"): + self._render_test("nginx.conf.j2", "-", extra_args='--fallback-format=ini') + # stdin + format override + with stdin_from(self.DATADIR / "nginx_data_ini"): + self._render_test("nginx.conf.j2", ":ini") + with stdin_from(self.DATADIR / "nginx_data_ini"): + self._render_test("nginx.conf.j2", "-:ini", extra_args='--') + # file + default fallback format - success + self._render_test("nginx.conf.j2", "nginx_data_ini") + + def test_json(self): + """ Tests rendering with a single data file in json format. + """ + # simple render + self._render_test("nginx.conf.j2", "nginx_data.json") + # file + fallback format + self._render_test("nginx.conf.j2", "nginx_data_json", extra_args='--fallback-format=json') + # file + format override + self._render_test("nginx.conf.j2", "badext_nginx_data_json.ini:json") + # stdin + fallback format + with stdin_from(self.DATADIR / "nginx_data_json"): + self._render_test("nginx.conf.j2", "-", extra_args='--fallback-format=json') + # stdin + format override + with stdin_from(self.DATADIR / "nginx_data_json"): + self._render_test("nginx.conf.j2", ":json") + with stdin_from(self.DATADIR / "nginx_data_json"): + self._render_test("nginx.conf.j2", "-:json", extra_args='--') + # file + default fallback format - failure + self._render_test("nginx.conf.j2", "nginx_data_json", + exception='configparser.MissingSectionHeaderError', + exception_msg='no section headers') + + def test_yaml(self): + """ Tests rendering with a single data file in yaml format. + """ + try: + importlib.import_module('yaml') + except ImportError: + raise unittest.SkipTest('yaml module not available') + # simple render + self._render_test("nginx.conf.j2", "nginx_data.yaml") + self._render_test("nginx.conf.j2", "nginx_data.yml") + # file + fallback format + self._render_test("nginx.conf.j2", "nginx_data_yaml", extra_args='--fallback-format=yaml') + # file + format override + self._render_test("nginx.conf.j2", "badext_nginx_data_yaml.json:yaml") + # stdin + fallback format + with stdin_from(self.DATADIR / "nginx_data_yaml"): + self._render_test("nginx.conf.j2", "-", extra_args='--fallback-format=yaml') + # stdin + format override + with stdin_from(self.DATADIR / "nginx_data_yaml"): + self._render_test("nginx.conf.j2", ":yaml") + with stdin_from(self.DATADIR / "nginx_data_yaml"): + self._render_test("nginx.conf.j2", "-:yaml", extra_args='--') + # file + default fallback format - failure + self._render_test("nginx.conf.j2", "nginx_data_yaml", + exception='configparser.MissingSectionHeaderError', + exception_msg='no section headers') + + def test_ignore_missing(self): + """ Tests the -I/--ignore missing flag. + """ + self._render_test("nginx.conf.j2", ["nginx_data_json", "nginx_data_missing"], + exception='FileNotFoundError', + exception_msg='nginx_data_missing', + extra_args='-f json') + self._render_test("nginx.conf.j2", ["nginx_data_json", "nginx_data_missing"], + extra_args='-I -f json') + self._render_test("nginx.conf.j2", ["nginx_data_json", "nginx_data_missing"], + extra_args='--ignore-missing -f json') + + def test_undefined(self): + """ Tests the -U/--undefined flag. + """ + with temp_files(( + ("XXX{{ undefined_var }}XXX", ".j2"), + ("{}", ".json"), + ("XXXXXX", ".out"), + ("XXX{{ undefined_var }}XXX", ".out"), + )) as (tpl, data, out_normal, out_debug): + # default (strict) + self._render_test(tpl, data, out_normal, + exception='jinja2.exceptions.UndefinedError', + exception_msg='undefined_var') + # strict + self._render_test(tpl, data, out_normal, + exception='jinja2.exceptions.UndefinedError', + exception_msg='undefined_var', + extra_args='--undefined strict') + # normal + self._render_test(tpl, data, out_normal, extra_args='--undefined normal') + # debug + self._render_test(tpl, data, out_debug, extra_args='--undefined debug') diff --git a/tox.ini b/tox.ini index b7db8b5..73627e0 100644 --- a/tox.ini +++ b/tox.ini @@ -1,26 +1,39 @@ [tox] -envlist=py{27,34,35,36,37},pypy, - py36-pyyaml5.1 - py36-pyyaml3.13 - py36-pyyaml3.12 - py36-pyyaml3.11 - py36-pyyaml3.10 +envlist= + py{3.10, 3.11, 3.12}-pyyaml{0,6} skip_missing_interpreters=True [testenv] +# See: https://tox.wiki/en/latest/config.html#tox-ini +basepython= + py3.10: python3.10 + py3.11: python3.11 + py3.12: python3.12 deps= - -rrequirements-dev.txt - py{27,34,35,36},pypy: -e.[yaml] - py37: pyyaml - py36-pyyaml5.1: pyyaml==5.1 - py36-pyyaml3.13: pyyaml==3.13 - py36-pyyaml3.12: pyyaml==3.12 - py36-pyyaml3.11: pyyaml==3.11 - py36-pyyaml3.10: pyyaml==3.10 + pyyaml0: null + pyyaml6: pyyaml~=6.0 +allowlist_externals = + poetry + pytest commands= - nosetests {posargs:tests/} -whitelist_externals=make - -[testenv:dev] -deps=-rrequirements-dev.txt + poetry install + pytest {posargs:tests/} usedevelop=True + +#poetry install --no-interaction +#poetry run pytest {posargs:tests/} + +# NB: matrix should extrapolate to poetry with/without settings rather than specific versions + +#[testenv:lint] +#basepython= +# py{3.10}-pyyaml{6} +#deps= +# -rtests/requirements.txt +# pyyaml6: pyyaml>=6 +#commands= +# prospector + +#[testenv:dev] +#deps= +#-rtest/requirements.txt