From 7a7a1ae798de4fae8f0f67bddce728648ea8896c Mon Sep 17 00:00:00 2001 From: Guillaume Viger Date: Mon, 2 Jun 2025 13:25:10 -0400 Subject: [PATCH] docs-reorg: refactor Install section --- docs/community/code/package-development.md | 2 +- docs/install/build-setup-run.md | 173 ++++++++------- docs/install/cli.md | 40 ++-- docs/install/destroy.md | 38 ---- docs/install/{run.md => explore.md} | 53 ++--- .../jaycee-xie-unsplash-shiba.png | Bin docs/install/index.md | 205 ++++++++++-------- docs/install/{scaffold.md => initialize.md} | 53 +++-- docs/install/requirements.md | 34 +-- docs/install/stop.md | 48 ++++ docs/install/troubleshoot.md | 58 ++--- docs/{install => operate/code}/debugging.md | 84 +++---- .../code/imgs}/console_log_output.png | Bin .../code/imgs}/debugger_output.png | Bin docs/operate/customize/users.md | 10 +- docs/operate/index.md | 2 + docs/operate/ops/deploy.md | 2 +- docs/operate/ops/logging.md | 41 +++- docs/{install => operate/ops}/migrate.md | 2 +- .../{install/img => reference/imgs}/venvs.svg | 0 docs/{install => reference}/virtualenvs.md | 2 +- mkdocs.yml | 24 +- 22 files changed, 470 insertions(+), 401 deletions(-) delete mode 100644 docs/install/destroy.md rename docs/install/{run.md => explore.md} (95%) rename docs/install/{img => imgs}/jaycee-xie-unsplash-shiba.png (100%) rename docs/install/{scaffold.md => initialize.md} (73%) create mode 100644 docs/install/stop.md rename docs/{install => operate/code}/debugging.md (97%) rename docs/{install/img => operate/code/imgs}/console_log_output.png (100%) rename docs/{install/img => operate/code/imgs}/debugger_output.png (100%) rename docs/{install => operate/ops}/migrate.md (98%) rename docs/{install/img => reference/imgs}/venvs.svg (100%) rename docs/{install => reference}/virtualenvs.md (99%) diff --git a/docs/community/code/package-development.md b/docs/community/code/package-development.md index 25d3a02b..d49aeeaf 100644 --- a/docs/community/code/package-development.md +++ b/docs/community/code/package-development.md @@ -16,7 +16,7 @@ editable install of the Python package: !!! note ``mkvirtualenv`` is a tool provided by virtualenv-wrapper to manage Python - virtualenvs. See [Python virtual environments](../../install/virtualenvs.md) + virtualenvs. See [Python virtual environments](../../reference/virtualenvs.md) ```bash cd ~/src/invenio-app-rdm diff --git a/docs/install/build-setup-run.md b/docs/install/build-setup-run.md index 4e188df7..642ad8b0 100644 --- a/docs/install/build-setup-run.md +++ b/docs/install/build-setup-run.md @@ -1,32 +1,35 @@ # Build, setup and run -Now, that we have created a project folder, we need to build and set up the InvenioRDM application. +Now that we have created a project folder, we need to build and set up the InvenioRDM application. -The application can be built and installed in two different ways: +As mentioned before, the application can be built and installed in two different ways: -- Local (good for developers or for customizing your instance): The application is installed locally on your machine in a Python virtual environment managed by the ``pipenv`` tool. -- Container (good for quick preview): The application is built inside a Docker image. +- *Local development* (good for developers or for customizing your instance): The application is installed directly on your machine in a Python [virtual environment](../reference/virtualenvs.md) managed by the ``pipenv`` tool (its services are containerized though). +- *Containerized preview* (good for quick preview): The application is built inside a Docker image and, it, along with the services are all containerized. -## For the impatient +## Condensed version For the impatient, here are the commands to build, setup and run InvenioRDM. For clarity, we have decomposed them into individual steps in the guide below. Follow the guide below for step-by-step details: -Container: +=== "Local development" -```bash -invenio-cli containers start --lock --build --setup -``` + ```shell + # Install Python and Javascript packages + invenio-cli install + # Set up containerized database, cache, OpenSearch, etc. + invenio-cli services setup + # Serve the application locally through a development server + invenio-cli run + ``` -Local: +=== "Containerized preview" -```bash -invenio-cli install -invenio-cli services setup -invenio-cli run -``` + ```shell + invenio-cli containers start --lock --build --setup + ``` !!! info - If you run the above commands, you're all set for this section. If you like to learn more about the build process, + If you run the above commands, you're all set for this section. If you would like to learn more about the build process, then follow the steps below instead. @@ -34,9 +37,9 @@ invenio-cli run First, we start by locking the Python dependencies, which ensures that you will always have the same versions of dependencies if you reinstall in the future. Locked dependencies are important for having reproducible installs. -This step will normally be executed automatically by both the local and container installation options, but here we run it explicitly. +This step is executed automatically by both the local development (under `invenio-cli install`) and containerized preview (`--lock` option) installation options, but here we run it explicitly. -Let's try it: +It's the same command for either installation option. Let's try it: ```bash cd my-site/ @@ -70,49 +73,43 @@ Dependencies locked successfully. A new file ``Pipfile.lock`` has now been created with the locked dependencies. -Next, choose option 1 or option 2 for your preferred installation method. +Next, follow the *local development* option or *containerized preview* option according to your preferred installation method. -## Option 1: Container install +## Local development option -The container install is good for a quick preview, or if you don't want all dependencies locally. It is not good for development or for customizing your instance, as it requires you to rebuild the Docker image for every change which can be time consuming. +The local install is good for developers or if you like to customize InvenioRDM as it avoids the waiting time for building a new Docker image. For instance, changing the layout will be much faster with a local install. ### Build -For the container build, you first build the Docker application image using -the following command: +The local build steps involve installing all the Python dependencies into a local Python virtual environment, as well as all the Javascript dependencies. The Javascript dependencies are defined in the Python packages. This is done with the command: ```bash -invenio-cli containers build +invenio-cli install ``` ```console -Building images... Pull newer versions True, use cache True -Checking if dependencies are locked. -Dependencies are locked -Building images... +Installing python dependencies... Please be patient, this operation might take some time... +Installing dependencies from Pipfile.lock (271a6d)… + 🐍 β–‰β–‰β–‰β–‰β–‰β–‰β–‰β–‰β–‰β–‰β–‰β–‰β–‰β–‰β–‰β–‰β–‰β–‰β–‰β–‰β–‰β–‰β–‰β–‰β–‰β–‰β–‰β–‰β–‰β–‰β–‰β–‰ 200/200 β€” 00:02:00 [...] -Successfully built f1fc95ce1037 -Successfully tagged my-site:latest -Images built successfully. +Built webpack project. +Assets and statics updated. +Dependencies installed successfully. ``` -The command will: +The command does the following: -- Download the Docker images for the services (database, cache, search engine, etc.) -- Build the Docker image for the InvenioRDM application using the ``Dockerfile`` in your project. - Install Python dependencies (according the ``Pipfile.lock``) - Install JavaScript dependencies - Build the JavaScript/CSS web assets ### Setup -Once the Docker application image has been built, we need to initialize the database, the indices and so on. For this, we use again the ``containers`` command. - -The first time this command is run, the services will be setup correctly and the containers running them will even restart upon a reboot of your machine. If you stop and restart those containers, your data will still be there. Upon running this command again, the initial setup is skipped. +We need to initialize the database, the indices and so on. For this, we use the services command. The first time this command is run, the services will be setup correctly and the containers running them will even restart upon a reboot of your machine. If you stop and restart those containers, your data will still be there. Upon running this command again, the initial setup is skipped. ```bash -invenio-cli containers setup +invenio-cli services setup ``` ```console Making sure containers are up... @@ -132,71 +129,83 @@ Creating indexes... Putting templates... ``` -The command will: - -- Create the database and Elasticsearch indexes. -- Load fixtures data into InvenioRDM. -- Create demo records. - !!! tip You can skip the creation of demo records by using the ``--no-demo-data`` option (``-N`` for short): + ```shell + invenio-cli services setup --no-demo-data ``` - invenio-cli containers setup --no-demo-data + + You can forcefully redo the setup step with the ``--force`` option (``-f`` for short): + + ```shell + invenio-cli services setup --force ``` ### Run -You can now run the application container and related services: +You can now run the application locally: ```bash -invenio-cli containers start +invenio-cli run ``` -Go and explore your InvenioRDM instance at [https://127.0.0.1](https://127.0.0.1). +Go and explore your InvenioRDM instance at [https://127.0.0.1:5000](https://127.0.0.1:5000). This is the default host and port configured in ``invenio.cfg``. -!!! warning "Visit 127.0.0.1, not localhost" - Due to Content Security Policy (CSP) headers it is important that you visit ``127.0.0.1``, and not ``localhost`` unless you set ``INVENIO_SITE_UI_URL`` and ``INVENIO_SITE_API_URL`` to ``https://localhost`` and ``https://localhost/api`` respectively. -!!! tip - You can provide other configuration variables by setting them as environment variables with the ``INVENIO_`` prefix. +!!! tip "Change the host and port" + By default, the host is `127.0.0.1` and the port is `5000`. Pass `--host` and `--port` + to change them: -## Option 2: Local install + `invenio-cli run --host 0.0.0.0 --port 443` -The local install is good for developers or if you like to customize InvenioRDM as it avoids the waiting time for building a new Docker image. For instance, changing the layout will be much faster with a local install. + It's a development server though, so don't use it for production. + +!!! warning "Visit 127.0.0.1, not localhost" + Due to Content Security Policy (CSP) headers it is important that you use ``127.0.0.1``, and not ``localhost``. + +## Containerized preview option + +The container install is good for a quick preview, or if you don't want all dependencies locally. It is not good for development or for customizing your instance, as it requires you to rebuild the Docker image for every change which can be time consuming. ### Build -The local build steps involve installing all the Python dependencies into a local Python virtual environment. This is done with the command (the virtualenv is managed by ``pipenv``): +For the container build, you first build the Docker application image using +the following command: ```bash -invenio-cli install +invenio-cli containers build ``` ```console -Installing python dependencies... Please be patient, this operation might take some time... -Installing dependencies from Pipfile.lock (271a6d)… - 🐍 β–‰β–‰β–‰β–‰β–‰β–‰β–‰β–‰β–‰β–‰β–‰β–‰β–‰β–‰β–‰β–‰β–‰β–‰β–‰β–‰β–‰β–‰β–‰β–‰β–‰β–‰β–‰β–‰β–‰β–‰β–‰β–‰ 200/200 β€” 00:02:00 +Building images... Pull newer versions True, use cache True +Checking if dependencies are locked. +Dependencies are locked +Building images... [...] -Built webpack project. -Assets and statics updated. -Dependencies installed successfully. +Successfully built f1fc95ce1037 +Successfully tagged my-site:latest +Images built successfully. ``` -Similar to the ``containers`` command, the command does the following: +The command will: +- Download the Docker images for the services (database, cache, search engine, etc.) +- Build the Docker image for the InvenioRDM application using the ``Dockerfile`` in your project. - Install Python dependencies (according the ``Pipfile.lock``) - Install JavaScript dependencies - Build the JavaScript/CSS web assets ### Setup -We need to initialize the database, the indices and so on. For this, we use the services command. The first time this command is run, the services will be setup correctly and the containers running them will even restart upon a reboot of your machine. If you stop and restart those containers, your data will still be there. Upon running this command again, the initial setup is skipped. +Once the Docker application image has been built, we need to initialize the database, the indices and so on. For this, we use again the ``containers`` command. + +The first time this command is run, the services will be setup correctly and the containers running them will even restart upon a reboot of your machine. If you stop and restart those containers, your data will still be there. Upon running this command again, the initial setup is skipped. ```bash -invenio-cli services setup +invenio-cli containers setup ``` ```console Making sure containers are up... @@ -216,35 +225,41 @@ Creating indexes... Putting templates... ``` +The command will: + +- Create the database and Elasticsearch indexes. +- Load fixtures data into InvenioRDM. +- Create demo records. + !!! tip You can skip the creation of demo records by using the ``--no-demo-data`` option (``-N`` for short): - ``` - invenio-cli services setup --no-demo-data + ```shell + invenio-cli containers setup --no-demo-data ``` + You can forcefully redo the setup step with the ``--force`` option (``-f`` for short): + + ```shell + invenio-cli containers setup --force + ``` ### Run -You can now run the application locally: +You can now run the application container and related services: ```bash -invenio-cli run +invenio-cli containers start ``` -Go and explore your InvenioRDM instance at [https://127.0.0.1:5000](https://127.0.0.1:5000). This is the default host and port configured in ``invenio.cfg``. - -!!! warning "Visit 127.0.0.1, not localhost" - Due to Content Security Policy (CSP) headers it is important that you use ``127.0.0.1``, and not ``localhost``. - -!!! tip "Change the host and port" - By default, the host is `127.0.0.1` and the port is `5000`. Pass `--host` and `--port` - to change them e.g.: +Go and explore your InvenioRDM instance at [https://127.0.0.1](https://127.0.0.1). - `invenio-cli run --host 0.0.0.0 --port 443` +!!! tip + You can provide other configuration variables by setting them as environment variables with the ``INVENIO_`` prefix. - It's a development server, so don't use it for production. +!!! warning "Visit 127.0.0.1, not localhost" + Due to Content Security Policy (CSP) headers it is important that you visit ``127.0.0.1``, and not ``localhost`` unless you set ``INVENIO_SITE_UI_URL`` and ``INVENIO_SITE_API_URL`` to ``https://localhost`` and ``https://localhost/api`` respectively. ## Troubleshooting diff --git a/docs/install/cli.md b/docs/install/cli.md index d6e87014..eea72407 100644 --- a/docs/install/cli.md +++ b/docs/install/cli.md @@ -1,39 +1,41 @@ -# Install CLI +# Install the command line tool -InvenioRDM comes with a CLI management tool called Invenio-CLI which is used to manage and work with your local installation. +InvenioRDM is set up with a command line management tool, `invenio-cli`, which is used to manage and work with your local installation. -You can install the Invenio CLI package named `invenio-cli`. The package is available on [PyPI](https://pypi.org/project/invenio-cli/). Use your favorite way to install a Python package: +## Installation -Via pip: +`invenio-cli` is available on [PyPI](https://pypi.org/project/invenio-cli/). Use your favorite way to install it: -```bash -pip install invenio-cli -``` +=== "pip" -Via pipenv: + ```shell + pip install invenio-cli + ``` -```bash -pipenv install invenio-cli -``` +=== "uv" -Via pipx: + ```shell + uv tool install invenio-cli + ``` -```bash -pipx install invenio-cli -``` +=== "pipx" + + ```shell + pipx install invenio-cli + ``` -To make sure you've installed successfully: +To make sure you've installed it successfully: ```bash invenio-cli --version ``` -You'll find the latest released version number on [PyPi](https://pypi.org/project/invenio-cli/). +You'll find the latest released version number on [PyPi](https://pypi.org/project/invenio-cli/). Each new version of InvenioRDM is accompanied by a new version of `invenio-cli` if only to have it select the new version as the default when initializing a project. -### Commands reference +## Commands reference For a full reference of available commands, see the [CLI reference](../reference/cli.md) !!! tip "Shell tab completion" - Invenio-CLI has support for Shell tab completion of commands. See [Shell completion](../reference/cli.md#shell-completion). + `invenio-cli` has support for shell tab completion of commands. See [shell completion](../reference/cli.md#shell-completion). diff --git a/docs/install/destroy.md b/docs/install/destroy.md deleted file mode 100644 index 180433f0..00000000 --- a/docs/install/destroy.md +++ /dev/null @@ -1,38 +0,0 @@ -# Stop or destroy your instance - -## Stoping containers - -If you want to temporarily stop the instance without losing the data that -was generated, you can use the `stop` command: - -```bash -invenio-cli containers stop -# or -invenio-cli services stop -``` - -## Destroying containers - -!!! warning - The ``destroy`` command WILL permanently erase all your data in the Docker containers. - - -On the other hand, if you wish to clean up and delete all Docker artefacts, -you can use the `destroy` command. It removes all containers, images and volumes. - -Note that `destroy` will also `stop` the containers, so there is no need to run the previous command: - - -```bash -invenio-cli containers destroy -# or -invenio-cli services destroy -``` - -## Destroying all - -If you also want to destroy a pipenv managed Python virtual environment along with the containers you can use the global `destroy command`: - -```bash -invenio-cli destroy -``` diff --git a/docs/install/run.md b/docs/install/explore.md similarity index 95% rename from docs/install/run.md rename to docs/install/explore.md index 708d0836..f6a06861 100644 --- a/docs/install/run.md +++ b/docs/install/explore.md @@ -1,18 +1,18 @@ -# Use InvenioRDM +# Explore InvenioRDM -!!! warning "Use the specified host and port" - Due to CSP, it is important that you use the host and port specified when - running the API calls. This means `localhost` and `127.0.0.1` are not - interchangable as they usually are. The default is `127.0.0.1:5000` for the - development installation and `127.0.0.1` for the containerized installation. Here we - show examples with the development installation. +## Use your browser -### Use your browser +For the local development installation, navigate to [https://127.0.0.1:5000](https://127.0.0.1:5000). For the containerized preview, navigate to [https://127.0.0.1](https://127.0.0.1). Note that you might need to accept the SSL exception since it's using a test certificate. For the rest of this guide we assume a local development installation. -Navigate to [https://127.0.0.1:5000](https://127.0.0.1:5000). Note that you might need to accept the SSL exception since it's using a test certificate. Below, we list a small subset of the common API calls. For more, see the [API reference section](../reference/rest_api_index.md). -### List records +!!! warning "Use the specified host and port" + Due to CSP, it is important that you use the correct host and port specified when + running the API calls. This means `localhost` and `127.0.0.1` are not + interchangable as they usually are. The default is `127.0.0.1:5000` for the + development installation and `127.0.0.1` for the containerized installation. + +## List records Let's see what is in the instance by querying the API. Using another terminal: @@ -315,7 +315,7 @@ curl -k -XGET https://127.0.0.1:5000/api/records | python3 -m json.tool } ``` -#### Notes +**Notes** - The output is shortened for readability and your records will be different because they are generated randomly. - You can use [jq](https://github.com/stedolan/jq) for color highlighting: @@ -324,13 +324,13 @@ curl -k -XGET https://127.0.0.1:5000/api/records | python3 -m json.tool curl -k -XGET https://127.0.0.1:5000/api/records | jq . ``` -### Create records +## Create records !!! info "Authentication required" All requests to the create-related REST API endpoints require authentication. To create a token, login to the application, click on your user -> `Applications` -> `New token`, enter a name and click on `Create`. Copy the token, since it will not be displayed again. -#### Create a draft +### Create a draft You can create a new record **draft** using the API: @@ -368,7 +368,7 @@ curl -k -XPOST -H "Content-Type: application/json" -H "Authorization: Bearer " https://127.0.0.1:5000/api/records/jnmmp-51n47/draft/actions/publish ``` -### Get a record +## Get a record To confirm the record is published, you can search for it: @@ -402,19 +402,20 @@ To confirm the record is published, you can search for it: curl -k -XGET https://127.0.0.1:5000/api/records?q=Gladiator | python3 -m json.tool ``` -or access it directly using the id from the publish response (e.g. `90xv7-xwd20`): +or access it directly using the id from the publish response (e.g., `90xv7-xwd20`): ``` bash curl -k -XGET https://127.0.0.1:5000/api/records/90xv7-xwd20 | python3 -m json.tool ``` -### Upload a file to a record +## Upload a file to a record For demonstration purposes, we will attach this scientific photo to a record: -![Very scientific picture of a shiba in autumn](img/jaycee-xie-unsplash-shiba.png) - +![Very scientific picture of a shiba in autumn](imgs/jaycee-xie-unsplash-shiba.png) +/// caption Photo by Jaycee Xie on Unsplash. +/// Save it as `leaf_shiba.png` in your current directory. @@ -459,18 +460,6 @@ curl -k -XPOST -H "Authorization: Bearer " https://127.0.0.1:5000/api/rec This file can then be previewed on the record page and even downloaded. -## Explore the API +## Consult the API To see what other API calls exist, go to the [API reference section](../reference/rest_api_index.md). - -## Stop it - -If you want to temporarily stop the instance, without losing the data that -was generated you can use the `stop` command: - -```bash -invenio-cli services stop -``` - -Check the [Cleanup section](./destroy.md) if you wish to remove every -reference to InvenioRDM from Docker (containers, images, networks, etc.). diff --git a/docs/install/img/jaycee-xie-unsplash-shiba.png b/docs/install/imgs/jaycee-xie-unsplash-shiba.png similarity index 100% rename from docs/install/img/jaycee-xie-unsplash-shiba.png rename to docs/install/imgs/jaycee-xie-unsplash-shiba.png diff --git a/docs/install/index.md b/docs/install/index.md index 814287ef..95f2af6d 100644 --- a/docs/install/index.md +++ b/docs/install/index.md @@ -1,147 +1,170 @@ -# Installation +# Install -**Intended audience** +**Scope** -The guide is intended for system administrators and developers who want to try, customize or develop with InvenioRDM on their _local machine_. +This guide covers how to set up an InvenioRDM instance locally on your development machine. +How to host it in a production environment is the topic of the [deployment section](../operate/ops/deploy.md). -**Scope** +## Overview -This guide covers how to install InvenioRDM locally on your machine, how to set up and configure your system for InvenioRDM. +Getting started with a new piece of software is often challenging. Getting all the requirements in place and familiarizing oneself with the underlying technologies is a lot of work. With web applications there are multiple separate applications that need to be orchestrated together (web server, database, cache, ...) which adds to the challenge. And all this upfront effort is often expended before the software can even be assessed for appropriateness! -## Quick start +To help with this, InvenioRDM proposes multiple avenues to get you familiar with it in the most convenient way for you possible. -#### [1. Install CLI tool](cli.md) +### Try out the demo site: [https://inveniordm.web.cern.ch/](https://inveniordm.web.cern.ch/) -Install the InvenioRDM CLI tool (see [reference](../reference/cli.md)), e.g. via [`pip`](https://pip.pypa.io/en/stable/): +This will give you a good sense of what InvenioRDM provides out of the box. It's the simplest way to see the software in action. You can even compare it to https://zenodo.org/ which also runs InvenioRDM. It will give you a sense for how it can be customized. -```shell -pip install invenio-cli -``` +### Preview InvenioRDM via local containers -#### [2. Check system requirements](requirements.md) +To run an InvenioRDM instance on your local machine without much of the ceremony involved in fulfilling requirements, you can try our *"preview"*, also known as *"containerized"*, installation. This "installation" containerizes the application itself and every service used by the instance (from the web server to the database) via Docker and combines them via Docker Compose in order to quickly give you a local setup to play with. It doubles as a preview of how you could setup InvenioRDM in a production environment. Note that the [deployment section](../operate/ops/deploy.md) provides more information about what to consider when that time comes. The shortcoming of this approach is that the isolation of Docker containers may prove to be cumbersome for customizations and local development. -!!! info "Information on requirements" +### Install InvenioRDM locally - Please do read the [system requirements](requirements.md) section! - There's important information related to supported versions. +When you are ready to adopt InvenioRDM or want to truly customize and extend it to your needs, we recommend you follow the steps to install it locally. Often this installation is referred as the *"local development"* one. This approach will still containerize the services, but it will install the InvenioRDM application itself on your machine (in a virtual environment so most requirements will be isolated from your system). It is what most institutions go for once they need to work on their instance, and the command line tool we provide makes it straightforward to manage. -You can check if the proper requirements are installed via `invenio-cli`: +Both the *preview* and *local development* installations use the `invenio-cli` tool to get up and running. A common workflow/arrangement is to use the *local development* installation on your local machine for, well, development, and use an equivalent of the *containerized preview* set up in production. -```shell -invenio-cli check-requirements -``` +## Quick start -#### [3. Scaffold project](scaffold.md) +Here is the most succinct outline of the steps to take to get started whether you take the *containerized preview* approach or the *local development* approach. The header of each section links to its expanded documentation. We highly recommend you take the scenic route and follow the expanded documentation as it covers common issues and their solutions, as well as variants and deeper explanations. -Scaffold your InvenioRDM instance. Replace ```` with the version you want to install: +### [1. Install the CLI tool](cli.md) -- latest release (for production systems): ``v12.0`` -- under development release (for feature previews): ``master`` +Irrespective of *preview* or *development* installation, you will need this command line tool: -```shell -invenio-cli init rdm -c -# e.g: -invenio-cli init rdm -c v12.0 -``` +=== "pip" -You will be asked several questions. If in doubt, choose the default. + ```shell + pip install invenio-cli + ``` +=== "uv" -#### [4. Build, setup and run](build-setup-run.md) + ```shell + uv tool install invenio-cli + ``` -Now that the scaffolding is complete, it is time to check the development requirements +=== "pipx" -```shell -cd my-site/ -invenio-cli check-requirements --development -``` + ```shell + pipx install invenio-cli + ``` +### [2. Check system requirements](requirements.md) -You can run the main InvenioRDM application in two modes (choose one): +You can check if the proper requirements are installed via `invenio-cli`: -- Containerized application and services (good for a quick preview). -- Local application with containerized services (good for developers or if you want to customize InvenioRDM). +=== "Local development" -**Containerized application** + ```shell + invenio-cli check-requirements --development + ``` -```shell -invenio-cli containers start --lock --build --setup -``` +=== "Containerized preview" -**Local application** + ```shell + invenio-cli check-requirements + ``` -```shell -invenio-cli install -invenio-cli services setup -invenio-cli run -``` +!!! info "Information on requirements" -!!! warning "Linux: Managing Docker as a non-root user & Context Errors" + Please do read the [system requirements](requirements.md) section! + There's important information related to supported versions. - If you encounter Docker errors running `invenio-cli services setup`, see our section on [Docker pre-requisites](./requirements.md#docker). -#### [5. Explore InvenioRDM](run.md) +### [3. Initialize the instance's directory](initialize.md) -Go and explore your InvenioRDM instance on: +Scaffold your InvenioRDM instance. This is the same operation for *local development* as for *containerized preview*. -- Local: [https://127.0.0.1:5000](https://127.0.0.1:5000) -- Container: [https://127.0.0.1](https://127.0.0.1) +=== "Latest release (default)" -!!! warning "Self-signed SSL certificate" + ```shell + invenio-cli init rdm + ``` - Your browser will display a big warning about an invalid SSL certificate. This is because InvenioRDM generates a self-signed SSL certificate when you scaffold a new instance and because InvenioRDM requires that all traffic is over a secure HTTPS connection. +=== "Specific version" - All major browsers allow you to bypass the warning (not easily though). In Chrome/Edge you have to click in the browser window and type ``thisisunsafe``. + ```shell + invenio-cli init rdm -c + # e.g: + invenio-cli init rdm -c v12.0 + # for pre-release (InvenioRDM development branch) + invenio-cli init rdm -c master + ``` -To create a new administrator account: +You will be asked several questions and given default options. If in doubt, accept the default. -Depending on whether you are in a local or containerized setup, take note of the variations immediately following before stepping through the subsequently outlined steps. -**Local application** +### [4. Build, setup and run](build-setup-run.md) -In a local application context, precede the `invenio` commands by `pipenv run` (e.g., `pipenv run invenio users create --password --active --confirm`). +=== "Local development" -**Containerized application** -In a fully containerized context, connect to a container first e.g. the web-api container: `docker exec -it my-site-web-api-1 /bin/bash`. Then run the commands from within the container as-is. + ```shell + # Install Python and Javascript packages + invenio-cli install + # Set up containerized database, cache, OpenSearch, etc. + invenio-cli services setup + # Serve the application locally through a development server + invenio-cli run + ``` -**Steps** -The following command creates an activated and confirmed user (assuming you have email verification enabled as is the default). +=== "Containerized preview" -```shell -invenio users create --password --active --confirm -``` + ```shell + invenio-cli containers start --lock --build --setup + ``` -Then, allow the user to access the administration panel: +!!! warning "Linux: Managing Docker as a non-root user & Context Errors" -```shell -invenio access allow administration-access user -``` + If you encounter Docker errors running `invenio-cli services setup`, see our section on [Docker pre-requisites](./requirements.md#docker). -#### [6. Stop it](destroy.md) +### [5. Explore InvenioRDM](explore.md) -When you are done, you can stop your instance and optionally destroy the containers: +Go and explore your InvenioRDM instance! -**Containerized application** -To just stop the containers: +=== "Local development" -```shell -invenio-cli containers stop -``` + Visit [https://127.0.0.1:5000](https://127.0.0.1:5000) -To destroy them: +=== "Containerized preview" -```shell -invenio-cli containers destroy -``` + Visit [https://127.0.0.1](https://127.0.0.1) -**Local application** +!!! warning "Self-signed SSL certificate" -```shell -^C [CTRL+C] -invenio-cli services stop -``` + Your browser will display a big warning about an invalid SSL certificate. This is because InvenioRDM generates a self-signed SSL certificate when you scaffold a new instance and because InvenioRDM requires that all traffic is over a secure HTTPS connection. + + All major browsers allow you to bypass the warning (not easily though). In Chrome/Edge you have to click in the browser window and type ``thisisunsafe``. + +### [6. Stop it](stop.md) + +When you are done, you can stop your instance and optionally destroy the containers: -```shell -invenio-cli services destroy -``` +=== "Local development" + + ```shell + # To stop the application server: + # in terminal running invenio-cli run + ^C [CTRL+C] + # --- + # To stop the service containers: + invenio-cli services stop + # --- + # To destroy the service containers + # (this will lose ALL data): + invenio-cli services destroy + ``` + +=== "Containerized preview" + + + ```shell + # To stop all containers: + invenio-cli containers stop + # --- + # To destroy all containers: + # (this will lose ALL data): + invenio-cli containers destroy + ``` diff --git a/docs/install/scaffold.md b/docs/install/initialize.md similarity index 73% rename from docs/install/scaffold.md rename to docs/install/initialize.md index af0c03f8..3fcbada5 100644 --- a/docs/install/scaffold.md +++ b/docs/install/initialize.md @@ -1,12 +1,12 @@ -# Scaffold +# Initialize -Now that you have Invenio-CLI [installed](cli.md), we will use it to scaffold a new instance of InvenioRDM. +Now that you have `invenio-cli` [installed](cli.md), we will use it to scaffold a new instance of InvenioRDM. -## Initialize project +## Initialize your project First, we need to create the project - the necessary files and folders for your InvenioRDM instance. You'll want to commit -the generated project into a version control system, as it will contain the customizations for your repository. If you're working -on an existing repository you can skip the scaffolding step, as it only has to be done once. +the generated project into a version control system, as it will contain your repository and its customizations. If you're working +on an existing repository you can skip the scaffolding step, as it only has to be done once (and clearly has been). This documentation describes `invenio-cli` version 1.6.0. Other versions may have different data requests. @@ -28,17 +28,27 @@ The CLI will require the following data: It will also generate a test private key which is needed for SSL support in the development server. -Let's do it! Pressing `[Enter]` selects the option in brackets `[]`. +Let's do it! Pressing `[Enter]` selects the default option in brackets `[]`. -``` bash -invenio-cli init rdm -c -``` +=== "Latest release (default)" + + ```shell + invenio-cli init rdm + ``` + +=== "Specific version" -Where ```` should be replaced with the version you want to install, e.g.: + ```shell + invenio-cli init rdm -c + # e.g: + invenio-cli init rdm -c v12.0 + # for pre-release (InvenioRDM development branch) + invenio-cli init rdm -c master + ``` -- Current release (for production systems): ``v12.0`` +The current InvenioRDM release (for production systems) is ``v12.0`` -``` console +```console Initializing RDM application... Running cookiecutter... project_name [My Site]: @@ -77,7 +87,7 @@ Choose from 1, 2 [1]: You can now inspect the generated project structure: -``` +```shell cd my-site ls -a1 ``` @@ -110,22 +120,21 @@ Following is an overview of the generated files and folders: |---|---| | ``Dockerfile`` | Dockerfile used to build your application image. | | ``Pipfile`` | Python requirements installed via [pipenv](https://pipenv.pypa.io) | -| ``Pipfile.lock`` | Locked requirements (generated on first install). | -| ``app_data`` | Application data for e.g. vocabularies. | -| ``assets`` | Web assets (CSS, JavaScript, LESS, JSX templates) used in the Webpack build. | -| ``docker`` | Example configuration for NGINX and uWSGI for running InvenioRDM. | +| ``app_data/`` | Application data for e.g. vocabularies. | +| ``assets/`` | Web assets (CSS, JavaScript, LESS, JSX templates) used in the Webpack build. | +| ``docker/`` | Example configuration for NGINX and uWSGI for running InvenioRDM. | | ``docker-compose.full.yml`` | Example of a full infrastructure stack (DO NOT use in production!) | | ``docker-compose.yml`` | Backend services needed for local development. | | ``docker-services.yml`` | Common services for the Docker Compose files. | | ``invenio.cfg`` | The Invenio application configuration. | -| ``logs`` | Log files. | -| ``static`` | Static files that need to be served as-is (e.g. images). | -| ``templates`` | Folder for your Jinja templates. | +| ``logs/`` | Log files. | +| ``static/`` | Static files that need to be served as-is (e.g. images). | +| ``templates/`` | Folder for your Jinja templates. | | ``.invenio`` | Common file used by Invenio-CLI to be version controlled. | | ``.invenio.private`` | Private file used by Invenio-CLI *not* to be version controlled. | -#### Notes and known issues +### Notes and known issues - You may be prompted with `You've downloaded /home//.cookiecutters/cookiecutter-invenio-rdm before. Is it okay to delete and re-download it? [yes]:`. Press `[Enter]` in that case. This will download the latest cookiecutter template. -- Some OpenSSL versions display an error message when obtaining random numbers, but this has no incidence (as far as we can tell) on functionality. We are investigating a possible solution to raise less eyebrows for appearance’s sake. +- Some OpenSSL versions display an error message when obtaining random numbers, but this has no incidence on functionality. diff --git a/docs/install/requirements.md b/docs/install/requirements.md index dce6b6e6..560088b0 100644 --- a/docs/install/requirements.md +++ b/docs/install/requirements.md @@ -1,6 +1,6 @@ # System requirements -### Tools +## Tools InvenioRDM depends on the following requirements to be installed on your local system: @@ -34,7 +34,7 @@ For running and building the application locally you will also need: !!! note "ARM-based CPUs" If you are developing locally with an ARM-based CPU, notably a recent Apple M1/M2 Mac, the minimum support version of InvenioRDM is v10. Previous versions cannot be installed because of an incompatibility with `node-sass`. -### Services +## Services InvenioRDM depends on the following services. During the installation we start these services in containers, but you could as well use externally hosted services for them: @@ -48,23 +48,23 @@ InvenioRDM depends on the following services. During the installation we start t InvenioRDM transitioned from Elasticsearch to OpenSearch due to license changes in Elasticsearch which meant it was no longer an open source product. -### Hardware +## Hardware InvenioRDM runs on machines that have at least 8GB of RAM and at least 4 cores. -### Python virtual environments +## Python packages -Because we want to avoid cluttering the Python packages of the system or user with InvenioRDM dependencies, `invenio-cli` uses virtual environments. -This is done by interacting with `pipenv` behind the scenes, which is listed as a requirement and can simply be installed via `pip`. -To enable the use of a different Python version in the virtual environment than the one installed globally, a Python version manager such as `pyenv` (or `asdf` with `asdf-python`) is required. +Because we want to avoid cluttering the Python packages of the system or user with InvenioRDM dependencies, `invenio-cli` uses virtual environments. This is done by interacting with `pipenv` behind the scenes. -For simplicity, we recommend to go with `pyenv` here. -In some cases, it can be installed via the system's package manager (e.g. `pacman -S pyenv`). -Otherwise, you can find the installation instructions on the [project's GitHub page](https://github.com/pyenv/pyenv/#installation) or use their [automatic installer](https://github.com/pyenv/pyenv-installer) (note the required [dependencies for locally building Python](https://github.com/pyenv/pyenv/wiki#suggested-build-environment)). +To use a different Python version in the virtual environment than the one installed globally, a Python version manager such as `pyenv` or `asdf` with `asdf-python` or `mise` is required. -### Docker +For simplicity, we recommend going with `pyenv` here. You can find the installation instructions on the [project's GitHub page](https://github.com/pyenv/pyenv/#installation) or use their [automatic installer](https://github.com/pyenv/pyenv-installer) (note the required [dependencies for locally building Python](https://github.com/pyenv/pyenv/wiki#suggested-build-environment)). -#### Permissions to run Docker (Linux) +To learn more about virtual environments and their role in InvenioRDM, consult the [virtual environments reference](../reference/virtualenvs.md). + +## Docker + +### Permissions to run Docker (Linux) Your user that will be executing the CLI tool MUST be able to execute the `docker` command (i.e. it is not only available for the root user): @@ -84,7 +84,7 @@ sudo chgrp docker /var/run/docker.sock sudo chmod g+rw /var/run/docker.sock ``` -#### Available memory for Docker (macOS) +### Available memory for Docker (macOS) On the same topic, make sure that Docker itself has enough memory to run. @@ -95,16 +95,16 @@ and adjust the `Memory` to the corresponding value. If you have a few cores more to spare, it might be a good idea to give more than 2. Take into account that you will run between 4 and 8 containers. -#### Docker socket (macOS) +### Docker socket (macOS) invenio-cli uses the Docker Python API to check things like the Docker version. The Docker Python API uses the Docker socket, which may not be enabled by default in Docker Desktop on a Mac. You'll know you have this problem if you see the error message `docker.errors.DockerException: Error while fetching server API version:`. -You can enable the Docker socket by going to Docker Desktop > Settings > Advanced, and checking the box for +You can enable the Docker socket by going to Docker Desktop > Settings > Advanced, and checking the box for "Allow the default Docker socket to be used". You will need to enter your Mac password after you "Apply & restart" this change. -#### OpenSearch and Docker (macOS and Linux) +### OpenSearch and Docker (macOS and Linux) Among the containers you will run is an OpenSearch container which is quite demanding. Per OpenSearch's [Docker documentation](https://opensearch.org/docs/latest/install-and-configure/install-opensearch/docker/), @@ -125,7 +125,7 @@ screen ~/Library/Containers/com.docker.docker/Data/vms/0/tty sysctl -w vm.max_map_count=262144 ``` -#### Use same contexts (macOS and Linux) +### Use same contexts (macOS and Linux) Make sure to always use the same context when using both Docker from the terminal and Docker Desktop. For more information see [Docker Contexts](https://docs.docker.com/engine/context/working-with-contexts/). diff --git a/docs/install/stop.md b/docs/install/stop.md new file mode 100644 index 00000000..2b45541b --- /dev/null +++ b/docs/install/stop.md @@ -0,0 +1,48 @@ +# Stop or destroy your instance + +## Stop it + +If you want to temporarily stop the instance without losing the data that +was generated, you can use the `stop` command: + +=== "Local development" + + ```bash + invenio-cli services stop + ``` + +=== "Containerized preview" + + ```shell + invenio-cli containers stop + ``` + +## Destroy it + +If you wish to clean up and delete all Docker artefacts, +you can use the `destroy` command. It removes ALL InvenioRDM-related containers, images and volumes. + +!!! warning + The ``destroy`` command WILL permanently erase all your data in the Docker containers. + +Note that `destroy` will also `stop` the containers, so there is no need to run the previous command: + +=== "Local development" + + ```bash + invenio-cli services destroy + ``` + +=== "Containerized preview" + + ```shell + invenio-cli containers destroy + ``` + +## Destroy everything + +If you want to also destroy any ``pipenv``-managed Python virtual environment, you can use the global `destroy command`. This command is the same for both installation options and wipes everything but your project files: + +```bash +invenio-cli destroy +``` diff --git a/docs/install/troubleshoot.md b/docs/install/troubleshoot.md index 5e5ecd52..8aab0dad 100644 --- a/docs/install/troubleshoot.md +++ b/docs/install/troubleshoot.md @@ -1,46 +1,16 @@ -# Troubleshooting +# Troubleshooting & getting help -Something has gone wrong, now what? InvenioRDM provides logs in two ways, depending on where the error happened. -If the error comes from the local development instance (`invenio-cli run`), look at the terminal, the logs show up there. -On the other hand, if the error comes from the fully containerized application (`invenio-cli containers start`), you won't see logs on the terminal directly. -See below. - -## Web Application +Something has gone wrong, now what? Your first reflex should be to consult the logs. Then consult this documentation site or [appropriate repositories](../maintenance/modules.md) for known issues. You can also always [reach out to the community](#getting-help) to ask for help. -If you are running the containerized environment you need to get the logs from the corresponding containers. It can be done in two steps, first obtaining the container IDs and then getting their logs: +## Retrieving logs -``` bash -docker ps -a -``` -``` console -CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES -5cb64814ed2a my-site-frontend "nginx -g 'daemon of…" 24 minutes ago Up 1 minute mysite_frontend_1 -39993dcbb84f my-site "bash -c 'celery wor…" 24 minutes ago Up 1 minute mysite_worker_1 -ff9a589845e4 my-site "bash -c 'uwsgi /opt…" 24 minutes ago Up 1 minute 0.0.0.0:32810->5000/tcp mysite_web-api_1 -a99532c10a8b my-site "bash -c 'uwsgi /opt…" 24 minutes ago Up 1 minute 0.0.0.0:32811->5000/tcp mysite_web-ui_1 -d9afc36a573c redis "docker-entrypoint.s…" 24 minutes ago Up 3 minute 0.0.0.0:6379->6379/tcp mysite_cache_1 -cbdac8cbd6a9 rabbitmq:3-management "docker-entrypoint.s…" 24 minutes ago Up 3 minute 4369/tcp, 5671/tcp, 0.0.0.0:5672->5672/tcp, 15671/tcp, 25672/tcp, 0.0.0.0:15672->15672/tcp mysite_mq_1 -38d63e050e6b postgres:9.6 "docker-entrypoint.s…" 24 minutes ago Up 3 minute 0.0.0.0:5432->5432/tcp mysite_db_1 -30356839105a docker.elastic.co/elasticsearch/elasticsearch-oss:7.2.0 "/usr/local/bin/dock…" 24 minutes ago Up 3 minute (health: starting) 0.0.0.0:9200->9200/tcp, 0.0.0.0:9300->9300/tcp mysite_es_1 -``` +If the error comes from the local development instance (`invenio-cli run`), look at the terminal, the logs show up there. -The most interesting ones will be the `web-ui` and `web-api` containers, which in this case have id `a99532c10a8b` and `ff9a589845e4` respectively. The logs can be obtained by using the `logs` command of `docker`. An example of a working instance of the `web-api` container would show the following (trimmed output for clarity): +On the other hand, if the error comes from the fully containerized application (`invenio-cli containers start`), you won't see logs on the terminal directly. They will be managed by docker. See the [Operate an Instance section on retrieving logs](../operate/ops/logging.md) in this case. The information is over there since this is a common approach in a small production environment. -``` bash -docker logs ff9a589845e4 -``` +## Common issues -``` console -[uWSGI] getting INI configuration from /opt/invenio/var/instance/uwsgi_rest.ini -*** Starting uWSGI 2.0.18 (64bit) on [Wed Jan 8 13:09:07 2020] *** -[...] -spawned uWSGI master process (pid: 1) -spawned uWSGI worker 1 (pid: 255, cores: 2) -spawned uWSGI worker 2 (pid: 257, cores: 2) -*** Stats server enabled on 0.0.0.0:9001 fd: 11 *** -``` - -## Build on M1 +### Building on an M1 When building on a M1 (arm64) you might encounter the following error: ```bash pkg_resources.DistributionNotFound: The 'greenlet!=0.4.17; python_version >= "3" and (platform_machine == "aarch64" or (platform_machine == "ppc64le" or (platform_machine == "x86_64" or (platform_machine == "amd64" or (platform_machine == "AMD64" or (platform_machine == "win32" or platform_machine == "WIN32"))))))' distribution was not found and is required by SQLAlchemy @@ -52,4 +22,16 @@ As such, you should add the following line inside your `Pipfile`: ```python sqlalchemy = {version = "*", extras = ["asyncio"]} ``` -in the `[packages]` section. +in the `[packages]` section. + +## Getting help + +The development community is very active on Discord, so in case the documentation +wasn't sufficient or not clear, simply jump on Discord using the link below, +and ask: + +- Join Discord [``inveniosoftware``](https://discord.gg/8qatqBC) + +Please bring logs and/or detailed descriptions of what you did, what you got and what you expected. Without that information, community members will have a hard time diagnosing a report of something "not working"! + +And if you've encountered a problem someone else is asking about, contribute to the discussion! You don't have to be an expert to help others. Activity around a topic helps maintainers better understand scope and effect of particular situations. diff --git a/docs/install/debugging.md b/docs/operate/code/debugging.md similarity index 97% rename from docs/install/debugging.md rename to docs/operate/code/debugging.md index 2b620260..af72e9af 100644 --- a/docs/install/debugging.md +++ b/docs/operate/code/debugging.md @@ -11,46 +11,6 @@ To set a breakpoint place the following code: breakpoint() ``` -## VS Code Debugger - -To debug InvenioRDM using VS Code, configure the debugger with Flask settings: - -From your instance, create or edit `.vscode/launch.json`. -Add the following configuration, see link for more information: [VS Code debugging](https://code.visualstudio.com/docs/python/debugging) - -```json -// .vscode/launch.json -{ - "configurations": [ - { - "name": "Invenio Debugger", - "type": "python", - "request": "launch", - "module": "flask", - "env": { - "FLASK_APP": "invenio_app.wsgi:application", - "FLASK_ENV": "development", - "FLASK_DEBUG": "1", - "WERKZEUG_RUN_MAIN": "watchdog" - }, - "args": [ - "run", - "--host", - "0.0.0.0", - "--cert", - "${workspaceFolder}/docker/nginx/test.crt", - "--key", - "${workspaceFolder}/docker/nginx/test.key" - ], - "jinja": true, - "justMyCode": false - } - ] -} -``` - -Start debugging in VS Code by selecting Invenio Debugger in the debug panel and clicking Run. - **Flask DebugToolbar** You can also install [Flask-DebugToolbar](https://flask-debugtoolbar.readthedocs.io/en/latest/): @@ -133,7 +93,7 @@ function say(phrase) { **Note**: You can test the functionality above by copy/paste the code in your browser's console tab and run the `hello` function. You should see an output similar to: -![Browser debugger output](./img/debugger_output.png) +![Browser debugger output](./imgs/debugger_output.png) **console.log()** @@ -160,4 +120,44 @@ function say(phrase) { - Go to your browser and open the debugger console. - Check the output of your log command: -![Browser console log output](./img/console_log_output.png) +![Browser console log output](./imgs/console_log_output.png) + +## VS Code Debugger + +To debug InvenioRDM using VS Code, configure the debugger with Flask settings: + +From your instance, create or edit `.vscode/launch.json`. +Add the following configuration, see link for more information: [VS Code debugging](https://code.visualstudio.com/docs/python/debugging) + +```json +// .vscode/launch.json +{ + "configurations": [ + { + "name": "Invenio Debugger", + "type": "python", + "request": "launch", + "module": "flask", + "env": { + "FLASK_APP": "invenio_app.wsgi:application", + "FLASK_ENV": "development", + "FLASK_DEBUG": "1", + "WERKZEUG_RUN_MAIN": "watchdog" + }, + "args": [ + "run", + "--host", + "0.0.0.0", + "--cert", + "${workspaceFolder}/docker/nginx/test.crt", + "--key", + "${workspaceFolder}/docker/nginx/test.key" + ], + "jinja": true, + "justMyCode": false + } + ] +} +``` + +Start debugging in VS Code by selecting Invenio Debugger in the debug panel and clicking Run. diff --git a/docs/install/img/console_log_output.png b/docs/operate/code/imgs/console_log_output.png similarity index 100% rename from docs/install/img/console_log_output.png rename to docs/operate/code/imgs/console_log_output.png diff --git a/docs/install/img/debugger_output.png b/docs/operate/code/imgs/debugger_output.png similarity index 100% rename from docs/install/img/debugger_output.png rename to docs/operate/code/imgs/debugger_output.png diff --git a/docs/operate/customize/users.md b/docs/operate/customize/users.md index 7e3d907b..0d639540 100644 --- a/docs/operate/customize/users.md +++ b/docs/operate/customize/users.md @@ -73,18 +73,20 @@ db.session.commit() ## Adding users and roles -You may need to add users or make permission changes after the users vocabulary has been loaded. To create an account, run: +You may need to add users or make permission changes after the users vocabulary has been loaded. + +The following command creates an activated and confirmed user (assuming you have email verification enabled as is the default). ```shell -pipenv run invenio users create email@domain.edu --active --confirm +pipenv run invenio users create email@domain.edu --password --active --confirm ``` This will automatically confirm the account. If you want to force the user to verify their email address, leave off the `--confirm` parameter. -To give an account admin permissions, run: +To give an account access to the administration panel: ```shell -pipenv run invenio roles add admin +invenio access allow administration-access user ``` ## Confirm user diff --git a/docs/operate/index.md b/docs/operate/index.md index 12c7dc59..ce87177b 100644 --- a/docs/operate/index.md +++ b/docs/operate/index.md @@ -11,6 +11,7 @@ This guide is intended for developers that need to operate their own InvenioRDM - [Log](./ops/logging.md) - [Back up search indices](./ops/backup_search_indices.md) - [Redirect legacy routes](./ops/route_migration.md) +- [Migrate from another repository](./ops/migrate.md) ## Customization @@ -27,3 +28,4 @@ This guide is intended for developers that need to operate their own InvenioRDM ## Add custom code - [Custom Python & Javascript code](./code/custom_code.md) +- [Debugging](./code/debugging.md) diff --git a/docs/operate/ops/deploy.md b/docs/operate/ops/deploy.md index 3526e73b..65d24329 100644 --- a/docs/operate/ops/deploy.md +++ b/docs/operate/ops/deploy.md @@ -181,4 +181,4 @@ If the problem is in the InvenioRDM Python application, you can get more details in Sentry. Afterwards, you can try to reproduce it and debug in a local installation. -For more information see the [how to debug section](../../install/debugging.md). +For more information see the [how to debug section](../code/debugging.md). diff --git a/docs/operate/ops/logging.md b/docs/operate/ops/logging.md index 2daf92ea..4ba8ebfc 100644 --- a/docs/operate/ops/logging.md +++ b/docs/operate/ops/logging.md @@ -1,10 +1,45 @@ # Log -InvenioRDM enhances its logging capabilities with the [`invenio-logging`](https://github.com/inveniosoftware/invenio-logging) module, improving the observability and reliability of your repository. This ensures a smoother operational experience for administrators and end-users alike. +## Retrieving logs + +To get the logs if you are running a containerized environment, you will need to retrieve them from the corresponding containers. It can be done in two steps, first obtaining the container IDs and then getting their logs: + +``` bash +docker ps -a +``` +``` console +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +5cb64814ed2a my-site-frontend "nginx -g 'daemon of…" 24 minutes ago Up 1 minute mysite_frontend_1 +39993dcbb84f my-site "bash -c 'celery wor…" 24 minutes ago Up 1 minute mysite_worker_1 +ff9a589845e4 my-site "bash -c 'uwsgi /opt…" 24 minutes ago Up 1 minute 0.0.0.0:32810->5000/tcp mysite_web-api_1 +a99532c10a8b my-site "bash -c 'uwsgi /opt…" 24 minutes ago Up 1 minute 0.0.0.0:32811->5000/tcp mysite_web-ui_1 +d9afc36a573c redis "docker-entrypoint.s…" 24 minutes ago Up 3 minute 0.0.0.0:6379->6379/tcp mysite_cache_1 +cbdac8cbd6a9 rabbitmq:3-management "docker-entrypoint.s…" 24 minutes ago Up 3 minute 4369/tcp, 5671/tcp, 0.0.0.0:5672->5672/tcp, 15671/tcp, 25672/tcp, 0.0.0.0:15672->15672/tcp mysite_mq_1 +38d63e050e6b postgres:9.6 "docker-entrypoint.s…" 24 minutes ago Up 3 minute 0.0.0.0:5432->5432/tcp mysite_db_1 +30356839105a docker.elastic.co/elasticsearch/elasticsearch-oss:7.2.0 "/usr/local/bin/dock…" 24 minutes ago Up 3 minute (health: starting) 0.0.0.0:9200->9200/tcp, 0.0.0.0:9300->9300/tcp mysite_es_1 +``` + +The most interesting entries are the `web-ui` and `web-api` containers, which in this case have id `a99532c10a8b` and `ff9a589845e4` respectively. The logs can be obtained by using the `logs` command of `docker`. An example of a working instance of the `web-api` container would show the following (trimmed output for clarity): -## Overview +``` bash +docker logs ff9a589845e4 +``` + +``` console +[uWSGI] getting INI configuration from /opt/invenio/var/instance/uwsgi_rest.ini +*** Starting uWSGI 2.0.18 (64bit) on [Wed Jan 8 13:09:07 2020] *** +[...] +spawned uWSGI master process (pid: 1) +spawned uWSGI worker 1 (pid: 255, cores: 2) +spawned uWSGI worker 2 (pid: 257, cores: 2) +*** Stats server enabled on 0.0.0.0:9001 fd: 11 *** +``` + +## invenio-logging + +InvenioRDM enhances its logging capabilities with the [`invenio-logging`](https://github.com/inveniosoftware/invenio-logging) module, improving the observability and reliability of your repository. This ensures a smoother operational experience for administrators and end-users alike. -[`invenio-logging`](https://github.com/inveniosoftware/invenio-logging) is a key component of Invenio configures the Flask app logger, offering a standard Python logging interface for log creation and handler installation. +[`invenio-logging`](https://github.com/inveniosoftware/invenio-logging) configures the Flask app logger, offering a standard Python logging interface for log creation and handler installation. The following logger extensions exists: diff --git a/docs/install/migrate.md b/docs/operate/ops/migrate.md similarity index 98% rename from docs/install/migrate.md rename to docs/operate/ops/migrate.md index 9e5c4faa..765dfa0e 100644 --- a/docs/install/migrate.md +++ b/docs/operate/ops/migrate.md @@ -3,7 +3,7 @@ You may want to import your existing records into your new InvenioRDM instance. How to export your data is highly dependent on your existing repository. As of now, InvenioRDM provides no tooling for this. However, to import the records into your instance, the REST API is the current easiest mechanism to do so. -Beforehand, prepare your records to conform to the [REST API](../reference/rest_api_index.md). +Beforehand, prepare your records to conform to the [REST API](../../reference/rest_api_index.md). Then you will want to POST to the API to create the records, but InvenioRDM rate limits the API usage. By default, we don't want our API to be abused or subject to denial-of-service attacks. For the period of time when records are being ingested, however, you will want to allow a greater request rate. diff --git a/docs/install/img/venvs.svg b/docs/reference/imgs/venvs.svg similarity index 100% rename from docs/install/img/venvs.svg rename to docs/reference/imgs/venvs.svg diff --git a/docs/install/virtualenvs.md b/docs/reference/virtualenvs.md similarity index 99% rename from docs/install/virtualenvs.md rename to docs/reference/virtualenvs.md index 56d66a80..32575263 100644 --- a/docs/install/virtualenvs.md +++ b/docs/reference/virtualenvs.md @@ -66,7 +66,7 @@ another for program B each with a different version installed of package X. This means that a virtual environment has its own ``python`` and ``pip`` commands. -![Graphical view of virtual environments and distributions](./img/venvs.svg) +![Graphical view of virtual environments and distributions](./imgs/venvs.svg) ## Tools diff --git a/mkdocs.yml b/mkdocs.yml index 6d9b0339..233d7f80 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -58,19 +58,16 @@ nav: - Restricted records: "features/features-walk-through/restricted_records.md" - Metadata-only records: "features/features-walk-through/metadata_only.md" - Moderation: "features/features-walk-through/moderation.md" + - Install: - - Quick start: "install/index.md" + - Overview: "install/index.md" + - Install the CLI: "install/cli.md" - System requirements: "install/requirements.md" - - Install CLI: "install/cli.md" - - Scaffold: "install/scaffold.md" + - Initialize: "install/initialize.md" - Build, setup & run: "install/build-setup-run.md" - - Use: "install/run.md" - - Migrate: "install/migrate.md" - - Destroy: "install/destroy.md" - - Troubleshooting: "install/troubleshoot.md" - - Virtual environments: "install/virtualenvs.md" - - Debugging: "install/debugging.md" - - Getting help: "install/help.md" + - Explore: "install/explore.md" + - Stop: "install/stop.md" + - Troubleshooting & help: "install/troubleshoot.md" - Operate an Instance: - Overview: "operate/index.md" @@ -80,6 +77,7 @@ nav: - Log: operate/ops/logging.md - Back up search indices: operate/ops/backup_search_indices.md - Redirect legacy routes: operate/ops/route_migration.md + - Migrate from another repository: "operate/ops/migrate.md" - Customization philosophy: operate/customize/index.md - Customize look-and-feel: - Overview: "operate/customize/look-and-feel/index.md" @@ -128,6 +126,7 @@ nav: - Upload limits: "operate/customize/upload_limits.md" - Add custom code: - Custom Python and JS code: operate/code/custom_code.md + - Debugging: "operate/code/debugging.md" - Join the Community: - Overview: community/index.md @@ -266,7 +265,7 @@ nav: - File storage: "reference/file_storage.md" - Notifications: "reference/notifications.md" - Searching: "reference/search.md" - + - Virtual environments: "reference/virtualenvs.md" # Customization @@ -281,9 +280,10 @@ extra_css: markdown_extensions: - admonition - attr_list + - pymdownx.critic + - pymdownx.blocks.caption - pymdownx.highlight - pymdownx.superfences - - pymdownx.critic - pymdownx.tabbed: alternate_style: true - toc: