From beac90003cb816921f98ce2c23ff67cdab900eb9 Mon Sep 17 00:00:00 2001 From: Bodo Hugo Barwich Date: Sat, 13 Nov 2021 11:42:26 +0000 Subject: [PATCH 1/6] ElasticSearch Container Image --- data/{ => es}/.keep | 0 data/pg/.keep | 0 docker-compose.yml.example | 12 +++++++++++- 3 files changed, 11 insertions(+), 1 deletion(-) rename data/{ => es}/.keep (100%) create mode 100644 data/pg/.keep diff --git a/data/.keep b/data/es/.keep similarity index 100% rename from data/.keep rename to data/es/.keep diff --git a/data/pg/.keep b/data/pg/.keep new file mode 100644 index 0000000..e69de29 diff --git a/docker-compose.yml.example b/docker-compose.yml.example index 54058b3..b04f456 100644 --- a/docker-compose.yml.example +++ b/docker-compose.yml.example @@ -7,11 +7,21 @@ services: ports: - "5432:5432" volumes: - - /absolute/path/to/project/data:/var/lib/postgresql/data:Z + - /absolute/path/to/project/data/pg:/var/lib/postgresql/data:Z environment: - COMPONENT=perldoc_db - POSTGRES_USER=perldoc - POSTGRES_PASSWORD=secret + elasticsearch: + container_name: 'perldoc_es' + image: blacktop/elasticsearch:6.8 +# restart: always + ports: + - "9200:9200" + volumes: + - /home/bodo/Projekte/perl/perldoc_web/data/es:/usr/share/elasticsearch/data:Z + environment: + - COMPONENT=perldoc_es web: container_name: 'perldoc_web' image: perldoc_web From 585706fcdd855cae7c942b5f8bbcc92318f7f20a Mon Sep 17 00:00:00 2001 From: Bodo Hugo Barwich Date: Sat, 13 Nov 2021 11:47:45 +0000 Subject: [PATCH 2/6] set ElasticSearch data directory to be configured --- docker-compose.yml.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml.example b/docker-compose.yml.example index b04f456..9bbd956 100644 --- a/docker-compose.yml.example +++ b/docker-compose.yml.example @@ -19,7 +19,7 @@ services: ports: - "9200:9200" volumes: - - /home/bodo/Projekte/perl/perldoc_web/data/es:/usr/share/elasticsearch/data:Z + - /absolute/path/to/project/data/es:/usr/share/elasticsearch/data:Z environment: - COMPONENT=perldoc_es web: From 2445996a9425a2431e74861b01be30d4ddc919a9 Mon Sep 17 00:00:00 2001 From: Bodo Hugo Barwich Date: Sat, 13 Nov 2021 12:10:57 +0000 Subject: [PATCH 3/6] ignore constant changes on "data/pg/.keep" --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index 1c237bf..b441646 100644 --- a/.gitignore +++ b/.gitignore @@ -20,7 +20,6 @@ perl5/* #Database Container Storage data data/* -!data/.keep #Local Docker-Compose Configuration docker-compose.yml From 102371b9c28db1b7f34010984846ee01f5d82c95 Mon Sep 17 00:00:00 2001 From: Bodo Hugo Barwich Date: Sun, 14 Nov 2021 12:55:23 +0000 Subject: [PATCH 4/6] extending documentation for the "ElasticSearch" component --- docker-compose.yml.example | 1 + etc/docker/README.pod | 198 +++++++++++++++++++++++++++++++++++-- 2 files changed, 190 insertions(+), 9 deletions(-) diff --git a/docker-compose.yml.example b/docker-compose.yml.example index 9bbd956..2236012 100644 --- a/docker-compose.yml.example +++ b/docker-compose.yml.example @@ -22,6 +22,7 @@ services: - /absolute/path/to/project/data/es:/usr/share/elasticsearch/data:Z environment: - COMPONENT=perldoc_es + - JAVA_HOME=/usr/lib/jvm/default-jvm web: container_name: 'perldoc_web' image: perldoc_web diff --git a/etc/docker/README.pod b/etc/docker/README.pod index 74704db..ecce37e 100644 --- a/etc/docker/README.pod +++ b/etc/docker/README.pod @@ -96,7 +96,7 @@ with the C command. =item C Installation -As discussed in the task issue L the +As discussed in the task issue L the installation of the I Modules for the SQLite Backend from the F was executed at Image Build Time. So on updates of the F it is recommendable to rebuild the Container Image as described above under L>. @@ -114,12 +114,11 @@ Still the Container Start-Up Script F will detect a different bac or the C Command and check whether the key dependencies are met and run the C Installation accordingly -=item starting up the Docker Cluster +=item Starting up the Docker Cluster -The C Database is only within the C environment known with the hostname C. -So to use the database hostname C any command must be run within the C environment. -To startup the Docker Cluster with the C environment the following command -is needed at first: +The Docker Cluster consists of three components. +To access any of these components at first the Docker Cluster needs to be started with +this C command: docker-compose up -d @@ -128,7 +127,65 @@ It is important to verify that the containers are running correctly with: docker-compose ps -=item populating the search backend +This will make the three components accessible with the C command. + +=over 2 + +=item * The I + +The I is accessible with the component name C. +In the I component runs the C I Web Application. +To access the I component and get a C prompt +this C command can be used: + + docker-compose exec web bash + +=item * The I Database + +The I Database is only within the C environment known with the hostname C. +So, to use the database hostname C any command must be run within the C environment. +On a new installation the ownership of I data directory must be changed +to match the containers C user with this command: + + chown 70:70 data/pg + +=item * The I Engine + +The I Engine is only within the C environment known with the hostname C. +So, to use the I hostname C any command must be run within the C environment. +The I API is also accessible on the external port C<9200>. +So, it can also be queried with the URL C from outside the Docker Cluster. +According to the official I documentation the Virtual Memory per process must +be increased as documented at +L. +It can be increase temporary with this command: + + sysctl -w vm.max_map_count=262144 + +or it can be set permanently within a file like F or directly in F + + vm.max_map_count = 262144 + +and then reloaded with the command C. +The I Documentation explains very nicely how and whether this will affect the system +as documented at L Documentation on C|https://www.suse.com/support/kb/doc/?id=000016692> + +B + +=over 2 + +The I Engine is known to start up slow. It can take up to B<30 s>. +This is also understandable from the referenced I Documentation on the Virtual Memory Requirement. +So, a too early querying can produce an error as documented at +L. +See L Engine"> for instructions to check whether +the engine is ready for service + +=back + +=back + +=item Populating the search backend The new built Container Image contains an empty C Installation To run correctly the Search Backend needs to be populated. @@ -140,10 +197,10 @@ Now the command to populate the Search Backend is: This will execute command C in the project directory. The results will be stored persistently in the project directory for further container launches. -=item accessing the C Database +=item Accessing the I Database To be able to access the database the Docker Cluster must be launched as described -in L>. +in L">. Next the command C can be used within the C container. The C image is based on I @@ -178,6 +235,129 @@ The C table can contain for 1 I Version 1456 entries: 1456 (1 row) + +=item Querying the I Engine + +The I Engine needs to be queried in different occasions to check its +availability and health and the correctness of the indices. +This can be done easily over the I API by using the Web Endpoints that it publishes. + +=over 2 + +=item Ready for Service + +The I Engine is ready for service when its Root Web Endpoints produces +a HTTP Response with B C<200 OK> + + curl http://localhost:9200 + +The Response will look similar to this: + + > GET / HTTP/1.1 + > Host: localhost:9200 + > User-Agent: curl/7.64.0 + > Accept: */* + > + < HTTP/1.1 200 OK + < content-type: application/json; charset=UTF-8 + < content-length: 490 + < + { + "name" : "Evou766", + "cluster_name" : "elasticsearch", + "cluster_uuid" : "D7p_jR1TQBeK7J69Hk3QRg", + "version" : { + "number" : "6.8.13", + "build_flavor" : "oss", + "build_type" : "tar", + "build_hash" : "be13c69", + "build_date" : "2020-10-16T09:09:46.555371Z", + "build_snapshot" : false, + "lucene_version" : "7.7.3", + "minimum_wire_compatibility_version" : "5.6.0", + "minimum_index_compatibility_version" : "5.0.0" + }, + "tagline" : "You Know, for Search" + } + +=item Cluster Health Status + +The I Cluster Health is an indicator for any error within the engine +or the indices. A status of "I" or "I" indicates a correct and healthy status. +While a status "I" indicates an error within the indices. This can originate +from uncomplete or interrupted indexing or an sudden crash of the engine. +An re-indexing might fix this issue. +The Cluster Health can be checked over this API endpoint: + + curl -v http://localhost:9200/_cat/health + +The Response will look similar to this: + + > GET /_cat/health HTTP/1.1 + > Host: localhost:9200 + > User-Agent: curl/7.64.0 + > Accept: */* + > + < HTTP/1.1 200 OK + < content-type: text/plain; charset=UTF-8 + < content-length: 65 + < + 1636891612 12:06:52 elasticsearch yellow 1 1 5 5 0 0 5 0 - 50.0% + +=item Indices Health Status + +The I Indices Health Status is an indicator for any error within the indices. +A status of "I" or "I" indicates a correct and healthy status. +While a status "I" indicates an error within the indices. This can originate +from uncomplete or interrupted indexing or an sudden crash of the engine. +An re-indexing might fix this issue. +The Indices Health Status can be checked over this API endpoint: + + curl -v http://localhost:9200/_cat/indices + +The Response will look similar to this: + + > GET /_cat/indices HTTP/1.1 + > Host: localhost:9200 + > User-Agent: curl/7.64.0 + > Accept: */* + > + < HTTP/1.1 200 OK + < content-type: text/plain; charset=UTF-8 + < content-length: 455 + < + yellow open perldeltas_5.28.1_1636798290 BTS4QdaeQk6OJLFnyYUI9g 1 1 2164 0 3.2mb 3.2mb + yellow open faqs_5.28.1_1636798290 gyrqSq7mQrKXzAmQJ4cGVA 1 1 305 0 784.9kb 784.9kb + yellow open variables_5.28.1_1636798290 wjDlOrQrRaWb77HTKhdA5Q 1 1 150 0 17.2kb 17.2kb + yellow open pods_5.28.1_1636798290 PJ-EZ0IbQb67EOzkGrVj1w 1 1 1579 0 23.2mb 23.2mb + yellow open functions_5.28.1_1636798290 xzukrTriSNWiyPqKMpZU4w 1 1 292 0 570.6kb 570.6kb + +=item Aliases Associations + +The Project also uses aliases in I. It is important that they are set correctly. +The Aliases Associations can be checked over this API endpoint: + + curl -v http://localhost:9200/_cat/aliases + +The Response will look similar to this: + + > GET /_cat/aliases HTTP/1.1 + > Host: localhost:9200 + > User-Agent: curl/7.64.0 + > Accept: */* + > + < HTTP/1.1 200 OK + < content-type: text/plain; charset=UTF-8 + < content-length: 265 + < + functions_5.28.1 functions_5.28.1_1636798290 - - - + perldeltas_5.28.1 perldeltas_5.28.1_1636798290 - - - + faqs_5.28.1 faqs_5.28.1_1636798290 - - - + variables_5.28.1 variables_5.28.1_1636798290 - - - + pods_5.28.1 pods_5.28.1_1636798290 - - - + +=back + =back =cut From 904c0e2b01ffb9afcb5335861e38b4e9f74cf090 Mon Sep 17 00:00:00 2001 From: Bodo Hugo Barwich Date: Tue, 16 Nov 2021 23:32:33 +0000 Subject: [PATCH 5/6] correct instructions for the database installation --- etc/docker/README.pod | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/etc/docker/README.pod b/etc/docker/README.pod index ecce37e..090780f 100644 --- a/etc/docker/README.pod +++ b/etc/docker/README.pod @@ -99,7 +99,7 @@ with the C command. As discussed in the task issue L the installation of the I Modules for the SQLite Backend from the F was executed at Image Build Time. So on updates of the F it is recommendable to rebuild the Container Image as described above -under L>. +under L">. The used F can be found in F within the Docker Image. Also the C Installation Log is found inside the Image in F. @@ -144,10 +144,9 @@ this C command can be used: The I Database is only within the C environment known with the hostname C. So, to use the database hostname C any command must be run within the C environment. -On a new installation the ownership of I data directory must be changed -to match the containers C user with this command: +On a new installation the file F obstructs the process and must be removed with this command: - chown 70:70 data/pg + rm data/pg/.keep =item * The I Engine From 9f622994f1bc808963c7699796953c7316798cbe Mon Sep 17 00:00:00 2001 From: Bodo Hugo Barwich Date: Sun, 28 Nov 2021 19:05:09 +0000 Subject: [PATCH 6/6] Environment Variable "JAVA_HOME" is not needed anymore --- .gitignore | 2 +- docker-compose.yml.example | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index b441646..4c1f011 100644 --- a/.gitignore +++ b/.gitignore @@ -19,7 +19,7 @@ perl5/* #Database Container Storage data -data/* +data/** #Local Docker-Compose Configuration docker-compose.yml diff --git a/docker-compose.yml.example b/docker-compose.yml.example index 2236012..9bbd956 100644 --- a/docker-compose.yml.example +++ b/docker-compose.yml.example @@ -22,7 +22,6 @@ services: - /absolute/path/to/project/data/es:/usr/share/elasticsearch/data:Z environment: - COMPONENT=perldoc_es - - JAVA_HOME=/usr/lib/jvm/default-jvm web: container_name: 'perldoc_web' image: perldoc_web