From 0bdaa952e7ce7d63f2806ca7e0ec1abf45faa626 Mon Sep 17 00:00:00 2001 From: Bodo Hugo Barwich Date: Mon, 4 Oct 2021 19:30:38 +0100 Subject: [PATCH 01/10] indexing test --- .github/workflows/indexing.yml | 120 +++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 .github/workflows/indexing.yml diff --git a/.github/workflows/indexing.yml b/.github/workflows/indexing.yml new file mode 100644 index 0000000..43be3a6 --- /dev/null +++ b/.github/workflows/indexing.yml @@ -0,0 +1,120 @@ +# This is a basic workflow to help you get started with Actions + +name: Build ElasticSearch Index + +# Controls when the workflow will run +on: + # Triggers the workflow on push or pull request events but only for the master branch + pull_request: + branches: [ master ] + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: +# workflow_dispatch: + # branches: + # - master + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # This workflow contains a single job called "build" + build-index: + # The type of runner that the job will run on + runs-on: ubuntu-20.04 + +# env: + # Environment Variables for the Indexation Script +# ES_TEST: "elasticsearch:9200" +# HARNESS_ACTIVE: 0 +# TEST_VERBOSE: 0 + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Start Docker Engine +# sudo systemctl start docker + - name: Start Docker Engine + run: | + sudo systemctl status docker -l + + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v2 + + - name: Listing Directory Contents + run: | + echo 'User:' $(whoami) + echo 'Working Directory:' $(pwd) + echo 'Directory Content:' + ls -lah + + - name: Clone MetaCPAN API Project + run: | + mkdir -p src ; cd src + git clone https://github.com/bodo-hugo-barwich/metacpan-api.git metacpan-api + cd metacpan-api + git fetch && git checkout no-47_wrong-index && git pull && git log -1 + cd ../../ + + # Build Container Images +# sudo chmod a+w cache log perl5 + - name: Build Container Images with 'docker-compose' + run: | + mkdir -p cpan src ; chmod a+w cpan -R ; cd src + git clone https://github.com/metacpan/metacpan-web.git metacpan-web + git clone https://github.com/metacpan/metacpan-grep-front-end.git metacpan-grep-front-end + git clone https://github.com/metacpan/metacpan-cpan-extracted-lite.git metacpan-cpan-extracted-lite + ln -s metacpan-cpan-extracted-lite metacpan-cpan-extracted + cd ../ + docker volume create --driver local --opt device=:$(pwd)/src/metacpan-cpan-extracted --opt o=bind --opt type=none metacpan_git_shared + echo "Docker: Volumes listing ..." + docker volume ls + docker volume inspect metacpan_git_shared + echo "Docker: Images building ..." + docker-compose up --build --no-start traefik + docker-compose up --build --no-start elasticsearch + docker-compose up --build --no-start pgdb + docker-compose up --build --no-start api + echo "Docker: Images listing ..." + docker image ls + docker image inspect metacpan/metacpan-api + echo "Docker: Volumes listing ..." + docker volume ls + docker volume inspect metacpan_elasticsearch + echo "Docker: Service 'api' starting ..." + docker-compose up -d api + + # Check Container Health Exit the test when Containers are failed + - name: Check Container Health + run: | + constat=`docker-compose ps 2>&1` + echo -e "Container Status Complete:\n$constat" + constat=`echo "$constat" | sed -re 's/[[:space:]][[:space:]]+/|/g' | cut -d"|" -f1,3 | grep "|"` + echo -e "Container Status Up:\n$constat" + echo "Component 'api' Logs:" + docker-compose logs api + confailed=`echo "$constat" | grep -i "exit" | wc -l` + if [ $confailed -ne 0 ]; then exit 1 ; fi + echo "Component 'elasticsearch' Logs:" + docker-compose logs elasticsearch + + # Test the Elastic Search Indexation + - name: Run the Elastic Search Indexation + run: | + echo "Component 'api' Availability:" + curl -v "localhost:5000" 2>&1 + elasticsearchport=`docker-compose ps 2>&1 | grep -i elasticsearch | grep -ioE ":[0-9]+\->" | grep -ioE "[0-9]+" | sed -n 1p` + export ES_PORT=$elasticsearchport + echo "ElasticSearch Port: '$ES_PORT'" + echo "ElasticSearch Version:" + curl -v "localhost:$ES_PORT" 2>&1 + echo "Indexation: starting ..." + docker-compose exec -T api index-cpan.sh + echo "Indexation: Indices showing ..." + docker-compose exec -T api curl -v 'elasticsearch:9200/_cat/indices' + echo "Indexation: Activity Log showing ..." + cat src/metacpan-api/var/log/metacpan.log + doccount=`curl "localhost:$ES_PORT/_cat/indices" 2>&1 | grep -i "cpan_v1_01" | awk '{print $6}'` + alias=`curl "localhost:$ES_PORT/_cat/aliases" 2>&1 | grep -i "cpan_v1_01" | cut -d" " -f1,2` + if [ -z "$doccount"]; then doccount=0 ; fi + if [ -z "$alias" ]; then alias="none" ; fi + echo "Indexation: Document Count '$doccount'" + echo "Indexation: Alias '$alias'" + if [ $doccount -eq 0 ]; then exit 1 ; fi + if [ "$alias" != "cpan cpan_v1_01" ]; then exit 1 ; fi From c7673d51cd8d2c7b72ef513e82bfdc9a2b25076a Mon Sep 17 00:00:00 2001 From: Bodo Hugo Barwich Date: Mon, 4 Oct 2021 20:38:36 +0100 Subject: [PATCH 02/10] add module name and execution time --- bin/index-cpan.sh | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/bin/index-cpan.sh b/bin/index-cpan.sh index c3d99ea..161f88f 100755 --- a/bin/index-cpan.sh +++ b/bin/index-cpan.sh @@ -1,10 +1,40 @@ #!/bin/sh -./bin/run bin/metacpan mapping --delete +MODULE=`basename $0` + +NOW=$(date +"%F %T") +echo "${NOW} I ${MODULE}: Indices re-creating ..." + +sdeletelog=`./bin/run bin/metacpan mapping --delete --yes 2>&1` +ideleters=$? + +NOW=$(date +"%F %T") +echo "${NOW} I ${MODULE}: Re-creation finished with [$ideleters]" +echo "${NOW} I ${MODULE}: Re-creation Log:\n'$sdeletelog'" + +if [ $ideleters -ne 0 ]; then + echo "${NOW} I ${MODULE}: Re-creation failed!" + + exit $ideleters +fi + +NOW=$(date +"%F %T") +echo "${NOW} I ${MODULE}: ElasticSearch 'elasticsearch:9200': Indices showing ..." + +curl -v 'elasticsearch:9200/_cat/indices' + +NOW=$(date +"%F %T") +echo "${NOW} I ${MODULE}: Packages downloading ..." /bin/partial-cpan-mirror.sh +NOW=$(date +"%F %T") +echo "${NOW} I ${MODULE}: Indices rebuilding ..." + ./bin/run bin/metacpan release /CPAN/authors/id/ ./bin/run bin/metacpan latest ./bin/run bin/metacpan author ./bin/run bin/metacpan permission + +NOW=$(date +"%F %T") +echo "${NOW} I ${MODULE}: done." From 037fa7694c1f11a65c0103bf3bcf554910855e36 Mon Sep 17 00:00:00 2001 From: Bodo Hugo Barwich Date: Sun, 31 Oct 2021 11:17:09 +0000 Subject: [PATCH 03/10] options "--yes", "--show_cluster_info" and "$@" --- bin/index-cpan.sh | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/bin/index-cpan.sh b/bin/index-cpan.sh index 161f88f..4b18323 100755 --- a/bin/index-cpan.sh +++ b/bin/index-cpan.sh @@ -5,7 +5,7 @@ MODULE=`basename $0` NOW=$(date +"%F %T") echo "${NOW} I ${MODULE}: Indices re-creating ..." -sdeletelog=`./bin/run bin/metacpan mapping --delete --yes 2>&1` +sdeletelog=`./bin/run bin/metacpan mapping --delete --yes $@ 2>&1` ideleters=$? NOW=$(date +"%F %T") @@ -13,15 +13,26 @@ echo "${NOW} I ${MODULE}: Re-creation finished with [$ideleters]" echo "${NOW} I ${MODULE}: Re-creation Log:\n'$sdeletelog'" if [ $ideleters -ne 0 ]; then - echo "${NOW} I ${MODULE}: Re-creation failed!" + echo "${NOW} E ${MODULE}: Re-creation failed!" exit $ideleters fi NOW=$(date +"%F %T") -echo "${NOW} I ${MODULE}: ElasticSearch 'elasticsearch:9200': Indices showing ..." +echo "${NOW} I ${MODULE}: ElasticSearch - Info collecting ..." -curl -v 'elasticsearch:9200/_cat/indices' +sinfolog=`./bin/run bin/metacpan mapping --show_cluster_info $@ 2>&1` +iinfors=$? + +NOW=$(date +"%F %T") +echo "${NOW} I ${MODULE}: Info finished with [$iinfors]" +echo "${NOW} I ${MODULE}: Info Log:\n'$sinfolog'" + +if [ $iinfors -ne 0 ]; then + echo "${NOW} E ${MODULE}: ElasticSearch unavailable!" + + exit $iinfors +fi NOW=$(date +"%F %T") echo "${NOW} I ${MODULE}: Packages downloading ..." From aebe141c41d82efd9c6ab772ea45d44e56525312 Mon Sep 17 00:00:00 2001 From: Bodo Hugo Barwich Date: Sun, 31 Oct 2021 11:24:13 +0000 Subject: [PATCH 04/10] remove api check from test --- .github/workflows/indexing.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/indexing.yml b/.github/workflows/indexing.yml index 43be3a6..b537681 100644 --- a/.github/workflows/indexing.yml +++ b/.github/workflows/indexing.yml @@ -97,8 +97,6 @@ jobs: # Test the Elastic Search Indexation - name: Run the Elastic Search Indexation run: | - echo "Component 'api' Availability:" - curl -v "localhost:5000" 2>&1 elasticsearchport=`docker-compose ps 2>&1 | grep -i elasticsearch | grep -ioE ":[0-9]+\->" | grep -ioE "[0-9]+" | sed -n 1p` export ES_PORT=$elasticsearchport echo "ElasticSearch Port: '$ES_PORT'" From a5416d64c50e6e7313711b3eecdb63eb48349fc4 Mon Sep 17 00:00:00 2001 From: Bodo Hugo Barwich Date: Sun, 31 Oct 2021 11:32:23 +0000 Subject: [PATCH 05/10] inspect ElasticSearch after Indexation --- .github/workflows/indexing.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/indexing.yml b/.github/workflows/indexing.yml index b537681..169593f 100644 --- a/.github/workflows/indexing.yml +++ b/.github/workflows/indexing.yml @@ -97,13 +97,13 @@ jobs: # Test the Elastic Search Indexation - name: Run the Elastic Search Indexation run: | + echo "Indexation: starting ..." + docker-compose exec -T api index-cpan.sh elasticsearchport=`docker-compose ps 2>&1 | grep -i elasticsearch | grep -ioE ":[0-9]+\->" | grep -ioE "[0-9]+" | sed -n 1p` export ES_PORT=$elasticsearchport echo "ElasticSearch Port: '$ES_PORT'" echo "ElasticSearch Version:" curl -v "localhost:$ES_PORT" 2>&1 - echo "Indexation: starting ..." - docker-compose exec -T api index-cpan.sh echo "Indexation: Indices showing ..." docker-compose exec -T api curl -v 'elasticsearch:9200/_cat/indices' echo "Indexation: Activity Log showing ..." From 2996ad9223bc622f6a51014cc977701d879d201a Mon Sep 17 00:00:00 2001 From: Bodo Hugo Barwich Date: Sun, 31 Oct 2021 23:30:27 +0000 Subject: [PATCH 06/10] protect email address --- .github/workflows/indexing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/indexing.yml b/.github/workflows/indexing.yml index 169593f..ef3dddf 100644 --- a/.github/workflows/indexing.yml +++ b/.github/workflows/indexing.yml @@ -49,7 +49,7 @@ jobs: mkdir -p src ; cd src git clone https://github.com/bodo-hugo-barwich/metacpan-api.git metacpan-api cd metacpan-api - git fetch && git checkout no-47_wrong-index && git pull && git log -1 + git fetch && git checkout no-47_wrong-index && git pull && git log -1 | sed -re 's/@/ at /' | tr -s '<>' cd ../../ # Build Container Images From 127354f94fcdc65c9743739cc0e5c2e33c391305 Mon Sep 17 00:00:00 2001 From: Bodo Hugo Barwich Date: Sun, 31 Oct 2021 23:32:06 +0000 Subject: [PATCH 07/10] syntax correction --- .github/workflows/indexing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/indexing.yml b/.github/workflows/indexing.yml index ef3dddf..40e8e7e 100644 --- a/.github/workflows/indexing.yml +++ b/.github/workflows/indexing.yml @@ -49,7 +49,7 @@ jobs: mkdir -p src ; cd src git clone https://github.com/bodo-hugo-barwich/metacpan-api.git metacpan-api cd metacpan-api - git fetch && git checkout no-47_wrong-index && git pull && git log -1 | sed -re 's/@/ at /' | tr -s '<>' + git fetch && git checkout no-47_wrong-index && git pull && git log -1 | sed -re 's/@/ at /' | tr -s '<>' "'" cd ../../ # Build Container Images From a0d72c8b5ac9318979f82ea9253aa5baf22d0232 Mon Sep 17 00:00:00 2001 From: Bodo Hugo Barwich <42355798+bodo-hugo-barwich@users.noreply.github.com> Date: Fri, 10 Dec 2021 17:28:25 +0000 Subject: [PATCH 08/10] exclude "--yes" option from script logic the "--yes" option must not be the default behaviour --- bin/index-cpan.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/index-cpan.sh b/bin/index-cpan.sh index 4b18323..4e89254 100755 --- a/bin/index-cpan.sh +++ b/bin/index-cpan.sh @@ -5,7 +5,7 @@ MODULE=`basename $0` NOW=$(date +"%F %T") echo "${NOW} I ${MODULE}: Indices re-creating ..." -sdeletelog=`./bin/run bin/metacpan mapping --delete --yes $@ 2>&1` +sdeletelog=`./bin/run bin/metacpan mapping --delete $@ 2>&1` ideleters=$? NOW=$(date +"%F %T") From cacd2ec13503b96ee7530e5134d2345e03254a92 Mon Sep 17 00:00:00 2001 From: Bodo Hugo Barwich Date: Fri, 10 Dec 2021 20:56:21 +0000 Subject: [PATCH 09/10] update with mainstream --- .env | 2 +- configs/metacpan-web/metacpan_web.conf | 39 -------------------- configs/metacpan-web/metacpan_web_local.conf | 8 ++++ docker-compose.yml | 8 ++-- grep.env | 2 +- 5 files changed, 15 insertions(+), 44 deletions(-) delete mode 100644 configs/metacpan-web/metacpan_web.conf create mode 100644 configs/metacpan-web/metacpan_web_local.conf diff --git a/.env b/.env index 074398d..8ea602e 100644 --- a/.env +++ b/.env @@ -1,4 +1,4 @@ COMPOSE_PROJECT_NAME=metacpan PLACK_ENV=development PGDB=pgdb:5432 -API_SERVER=morbo -l http://*:5000 -w app.psgi -w bin -w lib -w templates --verbose +API_SERVER="morbo -l http://*:5000 -w app.psgi -w bin -w lib -w templates --verbose" diff --git a/configs/metacpan-web/metacpan_web.conf b/configs/metacpan-web/metacpan_web.conf deleted file mode 100644 index e8b4c81..0000000 --- a/configs/metacpan-web/metacpan_web.conf +++ /dev/null @@ -1,39 +0,0 @@ -# rename this file to metacpan::web.yml and put a ':' after 'name' if -# you want to use YAML like in old versions of Catalyst -name MetaCPAN::Web -default_view HTML - -api = http://api:5000 -api_external = http://localhost:5000 -api_secure = http://api:5000 -api_external_secure = http://localhost:5000 -source_host = http://localhost:5000 -web_host = http://localhost:5001 -consumer_key = metacpan.dev -cookie_secret = seekrit -consumer_secret = ClearAirTurbulence - -mark_unauthorized_releases = 0 - -#site_alert_message = The sky is falling. - - - INCLUDE_PATH root/ - TAG_STYLE asp - WRAPPER wrapper.html - TEMPLATE_EXTENSION .html - ENCODING utf8 - AUTO_FILTER html - STAT_TTL 1 - COMPILE_PERL 0 - COMPILE_DIR /var/tmp/templates - - - - WRAPPER "" - - - - public_key 6LeH2MsSAAAAANwz3AA73Gw5OjCVjT6I51Ev-ior - - diff --git a/configs/metacpan-web/metacpan_web_local.conf b/configs/metacpan-web/metacpan_web_local.conf new file mode 100644 index 0000000..16dc12c --- /dev/null +++ b/configs/metacpan-web/metacpan_web_local.conf @@ -0,0 +1,8 @@ +api = http://api:5000 +api_public = http://localhost:5000 +source_host = http://localhost:5000 +web_host = http://localhost:5001 + + + cache_dir = /var/tmp/templates + diff --git a/docker-compose.yml b/docker-compose.yml index 1a6d245..6ddb536 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -72,8 +72,8 @@ services: source: web_carton target: /carton - type: bind - source: ./configs/metacpan-web/metacpan_web.conf - target: /metacpan-web/metacpan_web.conf + source: ./configs/metacpan-web/metacpan_web_local.conf + target: /metacpan-web/metacpan_web_local.conf read_only: true - type: bind source: ./src/metacpan-web @@ -366,6 +366,8 @@ services: PG_TAG: "${PG_VERSION_TAG:-9.6-alpine}" environment: POSTGRES_PASSWORD: metacpan + POSTGRES_USERNAME: metacpan123 + POSTGRES_DB: metacpan networks: - database healthcheck: @@ -396,7 +398,7 @@ services: # mongodb: - image: mongo:latest + image: mongo:4.4.9 networks: - mongo healthcheck: diff --git a/grep.env b/grep.env index 6b03c65..8e1d6a5 100644 --- a/grep.env +++ b/grep.env @@ -1,4 +1,4 @@ # grep service - development environment GREP_SITE_PORT=3001 -GREP_PLACKUP_SERVER_ARGS=-E development -R lib,bin \ No newline at end of file +GREP_PLACKUP_SERVER_ARGS="-E development -R lib,bin" From 0ff95c3aaee3a10d26a9a9ab92f79877e2a2e40e Mon Sep 17 00:00:00 2001 From: Bodo Hugo Barwich Date: Mon, 20 Dec 2021 19:31:30 +0000 Subject: [PATCH 10/10] write mapping log directly to STDOUT --- bin/index-cpan.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bin/index-cpan.sh b/bin/index-cpan.sh index 4e89254..bbb6abe 100755 --- a/bin/index-cpan.sh +++ b/bin/index-cpan.sh @@ -5,12 +5,13 @@ MODULE=`basename $0` NOW=$(date +"%F %T") echo "${NOW} I ${MODULE}: Indices re-creating ..." -sdeletelog=`./bin/run bin/metacpan mapping --delete $@ 2>&1` +#sdeletelog=`echo "" 2>&1` +./bin/run bin/metacpan mapping --delete $@ ideleters=$? NOW=$(date +"%F %T") echo "${NOW} I ${MODULE}: Re-creation finished with [$ideleters]" -echo "${NOW} I ${MODULE}: Re-creation Log:\n'$sdeletelog'" +#echo "${NOW} I ${MODULE}: Re-creation Log:\n'$sdeletelog'" if [ $ideleters -ne 0 ]; then echo "${NOW} E ${MODULE}: Re-creation failed!"