(PE-41341) Allow primary prostgres host to be the failure (#594)

Author: davidmalloncares

.github/workflows/test-replace-failed-postgresql.yaml

@@ -99,6 +99,14 @@ jobs:
             version=${{ matrix.version }} \
             console_password=${{ secrets.CONSOLE_PASSWORD }} \
             code_manager_auto_configure=true
+      - name: Uninstall on primary-pdb-postgresql to emulate it failing
+        run: |
+          primary_pdb_postgresql_host=$(yq '.groups[].targets[] | select(.vars.role == "primary-pdb-postgresql") | .name' spec/fixtures/litmus_inventory.yaml)
+          bundle exec bolt plan run peadm::uninstall \
+            --inventoryfile spec/fixtures/litmus_inventory.yaml \
+            --modulepath spec/fixtures/modules \
+            -t $primary_pdb_postgresql_host \
+          --no-host-key-check
       - name: Replace failed PostgreSQL
         run: |
           echo ::group::prepare

plans/replace_failed_postgresql.pp

@@ -28,17 +28,23 @@
       $failed_postgresql_host,
       $replacement_postgresql_host,
   ])
-
   # verify we can connect to targets proded before proceeding
   run_command('hostname', $all_hosts)
 
-  # Get current peadm config before making modifications
-  $peadm_config = run_task('peadm::get_peadm_config', $primary_host).first.value
-  $compilers = $peadm_config['params']['compilers']
+  # Try to get current peadm config before making modifications
+  $peadm_config = run_task('peadm::get_peadm_config', $primary_host, { '_catch_errors' => true }).first.value
+
+  if $peadm_config == undef or getvar('peadm_config.params') == undef {
+    out::message('Unable to get config - the failed postgresql host may be the primary')
+  } else {
+    $config_replica_host = $peadm_config['params']['replica_host']
+    $postgresql_a_host = $peadm_config['role-letter']['postgresql']['A']
+    $postgresql_b_host = $peadm_config['role-letter']['postgresql']['B']
 
-  # Bail if we are not running this against an XL deployment with DR enabled - the parameters also enforce this to some extent
-  if $compilers.empty {
-    fail_plan('Plan peadm::replace_failed_postgresql is only applicable for XL deployments with DR enabled')
+    # Bail if we are not running this against an XL deployment with DR enabled - the parameters also enforce this to some extent
+    if $config_replica_host == undef or $postgresql_a_host == undef or $postgresql_b_host == undef {
+      fail_plan('Plan peadm::replace_failed_postgresql is only applicable for XL deployments with DR enabled')
+    }
   }
 
   $pe_hosts = peadm::flatten_compact([

spec/acceptance/peadm_spec/plans/test_replace_failed_postgres.pp

@@ -5,16 +5,8 @@
   Peadm::SingleTargetSpec   $failed_postgresql_host,
   Peadm::SingleTargetSpec   $replacement_postgresql_host,
 ) {
-  # run infra status on the primary
-  out::message("Running peadm::status on primary host ${primary_host}")
-  $primary_status = run_plan('peadm::status', $primary_host, { 'format' => 'json' })
-  out::message($primary_status)
-
-  if empty($primary_status['failed']) {
-    out::message('Cluster is healthy, continuing')
-  } else {
-    fail_plan('Cluster is not healthy, aborting')
-  }
+  # run puppet once on working_postgresql_host - gives some time in CI
+  run_task('peadm::puppet_runonce', $working_postgresql_host)
 
   # replace the failed postgres server
   run_plan('peadm::replace_failed_postgresql',