Skip to content

Commit 5359c98

Browse files
On every node change adjust dbroots in the read-only nodes
1 parent ff25d4e commit 5359c98

File tree

7 files changed

+150
-49
lines changed

7 files changed

+150
-49
lines changed

cmapi/cmapi_server/failover_agent.py

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -95,9 +95,7 @@ def enterStandbyMode(self, test_mode = False):
9595
try:
9696
# TODO: remove test_mode condition and add mock for testing
9797
if not test_mode:
98-
MCSProcessManager.stop_node(
99-
is_primary=nc.is_primary_node(),
100-
)
98+
MCSProcessManager.stop_node(is_primary=nc.is_primary_node())
10199
logger.info(
102100
'FA.enterStandbyMode(): successfully stopped node.'
103101
)

cmapi/cmapi_server/handlers/cluster.py

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
get_current_key, get_version, update_revision_and_manager,
1616
)
1717
from cmapi_server.node_manipulation import (
18-
add_node, add_dbroot, remove_node, switch_node_maintenance,
18+
add_node, add_dbroot, remove_node, switch_node_maintenance, update_dbroots_of_readonly_nodes,
1919
)
2020
from mcs_node_control.models.misc import get_dbrm_master
2121
from mcs_node_control.models.node_config import NodeConfig
@@ -181,11 +181,6 @@ def add_node(
181181
host=node, input_config_filename=config,
182182
output_config_filename=config
183183
)
184-
else:
185-
logger.debug(
186-
f'Node {node} is read-only, skipping dbroot addition'
187-
)
188-
189184
except Exception as err:
190185
raise CMAPIBasicError('Error while adding node.') from err
191186

@@ -228,6 +223,8 @@ def remove_node(node: str, config: str = DEFAULT_MCS_CONF_PATH) -> dict:
228223
node, input_config_filename=config,
229224
output_config_filename=config
230225
)
226+
with NodeConfig().modify_config(config) as root:
227+
update_dbroots_of_readonly_nodes(root)
231228
except Exception as err:
232229
raise CMAPIBasicError('Error while removing node.') from err
233230

cmapi/cmapi_server/helpers.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -378,7 +378,7 @@ async def update_config(node: str, headers: dict, body: dict) -> None:
378378
) as response:
379379
resp_json = await response.json(encoding='utf-8')
380380
response.raise_for_status()
381-
logging.info(f'Node {node} config put successfull.')
381+
logging.info(f'Node {node} config put successful.')
382382
except aiohttp.ClientResponseError as err:
383383
# TODO: may be better to check if resp status is 422 cause
384384
# it's like a signal that cmapi server raised it in

cmapi/cmapi_server/managers/transaction.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ def rollback_transaction(self, nodes: Optional[list] = None) -> None:
106106
try:
107107
rollback_transaction(self.txn_id, nodes=nodes)
108108
self.active_transaction = False
109-
logging.debug(f'Successfull rollback of transaction "{self.txn_id}".')
109+
logging.debug(f'Successful rollback of transaction "{self.txn_id}".')
110110
except Exception:
111111
logging.error(
112112
f'Error while rolling back transaction "{self.txn_id}"',

cmapi/cmapi_server/node_manipulation.py

Lines changed: 41 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -113,8 +113,8 @@ def add_node(
113113
if not read_only:
114114
_rebalance_dbroots(c_root)
115115
_move_primary_node(c_root)
116-
else:
117-
add_dbroots_of_other_nodes(c_root, pm_num)
116+
117+
update_dbroots_of_readonly_nodes(c_root)
118118
except Exception:
119119
logging.error(
120120
'Caught exception while adding node, config file is unchanged',
@@ -187,8 +187,7 @@ def remove_node(
187187
_rebalance_dbroots(c_root)
188188
_move_primary_node(c_root)
189189

190-
if is_read_only:
191-
remove_dbroots_of_node(c_root, pm_num)
190+
update_dbroots_of_readonly_nodes(c_root)
192191
else:
193192
# TODO:
194193
# - IMO undefined behaviour here. Removing one single node
@@ -262,7 +261,7 @@ def rebalance_dbroots(
262261
#
263262
# returns the id of the new dbroot on success
264263
# raises an exception on error
265-
def add_dbroot(input_config_filename = None, output_config_filename = None, host = None):
264+
def add_dbroot(input_config_filename = None, output_config_filename = None, host = None) -> int:
266265
node_config = NodeConfig()
267266
if input_config_filename is None:
268267
c_root = node_config.get_current_config_root()
@@ -1185,26 +1184,54 @@ class NodeNotFoundException(Exception):
11851184
pass
11861185

11871186

1187+
def get_pm_module_num_to_addr_map(root: etree.Element) -> dict[int, str]:
1188+
"""Get a mapping of PM module numbers to their IP addresses"""
1189+
module_num_to_addr = {}
1190+
smc_node = root.find("./SystemModuleConfig")
1191+
mod_count = int(smc_node.find("./ModuleCount3").text)
1192+
for i in range(1, mod_count + 1):
1193+
ip_addr = smc_node.find(f"./ModuleIPAddr{i}-1-3").text
1194+
module_num_to_addr[i] = ip_addr
1195+
return module_num_to_addr
1196+
1197+
1198+
def update_dbroots_of_readonly_nodes(root: etree.Element) -> None:
1199+
"""Read-only nodes do not have their own dbroots, but they must have all the dbroots of the other nodes
1200+
So this function sets list of dbroots of each read-only node to the list of all the dbroots in the cluster
1201+
"""
1202+
nc = NodeConfig()
1203+
pm_num_to_addr = get_pm_module_num_to_addr_map(root)
1204+
for ro_node in nc.get_read_only_nodes(root):
1205+
# Get PM num by IP address
1206+
this_ip_pm_num = None
1207+
for pm_num, pm_addr in pm_num_to_addr.items():
1208+
if pm_addr == ro_node:
1209+
this_ip_pm_num = pm_num
1210+
break
1211+
1212+
if this_ip_pm_num is not None:
1213+
# Add dbroots of other nodes to this read-only node
1214+
add_dbroots_of_other_nodes(root, this_ip_pm_num)
1215+
else: # This should not happen
1216+
err_msg = f"Could not find PM number for read-only node {ro_node}"
1217+
logging.error(err_msg)
1218+
raise NodeNotFoundException(err_msg)
1219+
1220+
11881221
def add_dbroots_of_other_nodes(root: etree.Element, module_num: int) -> None:
11891222
"""Adds all the dbroots listed in the config to this (read-only) node"""
11901223
existing_dbroots = _get_existing_db_roots(root)
11911224
sysconf_node = root.find("./SystemModuleConfig")
11921225

1226+
# Remove existing dbroots from this module
1227+
remove_dbroots_of_node(root, module_num)
1228+
11931229
# Write node's dbroot count
1194-
dbroot_count_node = sysconf_node.find(f"./ModuleDBRootCount{module_num}-3")
1195-
if dbroot_count_node is not None:
1196-
sysconf_node.remove(dbroot_count_node)
11971230
dbroot_count_node = etree.SubElement(
11981231
sysconf_node, f"ModuleDBRootCount{module_num}-3"
11991232
)
12001233
dbroot_count_node.text = str(len(existing_dbroots))
12011234

1202-
# Remove existing dbroot IDs of this module if present
1203-
for i in range(1, 100):
1204-
dbroot_id_node = sysconf_node.find(f"./ModuleDBRootID{module_num}-{i}-3")
1205-
if dbroot_id_node is not None:
1206-
sysconf_node.remove(dbroot_id_node)
1207-
12081235
# Write new dbroot IDs to the module mapping
12091236
for i, dbroot_id in enumerate(existing_dbroots, start=1):
12101237
dbroot_id_node = etree.SubElement(
@@ -1221,10 +1248,6 @@ def remove_dbroots_of_node(root: etree.Element, module_num: int) -> None:
12211248
dbroot_count_node = sysconf_node.find(f"./ModuleDBRootCount{module_num}-3")
12221249
if dbroot_count_node is not None:
12231250
sysconf_node.remove(dbroot_count_node)
1224-
else:
1225-
logging.error(
1226-
f"ModuleDBRootCount{module_num}-3 not found in SystemModuleConfig"
1227-
)
12281251

12291252
# Remove existing dbroot IDs
12301253
for i in range(1, 100):

cmapi/cmapi_server/test/test_node_manip.py

Lines changed: 81 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
import logging
22
import socket
33
import unittest
4-
from unittest.mock import ANY, patch
4+
from unittest.mock import patch
55

66
from lxml import etree
77

88
from cmapi_server import node_manipulation
99
from cmapi_server.constants import MCS_DATA_PATH
10-
from cmapi_server.node_manipulation import add_dbroots_of_other_nodes, remove_dbroots_of_node
10+
from cmapi_server.node_manipulation import add_dbroots_of_other_nodes, remove_dbroots_of_node, update_dbroots_of_readonly_nodes
1111
from cmapi_server.test.unittest_global import BaseNodeManipTestCase, tmp_mcs_config_filename
1212
from mcs_node_control.models.node_config import NodeConfig
1313

@@ -23,12 +23,18 @@ def test_add_remove_node(self):
2323
'./test-output0.xml','./test-output1.xml','./test-output2.xml'
2424
)
2525
hostaddr = socket.gethostbyname(socket.gethostname())
26-
node_manipulation.add_node(
27-
self.NEW_NODE_NAME, tmp_mcs_config_filename, self.tmp_files[0]
28-
)
29-
node_manipulation.add_node(
30-
hostaddr, self.tmp_files[0], self.tmp_files[1]
31-
)
26+
27+
with patch('cmapi_server.node_manipulation.update_dbroots_of_readonly_nodes') as mock_update_dbroots_of_readonly_nodes:
28+
node_manipulation.add_node(
29+
self.NEW_NODE_NAME, tmp_mcs_config_filename, self.tmp_files[0]
30+
)
31+
mock_update_dbroots_of_readonly_nodes.assert_called_once()
32+
mock_update_dbroots_of_readonly_nodes.reset_mock()
33+
34+
node_manipulation.add_node(
35+
hostaddr, self.tmp_files[0], self.tmp_files[1]
36+
)
37+
mock_update_dbroots_of_readonly_nodes.assert_called_once()
3238

3339
# get a NodeConfig, read test.xml
3440
# look for some of the expected changes.
@@ -42,10 +48,13 @@ def test_add_remove_node(self):
4248
node = root.find("./ExeMgr2/IPAddr")
4349
self.assertEqual(node.text, hostaddr)
4450

45-
node_manipulation.remove_node(
46-
self.NEW_NODE_NAME, self.tmp_files[1], self.tmp_files[2],
47-
test_mode=True
48-
)
51+
with patch('cmapi_server.node_manipulation.update_dbroots_of_readonly_nodes') as mock_update_dbroots_of_readonly_nodes:
52+
node_manipulation.remove_node(
53+
self.NEW_NODE_NAME, self.tmp_files[1], self.tmp_files[2],
54+
test_mode=True
55+
)
56+
mock_update_dbroots_of_readonly_nodes.assert_called_once()
57+
4958
nc = NodeConfig()
5059
root = nc.get_current_config_root(self.tmp_files[2])
5160
node = root.find('./PMS1/IPAddr')
@@ -67,8 +76,7 @@ def test_add_remove_read_only_node(self):
6776
# Mock _rebalance_dbroots and _move_primary_node (only after the first node is added)
6877
with patch('cmapi_server.node_manipulation._rebalance_dbroots') as mock_rebalance_dbroots, \
6978
patch('cmapi_server.node_manipulation._move_primary_node') as mock_move_primary_node, \
70-
patch('cmapi_server.node_manipulation.add_dbroots_of_other_nodes') as mock_add_dbroots_of_other_nodes, \
71-
patch('cmapi_server.node_manipulation.remove_dbroots_of_node') as mock_remove_dbroots_of_node:
79+
patch('cmapi_server.node_manipulation.update_dbroots_of_readonly_nodes') as mock_update_dbroots_of_readonly_nodes:
7280

7381
# Add a read-only node
7482
node_manipulation.add_node(
@@ -94,7 +102,8 @@ def test_add_remove_read_only_node(self):
94102

95103
mock_rebalance_dbroots.assert_not_called()
96104
mock_move_primary_node.assert_not_called()
97-
mock_add_dbroots_of_other_nodes.assert_called_once_with(ANY, 2)
105+
mock_update_dbroots_of_readonly_nodes.assert_called_once()
106+
mock_update_dbroots_of_readonly_nodes.reset_mock()
98107

99108
# Test read-only node removal
100109
node_manipulation.remove_node(
@@ -109,7 +118,7 @@ def test_add_remove_read_only_node(self):
109118

110119
mock_rebalance_dbroots.assert_not_called()
111120
mock_move_primary_node.assert_not_called()
112-
mock_remove_dbroots_of_node.assert_called_once_with(ANY, 2)
121+
mock_update_dbroots_of_readonly_nodes.assert_called_once()
113122

114123

115124
def test_add_dbroots_nodes_rebalance(self):
@@ -271,13 +280,23 @@ def test_unassign_dbroot1(self):
271280
self.assertTrue(caught_it)
272281

273282

274-
class TestReadOnlyNodeDBRootsManip(unittest.TestCase):
275-
our_module_idx = 2
283+
class TestDBRootsManipulation(unittest.TestCase):
284+
our_module_idx = 3
285+
ro_node1_ip = '192.168.1.3'
286+
ro_node2_ip = '192.168.1.4'
276287

277288
def setUp(self):
278-
# Mock initial XML structure (add two dbroots)
289+
# Mock initial XML structure (add two nodes and two dbroots)
279290
self.root = etree.Element('Columnstore')
280-
etree.SubElement(self.root, 'SystemModuleConfig')
291+
# Add two PM modules with IP addresses
292+
smc = etree.SubElement(self.root, 'SystemModuleConfig')
293+
module_count = etree.SubElement(smc, 'ModuleCount3')
294+
module_count.text = '2'
295+
module1_ip = etree.SubElement(smc, 'ModuleIPAddr1-1-3')
296+
module1_ip.text = '192.168.1.1'
297+
module2_ip = etree.SubElement(smc, 'ModuleIPAddr2-1-3')
298+
module2_ip.text = '192.168.1.2'
299+
281300
system_config = etree.SubElement(self.root, 'SystemConfig')
282301
dbroot_count = etree.SubElement(system_config, 'DBRootCount')
283302
dbroot_count.text = '2'
@@ -286,6 +305,15 @@ def setUp(self):
286305
dbroot2 = etree.SubElement(system_config, 'DBRoot2')
287306
dbroot2.text = '/data/dbroot2'
288307

308+
def test_get_pm_module_num_to_addr_map(self):
309+
result = node_manipulation.get_pm_module_num_to_addr_map(self.root)
310+
311+
expected = {
312+
1: '192.168.1.1',
313+
2: '192.168.1.2',
314+
}
315+
self.assertEqual(result, expected)
316+
289317
def test_add_dbroots_of_other_nodes(self):
290318
'''add_dbroots_of_other_nodes must add dbroots of other nodes into mapping of the node.'''
291319
add_dbroots_of_other_nodes(self.root, self.our_module_idx)
@@ -325,3 +353,36 @@ def test_remove_dbroots_of_node(self):
325353
dbroot2 = self.root.find(f'./SystemModuleConfig/ModuleDBRootID{self.our_module_idx}-2-3')
326354
self.assertIsNone(dbroot1)
327355
self.assertIsNone(dbroot2)
356+
357+
def test_update_dbroots_of_readonly_nodes(self):
358+
"""Test that update_dbroots_of_readonly_nodes adds all existing dbroots to all existing read-only nodes"""
359+
# Add two new new modules to the XML structure (two already exist)
360+
smc = self.root.find('./SystemModuleConfig')
361+
module_count = smc.find('./ModuleCount3')
362+
module_count.text = '4'
363+
module3_ip = etree.SubElement(smc, 'ModuleIPAddr3-1-3')
364+
module3_ip.text = self.ro_node1_ip
365+
module4_ip = etree.SubElement(smc, 'ModuleIPAddr4-1-3')
366+
module4_ip.text = self.ro_node2_ip
367+
# Add them to ReadOnlyNodes
368+
read_only_nodes = etree.SubElement(self.root, 'ReadOnlyNodes')
369+
for ip in [self.ro_node1_ip, self.ro_node2_ip]:
370+
node = etree.SubElement(read_only_nodes, 'Node')
371+
node.text = ip
372+
373+
update_dbroots_of_readonly_nodes(self.root)
374+
375+
# Check that read only nodes have all the dbroots
376+
for ro_module_idx in range(3, 5):
377+
module_count = self.root.find(f'./SystemModuleConfig/ModuleDBRootCount{ro_module_idx}-3')
378+
self.assertIsNotNone(module_count)
379+
self.assertEqual(module_count.text, '2')
380+
381+
dbroot1 = self.root.find(f'./SystemModuleConfig/ModuleDBRootID{ro_module_idx}-1-3')
382+
dbroot2 = self.root.find(f'./SystemModuleConfig/ModuleDBRootID{ro_module_idx}-2-3')
383+
self.assertIsNotNone(dbroot1)
384+
self.assertIsNotNone(dbroot2)
385+
self.assertEqual(dbroot1.text, '1')
386+
self.assertEqual(dbroot2.text, '2')
387+
388+

cmapi/mcs_node_control/models/node_config.py

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import configparser
2+
from contextlib import contextmanager
23
import grp
34
import logging
45
import pwd
@@ -7,6 +8,7 @@
78
from os import chown, mkdir, replace
89
from pathlib import Path
910
from shutil import copyfile
11+
from typing import Optional
1012
from xml.dom import minidom # to pick up pretty printing functionality
1113

1214
from lxml import etree
@@ -136,6 +138,26 @@ def write_config(self, tree, filename=DEFAULT_MCS_CONF_PATH):
136138
f.write(self.to_string(tree))
137139
replace(tmp_filename, filename) # atomic replacement
138140

141+
@contextmanager
142+
def modify_config(
143+
self,
144+
input_config_filename: str = DEFAULT_MCS_CONF_PATH,
145+
output_config_filename: Optional[str] = None,
146+
):
147+
"""Context manager to modify the config file
148+
If exception is raised, the config file is not modified and exception is re-raised
149+
If output_config_filename is not provided, the input config file is modified
150+
"""
151+
try:
152+
c_root = self.get_current_config_root(input_config_filename)
153+
yield c_root
154+
except Exception as e:
155+
logging.error(f"modify_config(): Caught exception: '{str(e)}', config file not modified")
156+
raise
157+
else:
158+
output_config_filename = output_config_filename or input_config_filename
159+
self.write_config(c_root, output_config_filename)
160+
139161
def to_string(self, tree):
140162
# TODO: try to use lxml to do this to avoid the add'l dependency
141163
xmlstr = minidom.parseString(etree.tostring(tree)).toprettyxml(

0 commit comments

Comments
 (0)