Reduce default values for buffer sizes (#66)

Author: jeffreylovitz

redisgraph_bulk_loader/bulk_insert.py

@@ -66,8 +66,8 @@ def process_entities(entities):
 @click.option('--escapechar', '-x', default='\\', help='the escape char used for the CSV reader (default \\). Use "none" for None.')
 # Buffer size restrictions
 @click.option('--max-token-count', '-c', default=1024, help='max number of processed CSVs to send per query (default 1024)')
-@click.option('--max-buffer-size', '-b', default=512, help='max buffer size in megabytes (default 512)')
-@click.option('--max-token-size', '-t', default=500, help='max size of each token in megabytes (default 500, max 512)')
+@click.option('--max-buffer-size', '-b', default=128, help='max buffer size in megabytes (default 128, max 1024)')
+@click.option('--max-token-size', '-t', default=128, help='max size of each token in megabytes (default 128, max 512)')
 @click.option('--index', '-i', multiple=True, help='Label:Propery on which to create an index')
 @click.option('--full-text-index', '-f', multiple=True, help='Label:Propery on which to create an full text search index')
 def bulk_insert(graph, host, port, password, user, unix_socket_path, nodes, nodes_with_label, relations, relations_with_type, separator, enforce_schema, skip_invalid_nodes, skip_invalid_edges, escapechar, quote, max_token_count, max_buffer_size, max_token_size, index, full_text_index):

redisgraph_bulk_loader/config.py

@@ -1,16 +1,16 @@
 class Config:
-    def __init__(self, max_token_count=1024 * 1023, max_buffer_size=2_048, max_token_size=512, enforce_schema=False, skip_invalid_nodes=False, skip_invalid_edges=False, separator=',', quoting=3, store_node_identifiers=False, escapechar='\\'):
+    def __init__(self, max_token_count=1024 * 1023, max_buffer_size=128, max_token_size=128, enforce_schema=False, skip_invalid_nodes=False, skip_invalid_edges=False, separator=',', quoting=3, store_node_identifiers=False, escapechar='\\'):
         """Settings for this run of the bulk loader"""
         # Maximum number of tokens per query
         # 1024 * 1024 is the hard-coded Redis maximum. We'll set a slightly lower limit so
         # that we can safely ignore tokens that aren't binary strings
         # ("GRAPH.BULK", "BEGIN", graph name, counts)
         self.max_token_count = min(max_token_count, 1024 * 1023)
         # Maximum size in bytes per query
-        self.max_buffer_size = max_buffer_size * 1_000_000
+        self.max_buffer_size = min(max_buffer_size * 1_000_000, 1024 * 1_000_000)
         # Maximum size in bytes per token
         # 512 megabytes is a hard-coded Redis maximum
-        self.max_token_size = min(max_token_size * 1_000_000, 512 * 1_000_000)
+        self.max_token_size = min(max_token_size * 1_000_000, 512 * 1_000_000, self.max_buffer_size)
 
         self.enforce_schema = enforce_schema
         self.skip_invalid_nodes = skip_invalid_nodes

test/test_config.py

@@ -7,8 +7,8 @@ def test01_default_values(self):
         """Verify the default values in the Config class."""
         config = Config()
         self.assertEqual(config.max_token_count, 1024 * 1023)
-        self.assertEqual(config.max_buffer_size, 2_048_000_000)
-        self.assertEqual(config.max_token_size, 512_000_000)
+        self.assertEqual(config.max_buffer_size, 128_000_000)
+        self.assertEqual(config.max_token_size, 128_000_000)
         self.assertEqual(config.enforce_schema, False)
         self.assertEqual(config.skip_invalid_nodes, False)
         self.assertEqual(config.skip_invalid_edges, False)
@@ -18,10 +18,10 @@ def test01_default_values(self):
 
     def test02_modified_values(self):
         """Verify that Config_set updates Config class values accordingly."""
-        config = Config(max_token_count=10, max_buffer_size=100, max_token_size=200, enforce_schema=True, skip_invalid_nodes=True, skip_invalid_edges=True, separator='|', quoting=0)
+        config = Config(max_token_count=10, max_buffer_size=500, max_token_size=200, enforce_schema=True, skip_invalid_nodes=True, skip_invalid_edges=True, separator='|', quoting=0)
         self.assertEqual(config.max_token_count, 10)
         self.assertEqual(config.max_token_size, 200_000_000) # Max token size argument is converted to megabytes
-        self.assertEqual(config.max_buffer_size, 100_000_000) # Buffer size argument is converted to megabytes
+        self.assertEqual(config.max_buffer_size, 500_000_000) # Buffer size argument is converted to megabytes
         self.assertEqual(config.enforce_schema, True)
         self.assertEqual(config.skip_invalid_nodes, True)
         self.assertEqual(config.skip_invalid_edges, True)