Projections support (#290)

* add pairwise addition/product

* add logical reductions

* add tests for section-based reductions

* wip

* reductions are now optimized for numpy types

* numpy reductions always preferred when available

* added checks that single-value reductions work with sections

* add section reduction test

* no fstrings allowed

* remove unused time import

* scookies -> _scookies

* add check so overflowing number of futures is not silent

* cleanup

* remove unused operator import

* use newest import

* operator import is actually required

* make new code flake8 compliant

* update

* add EP name registration

* projections support for groups

* added CkRegisterMainChare to list of methods that are now correctly traced

* section entry methods are now traced

* added registration to ArrayMaps

* add registration support for ArrayMap

* Add data required for projections to registration

* Removed global entryNames/entryNamesLen. Unnecessary 'emStart'
  parameter removed

* Moved entry method name tracking inside charmlib

* Do we want to free the memory?

* Add '--enable-tracing' option to setup.py

* Checkout the charm4py_projections branch for running tests

* Temporarily disable the ctypes interface in testing

* Add tracing of user events

* Add object store implementation

* Buffer task until object is available

* Fix buffering messages on target PE

* Fix segmentation fault with object store

* Add some documentation for the object store

* Implemented waiting for futures

* Add parameter server training example

* Add requested flag to futures to avoid multiple requests for the same object

* Add ray.wait function

* Add async parameter server

* Implement ray.put function to insert object in the object store

* Merge ray futures and charm futures

* Add store option to futures

* Check future arguments in group methods

* Try ray tasks implementation using charm4py pool

* pool example

* Remove checking for futures from groups

* Stateless tasks and exit calls work

* Working stateless tasks implementation

* Add more examples

* Fix ray.get calls from remote functions

* Add fib example to debug MPI issue

* For small objects, store them at home

* Revert "merge ray into projections support"

This reverts commit 44acea4, reversing
changes made to 4f65ec6.

undo merge of ray due to compilation issues

* remove generated files

* merge fixes

* merge fixes 2

* merge fixes 3

* add back entry names to libRegisterFunc:

---------

Co-authored-by: Zane Fink <finkzane@gmail.com>
Co-authored-by: Aditya Bhosale <adityapb1546@gmail.com>
Co-authored-by: Ritvik Rao <rao1@dt-login01.delta.ncsa.illinois.edu>
Co-authored-by: Ritvik Rao <rao1@dt-login02.delta.ncsa.illinois.edu>
Co-authored-by: Ritvik Rao <ritvikrao@li-4c4c4544-0039-4610-8033-b2c04f574833.ibm.com>

Author: 6 people

charm4py/charm.py

@@ -493,9 +493,8 @@ def packMsg(self, destObj, msgArgs, header):
     def registerInCharmAs(self, C, charm_type, libRegisterFunc):
         charm_type_id = charm_type.type_id
         entryMethods = self.classEntryMethods[charm_type_id][C]
-        # if self.myPe() == 0: print("charm4py:: Registering class " + C.__name__ + " in Charm with " + str(len(entryMethods)) + " entry methods " + str([e.name for e in entryMethods]))
-        C.idx[charm_type_id], startEpIdx = libRegisterFunc(C.__name__ + str(charm_type_id), len(entryMethods))
-        # if self.myPe() == 0: print("charm4py:: Chare idx=" + str(C.idx[charm_type_id]) + " ctor Idx=" + str(startEpIdx))
+        entryNames = [method.name for method in entryMethods]
+        C.idx[charm_type_id], startEpIdx = libRegisterFunc(C.__name__ + str(charm_type_id), entryNames, len(entryMethods))
         for i, em in enumerate(entryMethods):
             em.epIdx = startEpIdx + i
             self.entryMethods[em.epIdx] = em

charm4py/charmlib/ccharm.pxd

@@ -22,11 +22,11 @@ cdef extern from "charm.h":
     int CmiPhysicalRank(int pe);
 
     void CkRegisterReadonlyExt(const char *name, const char *type, size_t msgSize, char *msg);
-    void CkRegisterMainChareExt(const char *s, int numEntryMethods, int *chareIdx, int *startEpIdx);
-    void CkRegisterGroupExt(const char *s, int numEntryMethods, int *chareIdx, int *startEpIdx);
-    void CkRegisterSectionManagerExt(const char *s, int numEntryMethods, int *chareIdx, int *startEpIdx);
-    void CkRegisterArrayExt(const char *s, int numEntryMethods, int *chareIdx, int *startEpIdx);
-    void CkRegisterArrayMapExt(const char *s, int numEntryMethods, int *chareIdx, int *startEpIdx);
+    void CkRegisterMainChareExt(const char *s, const char **emNames, int emNamesStart, int numEntryMethods, int *chareIdx, int *startEpIdx);
+    void CkRegisterGroupExt(const char *s, const char **emNames, int emNamesStart, int numEntryMethods, int *chareIdx, int *startEpIdx);
+    void CkRegisterSectionManagerExt(const char *s, const char **emNames, int emNamesStart, int numEntryMethods, int *chareIdx, int *startEpIdx);
+    void CkRegisterArrayExt(const char *s, const char **emNames, int emNamesStart, int numEntryMethods, int *chareIdx, int *startEpIdx);
+    void CkRegisterArrayMapExt(const char *s, const char **emNames, int emNamesStart, int numEntryMethods, int *chareIdx, int *startEpIdx);
 
     int CkCreateGroupExt(int cIdx, int eIdx, int num_bufs, char **bufs, int *buf_sizes);
     int CkCreateArrayExt(int cIdx, int ndims, int *dims, int eIdx, int num_bufs, char **bufs, int *buf_sizes, int map_gid, char useAtSync);
@@ -72,6 +72,10 @@ cdef extern from "charm.h":
 
     void CkHapiAddCallback(long stream, void (*cb)(void*, void*), int fid);
 
+    int CkTraceRegisterUserEvent(char *EventDesc, int eventID);
+    void CkTraceBeginUserBracketEvent(int eventID);
+    void CkTraceEndUserBracketEvent(int eventID);
+
 cdef extern from "conv-header.h":
     ctypedef void (*CmiHandler)(void* )
     cdef const int CmiReservedHeaderSize

charm4py/charmlib/charmlib_cython.pyx

@@ -12,6 +12,11 @@ from cpython.ref cimport Py_INCREF
 from cpython.mem cimport PyMem_Malloc, PyMem_Realloc, PyMem_Free
 from cython.operator cimport dereference
 
+cdef extern from "Python.h":
+    char* PyUnicode_AsUTF8(object unicode)
+
+from libc.string cimport strcmp
+
 from ..charm import Charm4PyError
 from .. import reduction as red
 from cpython cimport array
@@ -108,6 +113,17 @@ ctypedef struct CkReductionTypesExt:
 
 cdef extern CkReductionTypesExt charm_reducers
 
+cdef char ** to_cstring_array(list_str):
+    cdef char **ret = <char **>malloc(len(list_str) * sizeof(char *))
+    for i in range(len(list_str)):
+        ret[i] = list_str[i]
+    return ret
+
+def addStringsToList(targetList, strings):
+  for idx, currentString in enumerate(strings):
+    targetList.append(<char*> malloc(len(currentString)))
+    targetList[idx] = strings[idx]
+
 class CkReductionTypesExt_Wrapper:
 
   def __init__(self):
@@ -347,6 +363,8 @@ class CharmLib(object):
     self.direct_copy_supported = (PY_MAJOR_VERSION >= 3)
     self.name = 'cython'
     self.chareNames = []
+    self.emNames = []
+    self.emStart = 0
     self.init()
     self.ReducerType = CkReductionTypesExt_Wrapper()
     #print(charm_reducers.sum_long, charm_reducers.product_ushort, charm_reducers.max_char, charm_reducers.max_float, charm_reducers.min_char)
@@ -503,34 +521,51 @@ class CharmLib(object):
     if msg is None: CkRegisterReadonlyExt(n1, n2, 0, NULL)
     else: CkRegisterReadonlyExt(n1, n2, len(msg), msg)
 
-  def CkRegisterMainchare(self, str name, int numEntryMethods):
+  def CkRegisterMainchare(self, str name, list entryMethodNames, int numEntryMethods):
     self.chareNames.append(name.encode())
     cdef int chareIdx, startEpIdx
-    CkRegisterMainChareExt(self.chareNames[-1], numEntryMethods, &chareIdx, &startEpIdx)
+    self.emNames += [name.encode() for name in entryMethodNames]
+    # TODO: do we want to track/free these pointers?
+    cdef char** c1 = to_cstring_array(self.emNames)
+    CkRegisterMainChareExt(self.chareNames[-1], c1, self.emStart, numEntryMethods, &chareIdx, &startEpIdx)
+    self.emStart = len(self.emNames)
     return chareIdx, startEpIdx
 
-  def CkRegisterGroup(self, str name, int numEntryMethods):
+  def CkRegisterGroup(self, str name, list entryMethodNames, int numEntryMethods):
     self.chareNames.append(name.encode())
     cdef int chareIdx, startEpIdx
-    CkRegisterGroupExt(self.chareNames[-1], numEntryMethods, &chareIdx, &startEpIdx)
+    self.emNames += [name.encode() for name in entryMethodNames]
+    cdef char** c1 = to_cstring_array(self.emNames)
+    CkRegisterGroupExt(self.chareNames[-1], c1, self.emStart, numEntryMethods, &chareIdx, &startEpIdx)
+    self.emStart = len(self.emNames)
     return chareIdx, startEpIdx
 
-  def CkRegisterSectionManager(self, str name, int numEntryMethods):
+  def CkRegisterSectionManager(self, str name, list entryMethodNames, int numEntryMethods):
     self.chareNames.append(name.encode())
     cdef int chareIdx, startEpIdx
-    CkRegisterSectionManagerExt(self.chareNames[-1], numEntryMethods, &chareIdx, &startEpIdx)
+    self.emNames += [name.encode() for name in entryMethodNames]
+    cdef char** c1 = to_cstring_array(self.emNames)
+
+    CkRegisterSectionManagerExt(self.chareNames[-1], c1, self.emStart, numEntryMethods, &chareIdx, &startEpIdx)
+    self.emStart = len(self.emNames)
     return chareIdx, startEpIdx
 
-  def CkRegisterArrayMap(self, str name, int numEntryMethods):
+  def CkRegisterArrayMap(self, str name, list entryMethodNames, int numEntryMethods):
     self.chareNames.append(name.encode())
     cdef int chareIdx, startEpIdx
-    CkRegisterArrayMapExt(self.chareNames[-1], numEntryMethods, &chareIdx, &startEpIdx)
+    self.emNames += [name.encode() for name in entryMethodNames]
+    cdef char** c1 = to_cstring_array(self.emNames)
+    CkRegisterArrayMapExt(self.chareNames[-1], c1, self.emStart, numEntryMethods, &chareIdx, &startEpIdx)
+    self.emStart = len(self.emNames)
     return chareIdx, startEpIdx
 
-  def CkRegisterArray(self, str name, int numEntryMethods):
+  def CkRegisterArray(self, str name, list entryMethodNames, int numEntryMethods):
     self.chareNames.append(name.encode())
     cdef int chareIdx, startEpIdx
-    CkRegisterArrayExt(self.chareNames[-1], numEntryMethods, &chareIdx, &startEpIdx)
+    self.emNames += [name.encode() for name in entryMethodNames]
+    cdef char** c1 = to_cstring_array(self.emNames)
+    CkRegisterArrayExt(self.chareNames[-1], c1, self.emStart, numEntryMethods, &chareIdx, &startEpIdx)
+    self.emStart = len(self.emNames)
     return chareIdx, startEpIdx
 
   def CkCreateGroup(self, int chareIdx, int epIdx, msg not None):
@@ -852,6 +887,20 @@ class CharmLib(object):
   def scheduleTagAfter(self, int tag, double msecs):
     CcdCallFnAfter(CcdCallFnAfterCallback, <void*>tag, msecs)
 
+  def traceRegisterUserEvent(self, str EventDesc, int eventNum=-1):
+    cdef bytes py_bytes = EventDesc.encode()
+    Py_INCREF(py_bytes)
+    # This memory needs to be managed somehow, I think
+    cdef char* c_string = py_bytes
+    cdef int eventID = CkTraceRegisterUserEvent(c_string, eventNum)
+    return eventID
+
+  def traceBeginUserBracketEvent(self, int EventID):
+    CkTraceBeginUserBracketEvent(EventID)
+
+  def traceEndUserBracketEvent(self, int EventID):
+    CkTraceEndUserBracketEvent(EventID)
+
   def CcsRegisterHandler(self, str handlername, object handler):
     cdef bytes handler_bytes = handlername.encode("utf-8")
     cdef const char* c_handlername = handler_bytes

charm4py/wait.py

@@ -192,7 +192,7 @@ def is_tag_cond(root_ast):
             idx = idx.n
         elif isinstance(idx, ast.Constant):
             idx = idx.value
-
+        
         if not isinstance(idx, int):
             return None
 

examples/hello/group_hello.py

@@ -26,4 +26,4 @@ def main(args):
     group_proxy[0].sayHi(17)
 
 
-charm.start(main)
+charm.start(main)

setup.py

@@ -17,7 +17,7 @@
 Cython.Compiler.Options.annotate = True
 
 build_mpi = False
-
+enable_tracing = False
 
 
 def get_build_machine():
@@ -149,6 +149,9 @@ def build_libcharm(charm_src_dir, build_dir):
         build_num_cores = max(int(os.environ.get('CHARM_BUILD_PROCESSES', multiprocessing.cpu_count() // 2)), 1)
         extra_build_opts = os.environ.get('CHARM_EXTRA_BUILD_OPTS', '')
 
+        if enable_tracing:
+         extra_build_opts += " --enable-tracing "
+        
         target_machine, os_target, target_layer = get_build_triple(build_mpi)
 
         build_triple = f'{target_layer}-{os_target}-{target_machine}'
@@ -198,17 +201,23 @@ def build_libcharm(charm_src_dir, build_dir):
 class custom_install(install, object):
 
     user_options = install.user_options + [
-        ('mpi', None, 'Build libcharm with MPI')
+        ('mpi', None, 'Build libcharm with MPI'),
+        ('enable-tracing', None, 'Build libcharm with tracing enabled')
     ]
 
     def initialize_options(self):
         install.initialize_options(self)
         self.mpi = False
+        self.enable_tracing = False
 
     def finalize_options(self):
         global build_mpi
         if not build_mpi:
             build_mpi = bool(self.mpi)
+
+        global enable_tracing
+        if not enable_tracing:
+            enable_tracing = bool(self.enable_tracing)
         install.finalize_options(self)
 
     def run(self):
@@ -218,17 +227,22 @@ def run(self):
 class custom_build_py(build_py, object):
 
     user_options = build_py.user_options + [
-        ('mpi', None, 'Build libcharm with MPI')
+        ('mpi', None, 'Build libcharm with MPI'),
+        ('enable-tracing', None, 'Build libcharm with tracing enabled')
     ]
 
     def initialize_options(self):
         build_py.initialize_options(self)
         self.mpi = False
+        self.enable_tracing = False
 
     def finalize_options(self):
         global build_mpi
         if not build_mpi:
             build_mpi = bool(self.mpi)
+        global enable_tracing
+        if not enable_tracing:
+            enable_tracing = bool(self.enable_tracing)
         build_py.finalize_options(self)
 
     def run(self):
@@ -241,17 +255,23 @@ def run(self):
 class custom_build_ext(build_ext, object):
 
     user_options = build_ext.user_options + [
-        ('mpi', None, 'Build libcharm with MPI')
+        ('mpi', None, 'Build libcharm with MPI'),
+        ('enable-tracing', None, 'Build libcharm with tracing enabled')
     ]
 
     def initialize_options(self):
         build_ext.initialize_options(self)
         self.mpi = False
+        self.enable_tracing = False
 
     def finalize_options(self):
         global build_mpi
         if not build_mpi:
             build_mpi = bool(self.mpi)
+
+        global enable_tracing
+        if not enable_tracing:
+            enable_tracing = bool(self.enable_tracing)
         build_ext.finalize_options(self)
 
     def run(self):