Start sending codeSourceId with Cloud Push (#593)

* Initial solution

* Add unit tests

* Add missing force parameter in recursive _push_project call

* Add --force suggestion

* Move --force hint to the general try-except

Author: JosueNina

lean/commands/cloud/push.py

@@ -34,7 +34,10 @@
 @option("--key",
               type=PathParameter(exists=True, file_okay=True, dir_okay=False),
               help="Path to the encryption key to use")
-def push(project: Optional[Path], encrypt: Optional[bool], decrypt: Optional[bool], key: Optional[Path]) -> None:
+@option("--force",
+        is_flag=True, default=False,
+        help="Force push even if there's a lock conflict")
+def push(project: Optional[Path], encrypt: Optional[bool], decrypt: Optional[bool], key: Optional[Path], force: Optional[bool]) -> None:
     """Push local projects to QuantConnect.
 
     This command overrides the content of cloud files with the content of their respective local counterparts.
@@ -61,11 +64,11 @@ def push(project: Optional[Path], encrypt: Optional[bool], decrypt: Optional[boo
 
         if encrypt and key is not None:
             from lean.components.util.encryption_helper import validate_encryption_key_registered_with_cloud
-            validate_encryption_key_registered_with_cloud(key, container.organization_manager, container.api_client) 
+            validate_encryption_key_registered_with_cloud(key, container.organization_manager, container.api_client)
 
-        push_manager.push_project(project, encryption_action, key)
+        push_manager.push_project(project, encryption_action, key, force)
     else:
         if key is not None:
             raise RuntimeError(f"Encryption key can only be specified when pushing a single project.")
         projects_to_push = [p.parent for p in Path.cwd().rglob(PROJECT_CONFIG_FILE_NAME)]
-        push_manager.push_projects(projects_to_push, [], encryption_action, key)
+        push_manager.push_projects(projects_to_push, [], encryption_action, key, force)

lean/components/api/project_client.py

@@ -83,7 +83,8 @@ def update(self,
                python_venv: Optional[int] = None,
                files: Optional[List[Dict[str, str]]] = None,
                libraries: Optional[List[int]] = None,
-               encryption_key: Optional[str] = None) -> None:
+               encryption_key: Optional[str] = None,
+               code_source_id: Optional[str] = None) -> None:
         """Updates an existing project.
 
         :param project_id: the id of the project to update
@@ -94,6 +95,7 @@ def update(self,
         :param python_venv: the python venv id for the project, or None if the python venv shouldn't be changed
         :param files: the list of files for the project
         :param libraries: the list of libraries referenced by the project
+        :param code_source_id: the source of the code changes (e.g., "cli")
         """
         request_parameters = {
             "projectId": project_id
@@ -136,7 +138,10 @@ def update(self,
 
         if encryption_key is not None:
             request_parameters["encryptionKey"] = encryption_key
-            
+
+        if code_source_id is not None:
+            request_parameters["codeSourceId"] = code_source_id
+
         self._api.post("projects/update", request_parameters, data_as_json=False)
 
     def delete(self, project_id: int) -> None:

lean/components/cloud/push_manager.py

@@ -46,17 +46,17 @@ def __init__(self,
         self._organization_manager = organization_manager
         self._cloud_projects = []
 
-    def push_project(self, project: Path, encryption_action: Optional[ActionType]=None, encryption_key: Optional[Path]=None) -> None:
+    def push_project(self, project: Path, encryption_action: Optional[ActionType]=None, encryption_key: Optional[Path]=None, force: Optional[bool]=False) -> None:
         """Pushes the given project from the local drive to the cloud.
 
         It will also push every library referenced by the project and add or remove references.
 
         :param project: path to the directory containing the local project that needs to be pushed
         """
         libraries = self._project_manager.get_project_libraries(project)
-        self.push_projects([project], libraries, encryption_action, encryption_key)
+        self.push_projects([project], libraries, encryption_action, encryption_key, force)
 
-    def push_projects(self, projects_to_push: List[Path], associated_libraries_to_push: Optional[List[Path]]=[], encryption_action: Optional[ActionType]=None, encryption_key: Optional[Path]=None) -> None:
+    def push_projects(self, projects_to_push: List[Path], associated_libraries_to_push: Optional[List[Path]]=[], encryption_action: Optional[ActionType]=None, encryption_key: Optional[Path]=None, force: Optional[bool]=False) -> None:
         """Pushes the given projects from the local drive to the cloud.
 
         It will also push every library referenced by each project and add or remove references.
@@ -78,11 +78,13 @@ def push_projects(self, projects_to_push: List[Path], associated_libraries_to_pu
             relative_path = path.relative_to(Path.cwd())
             try:
                 self._logger.info(f"[{index}/{len(all_projects_to_push)}] Pushing '{relative_path}'")
-                self._push_project(path, organization_id, encryption_action_value, encryption_key_value)
+                self._push_project(path, organization_id, encryption_action_value, encryption_key_value, force=force)
             except Exception as ex:
                 from traceback import format_exc
                 self._logger.debug(format_exc().strip())
                 self._logger.warn(f"Cannot push '{relative_path}': {ex}")
+                if "write permission" in str(ex).lower():
+                    self._logger.info("Please pull any required changes and push with --force")
 
     def _get_local_libraries_cloud_ids(self, project_dir: Path) -> List[int]:
         project_config = self._project_config_manager.get_project_config(project_dir)
@@ -95,7 +97,7 @@ def _get_local_libraries_cloud_ids(self, project_dir: Path) -> List[int]:
 
         return local_libraries_cloud_ids
 
-    def _push_project(self, project_path: Path, organization_id: str, encryption_action: Optional[ActionType], encryption_key: Optional[Path], suggested_rename_path: Path = None) -> None:
+    def _push_project(self, project_path: Path, organization_id: str, encryption_action: Optional[ActionType], encryption_key: Optional[Path], force: Optional[bool], suggested_rename_path: Path = None) -> None:
         """Pushes a single local project to the cloud.
 
         Raises an error with a descriptive message if the project cannot be pushed.
@@ -111,7 +113,6 @@ def _push_project(self, project_path: Path, organization_id: str, encryption_act
         if suggested_rename_path and suggested_rename_path != project_path:
             potential_new_name = suggested_rename_path.relative_to(Path.cwd()).as_posix()
 
-
         project_config = self._project_config_manager.get_project_config(project_path)
         cloud_id = project_config.get("cloud-id")
         local_encryption_state = project_config.get("encrypted", False)
@@ -149,7 +150,7 @@ def _push_project(self, project_path: Path, organization_id: str, encryption_act
             if cloud_project.name != project_name:
                 # cloud project name was changed. Repeat steps to validate the new name locally.
                 self._logger.info(f"Received new name '{cloud_project.name}' for project '{project_name}' from QuantConnect.com")
-                self._push_project(project_path, organization_id, encryption_action, encryption_key, Path.cwd() / cloud_project.name)
+                self._push_project(project_path, organization_id, encryption_action, encryption_key, force, Path.cwd() / cloud_project.name)
                 return
 
             self._cloud_projects.append(cloud_project)
@@ -163,7 +164,7 @@ def _push_project(self, project_path: Path, organization_id: str, encryption_act
             encryption_key = local_encryption_key
             encryption_action = ActionType.ENCRYPT if local_encryption_state else ActionType.DECRYPT
         # Finalize pushing by updating locally modified metadata, files and libraries
-        self._push_metadata(project_path, cloud_project, encryption_action, encryption_key)
+        self._push_metadata(project_path, cloud_project, encryption_action, encryption_key, force)
 
     def _get_files(self, project: Path, encryption_action: Optional[ActionType], encryption_key: Optional[Path]) -> List[Dict[str, str]]:
         """Pushes the files of a local project to the cloud.
@@ -193,7 +194,7 @@ def _get_files(self, project: Path, encryption_action: Optional[ActionType], enc
 
         return files
 
-    def _push_metadata(self, project: Path, cloud_project: QCProject, encryption_action: Optional[ActionType], encryption_key: Optional[Path]) -> None:
+    def _push_metadata(self, project: Path, cloud_project: QCProject, encryption_action: Optional[ActionType], encryption_key: Optional[Path], force: Optional[bool]) -> None:
         """Pushes local project description and parameters to the cloud.
 
         Does nothing if the cloud is already up-to-date.
@@ -255,18 +256,22 @@ def _push_metadata(self, project: Path, cloud_project: QCProject, encryption_act
                 encryption_key_id = get_project_key_hash(encryption_key)
                 update_args["encryption_key"] = encryption_key_id
 
+        if not force:
+            update_args["code_source_id"] = "cli"
         if update_args != {}:
             self._api_client.projects.update(cloud_project.projectId, **update_args)
 
             if "encryption_key" in update_args:
                 del update_args["encryption_key"]
+
             updated_keys = list(update_args)
             if len(updated_keys) == 1:
                 updated_keys_str = updated_keys[0]
             elif len(updated_keys) == 2:
                 updated_keys_str = " and ".join(updated_keys)
             else:
                 updated_keys_str = ", ".join(updated_keys[:-1]) + f", and {updated_keys[-1]}"
+
             self._logger.info(f"Successfully updated {updated_keys_str} for '{cloud_project.name}'")
 
     def _get_cloud_project(self, project_id: int, organization_id: str) -> QCProject: