Shift to the new engine

Author: iwatkot

.flake8

@@ -0,0 +1,3 @@
+[flake8]
+max-line-length = 100
+ignore = E203, E501, W503, E722, W605

.gitignore

@@ -4,3 +4,5 @@ venv
 secret_debug.env
 supervisely_lib
 src/debug
+__pycache__/
+.DS_Store

local.env

@@ -1,20 +1,11 @@
-PYTHONUNBUFFERED=1
-
-LOG_LEVEL="debug"
-
-TASK_ID=22377
-
-CONTEXT_TEAMID=8
-CONTEXT_WORKSPACEID=349
-CONTEXT_PROJECTID=14505
+TEAM_ID=448
+WORKSPACE_ID=690
+PROJECT_ID=37478
 
 modal.state.Options=all # all, annotated
 modal.state.allDatasets=true
-#modal.state.selectedDatasets= ["MOT15"] # ["ds1", "ds2"]
+modal.state.selectedDatasets= ["ds0"] # ["ds1", "ds2"]
 modal.state.batchSize=30
 
 modal.state.sampleResultFrames=true
-modal.state.framesStep=2
-
-DEBUG_APP_DIR="src/debug/app_debug_data"
-DEBUG_CACHE_DIR="src/debug/app_debug_cache"
+modal.state.framesStep=2

src/functions.py

@@ -18,7 +18,7 @@ def upload_frames(api: sly.Api, dataset_id, names, images, anns, metas, current_
         )
         new_image_ids = [img_info.id for img_info in new_image_infos]
         api.annotation.upload_anns(new_image_ids, anns)
-        g.logger.info(f"batch uploaded in {time() - local_time} seconds")
+        sly.logger.info(f"batch uploaded in {time() - local_time} seconds")
 
 
 def convert_tags(tags, prop_container, frame_container, frame_indices=None):
@@ -49,17 +49,25 @@ def download_frames_with_retry(api: sly.Api, video_id, frames_to_convert):
         try:
             images = api.video.frame.download_nps(video_id, frames_to_convert)
             if len(images) != len(frames_to_convert):
-                raise RuntimeError(f"Downloaded {len(images)} frames, but {len(frames_to_convert)} expected.")
+                raise RuntimeError(
+                    f"Downloaded {len(images)} frames, but {len(frames_to_convert)} expected."
+                )
             return images
         except Exception as e:
             curr_retry += 1
             if curr_retry <= retry_cnt:
-                sleep(2 ** curr_retry)
-                sly.logger.warn(f"Failed to download frames, retry {curr_retry} of {retry_cnt}... Error: {e}")
-    raise RuntimeError(f"Failed to download frames with ids {frames_to_convert}. Check your data and try again later.")
+                sleep(2**curr_retry)
+                sly.logger.warn(
+                    f"Failed to download frames, retry {curr_retry} of {retry_cnt}... Error: {e}"
+                )
+    raise RuntimeError(
+        f"Failed to download frames with ids {frames_to_convert}. Check your data and try again later."
+    )
 
 
-def get_frames_from_api(api: sly.Api, video_id, video_name, frames_to_convert, dataset_name):
+def get_frames_from_api(
+    api: sly.Api, video_id, video_name, frames_to_convert, dataset_name
+):
     if g.project.custom_data.get("original_images") is not None:
         image_names = []
         for frame in frames_to_convert:
@@ -79,7 +87,7 @@ def get_frames_from_api(api: sly.Api, video_id, video_name, frames_to_convert, d
 def get_progress_cb(message, total, is_size=False):
     progress = sly.Progress(message, total, is_size=is_size)
     progress_cb = partial(
-        update_progress, api=g.api, task_id=g.my_app.task_id, progress=progress
+        update_progress, api=g.api, task_id=g.task_id, progress=progress
     )
     progress_cb(0)
     return progress_cb

src/globals.py

@@ -10,42 +10,60 @@
     load_dotenv("local.env")
     load_dotenv(os.path.expanduser("~/supervisely.env"))
 
-logger = sly.logger
-
-my_app = sly.AppService()
-api: sly.Api = my_app.public_api
+# region envvars
+team_id = sly.env.team_id()
+workspace_id = sly.env.workspace_id()
+project_id = sly.env.project_id()
+task_id = sly.env.task_id(raise_not_found=False)
+if sly.is_development():
+    sly.logger.warning("Development mode, will set task_id to 0 to avoid errors")
+    # task_id = 0
 
-TEAM_ID = sly.env.team_id()
-WORKSPACE_ID = sly.env.workspace_id()
-PROJECT_ID = sly.env.project_id()
+# endregion
+sly.logger.info(
+    f"Api initialized. Team: {team_id}. Workspace: {workspace_id}. Project: {project_id}"
+)
 
-SAMPLE_RESULT_FRAMES = bool(strtobool(os.getenv("modal.state.sampleResultFrames")))
-if SAMPLE_RESULT_FRAMES:
-    FRAMES_STEP = int(os.environ["modal.state.framesStep"])
+api = sly.Api.from_env()
 
-LOG_LEVEL = str(os.environ["LOG_LEVEL"])
+# region modalvars
+sample_result_frames = bool(strtobool(os.getenv("modal.state.sampleResultFrames")))
+if sample_result_frames:
+    frames_step = int(os.environ["modal.state.framesStep"])
+else:
+    frames_step = None
 
-OPTIONS = os.environ["modal.state.Options"]
-BATCH_SIZE = int(os.environ["modal.state.batchSize"])
+options = os.environ["modal.state.Options"]
+batch_size = int(os.environ["modal.state.batchSize"])
 
-SELECTED_DATASETS = json.loads(
+selected_datasets = json.loads(
     os.environ["modal.state.selectedDatasets"].replace("'", '"')
 )
 
-ALL_DATASETS = os.getenv("modal.state.allDatasets").lower() in ("true", "1", "t")
-if ALL_DATASETS or len(SELECTED_DATASETS) == 0:
-    SELECTED_DATASETS = [dataset.name for dataset in api.dataset.get_list(PROJECT_ID)]
+all_datasets = os.getenv("modal.state.allDatasets").lower() in ("true", "1", "t")
+if all_datasets or len(selected_datasets) == 0:
+    selected_datasets = [dataset.name for dataset in api.dataset.get_list(project_id)]
+# endregion
+sly.logger.info(
+    f"Sample result frames: {sample_result_frames}. Frames step: {frames_step}. "
+    f"Options: {options}. Batch size: {batch_size}. Selected datasets: {selected_datasets}. "
+    f"All datasets: {all_datasets}"
+)
 
 need_download_threshold = 0.15
 
-storage_dir = os.path.join(my_app.data_dir, "sly_base_sir")
+storage_dir = os.path.join(os.getcwd(), "storage")
 mkdir(storage_dir, True)
 video_dir = os.path.join(storage_dir, "video")
 mkdir(video_dir)
 img_dir = os.path.join(storage_dir, "images")
 mkdir(img_dir)
+sly.logger.debug(
+    f"Storage directory: {storage_dir}, video directory: {video_dir}, images directory: {img_dir}"
+)
 
-project = api.project.get_info_by_id(PROJECT_ID)
+project = api.project.get_info_by_id(project_id)
+sly.logger.info(f"Working with project {project.name}...")
 if project is None:
     raise RuntimeError("Project {!r} not found".format(project.name))
 if project.type != str(sly.ProjectType.VIDEOS):
@@ -57,6 +75,7 @@
 
 meta_json = api.project.get_meta(project.id)
 meta = sly.ProjectMeta.from_json(meta_json)
+sly.logger.info("Project meta received...")
 
 if "object_id" not in [tag.name for tag in meta.tag_metas]:
     vobj_id_tag_meta = sly.TagMeta(
@@ -66,7 +85,7 @@
     )
     meta = meta.add_tag_meta(vobj_id_tag_meta)
 
-if OPTIONS == "annotated" and len(meta.obj_classes) == 0 and len(meta.tag_metas) == 0:
+if options == "annotated" and len(meta.obj_classes) == 0 and len(meta.tag_metas) == 0:
     raise ValueError(
         "Nothing to convert, there are no tags and classes in project {!r}".format(
             project.name

src/main.py

@@ -5,24 +5,23 @@
 from supervisely.video_annotation.key_id_map import KeyIdMap
 
 import functions as f
+
 import globals as g
 
 
-@g.my_app.callback("turn_into_images_project")
-@sly.timeit
-def turn_into_images_project(api: sly.Api, task_id, context, state, app_logger):
+def turn_into_images_project(api: sly.Api):
     res_project_name = f"{g.project.name}(images)"
     dst_project = api.project.create(
-        g.WORKSPACE_ID,
+        g.workspace_id,
         res_project_name,
         type=sly.ProjectType.IMAGES,
         change_name_if_conflict=True,
     )
     api.project.update_meta(dst_project.id, g.meta.to_json())
 
     key_id_map = KeyIdMap()
-    for dataset_name in g.SELECTED_DATASETS:
-        dataset = api.dataset.get_info_by_name(g.PROJECT_ID, dataset_name)
+    for dataset_name in g.selected_datasets:
+        dataset = api.dataset.get_info_by_name(g.project_id, dataset_name)
         dst_dataset = api.dataset.create(dst_project.id, dataset.name)
         videos = api.video.get_list(dataset.id)
         for batch in sly.batched(videos):
@@ -31,11 +30,11 @@ def turn_into_images_project(api: sly.Api, task_id, context, state, app_logger):
                 ann_info = api.video.annotation.download(video_info.id)
                 ann = sly.VideoAnnotation.from_json(ann_info, g.meta, key_id_map)
                 if (
-                    g.OPTIONS == "annotated"
+                    g.options == "annotated"
                     and len(ann.tags) == 0
                     and len(ann.frames) == 0
                 ):
-                    g.my_app.logger.warn(
+                    sly.logger.warn(
                         f"Video {video_info.name} annotation is empty in Dataset {dataset_name}"
                     )
                     continue
@@ -57,49 +56,40 @@ def turn_into_images_project(api: sly.Api, task_id, context, state, app_logger):
                     )
                     vobject_id = key_id_map.get_object_id(vobject.key())
                     f.add_object_id_tag(vobject_id, object_props[vobject.key()])
-                if g.OPTIONS == "annotated":
+                if g.options == "annotated":
                     frames_to_convert.extend(list(ann.frames.keys()))
                     frames_to_convert = list(dict.fromkeys(frames_to_convert))
                     frames_to_convert.sort()
                 else:
                     frames_to_convert = list(range(0, video_info.frames_count))
 
-                if g.SAMPLE_RESULT_FRAMES:
-                    if g.OPTIONS == "all":
-                        frames_to_convert = frames_to_convert[:: g.FRAMES_STEP]
+                if g.sample_result_frames:
+                    if g.options == "all":
+                        frames_to_convert = frames_to_convert[:: g.frames_step]
                     else:
                         frames_to_convert = f.calc_frame_step(
                             frames_to_convert=frames_to_convert,
-                            frame_step=g.FRAMES_STEP,
+                            frame_step=g.frames_step,
                         )
 
                 progress = sly.Progress(
                     "Processing video frames: {!r}".format(video_info.name),
                     len(frames_to_convert),
                 )
 
-                # total_images_size = 0
                 for batch_index, batch_frames in enumerate(
-                    sly.batched(frames_to_convert, batch_size=g.BATCH_SIZE)
+                    sly.batched(frames_to_convert, batch_size=g.batch_size)
                 ):
                     metas = []
                     anns = []
                     local_time = time()
                     images_names, images = f.get_frames_from_api(
                         api, video_info.id, video_info.name, batch_frames, dataset_name
                     )
-                    g.logger.debug(
+                    sly.logger.debug(
                         f"extracted {len(batch_frames)} by {time() - local_time} seconds"
                     )
 
-                    """
-                    too slow calculations, for extreme debug
-                    images_size = f.calculate_batch_size(images) / (1024 * 1024)  # in MegaBytes
-                    g.logger.debug(f'batch size: {images_size} MB')
-                    g.logger.debug(f'mean item size: {images_size / len(images)} MB')
-                    total_images_size += images_size
-                    """
-
                     for idx, frame_index in enumerate(batch_frames):
                         metas.append(
                             {
@@ -149,57 +139,21 @@ def turn_into_images_project(api: sly.Api, task_id, context, state, app_logger):
                             )
                         )
 
-                    if g.LOG_LEVEL == "debug":
-                        f.distort_frames(images)
-                        g.logger.debug(f"{len(images)} frames distorted")
-
                     f.upload_frames(
                         api,
                         dst_dataset.id,
                         images_names,
                         images,
                         anns,
                         metas,
-                        f"{batch_index}/{int(len(frames_to_convert) / g.BATCH_SIZE)}",
+                        f"{batch_index}/{int(len(frames_to_convert) / g.batch_size)}",
                     )
                     progress.iters_done_report(len(images_names))
 
-                    if g.my_app.is_stopped():
-                        g.logger.debug("turn_into_images_project is finished.")
-                        return
-
-                # g.logger.debug(f'total images size for video: {total_images_size} MB')
-                g.logger.info(
+                sly.logger.info(
                     f"video {video_info.name} converted in {time() - general_time} seconds"
                 )
 
-    g.my_app.stop()
-
-
-@g.my_app.callback("stop")
-@sly.timeit
-def stop(api: sly.Api, task_id, context, state, app_logger):
-    g.my_app.stop()
-
-
-def main():
-    if g.LOG_LEVEL == "debug":
-        g.logger.debug(
-            f"DEBUG MODE ACTIVATED, we will add 50 random pixels to distort images"
-        )
-
-    sly.logger.info(
-        "Script arguments",
-        extra={
-            "TEAM_ID": g.TEAM_ID,
-            "WORKSPACE_ID": g.WORKSPACE_ID,
-            "PROJECT_ID": g.PROJECT_ID,
-        },
-    )
-
-    # Run application service
-    g.my_app.run(initial_events=[{"command": "turn_into_images_project"}])
-
 
 if __name__ == "__main__":
-    sly.main_wrapper("main", main)
+    turn_into_images_project(g.api)