Skip to content

Commit 131b8a2

Browse files
committed
Add API endpoint for background job progress
1 parent 42c1c6b commit 131b8a2

File tree

4 files changed

+85
-2
lines changed

4 files changed

+85
-2
lines changed

backend/btrixcloud/background_jobs.py

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
StorageRef,
2929
User,
3030
SuccessResponse,
31+
JobProgress,
3132
)
3233
from .pagination import DEFAULT_PAGE_SIZE, paginated_format
3334
from .utils import dt_now
@@ -584,6 +585,52 @@ def _get_job_by_type_from_data(self, data: dict[str, object]):
584585

585586
return DeleteOrgJob.from_dict(data)
586587

588+
async def get_job_progress(self, job_id: str) -> JobProgress:
589+
"""Return progress of background job for supported types"""
590+
job = await self.get_background_job(job_id)
591+
592+
if job.type != BgJobType.COPY_BUCKET:
593+
raise HTTPException(status_code=403, detail="job_type_not_supported")
594+
595+
if job.success is False:
596+
raise HTTPException(status_code=400, detail="job_failed")
597+
598+
if job.finished:
599+
return JobProgress(percentage=1.0)
600+
601+
log_tail = await self.crawl_manager.tail_background_job(job_id)
602+
if not log_tail:
603+
raise HTTPException(status_code=400, detail="job_log_not_available")
604+
605+
lines = log_tail.splitlines()
606+
reversed_lines = list(reversed(lines))
607+
608+
progress = JobProgress(percentage=0.0)
609+
610+
# Parse lines in reverse order until we find one with latest stats
611+
for line in reversed_lines:
612+
try:
613+
if "ETA" not in line:
614+
continue
615+
616+
stats_groups = line.split(",")
617+
for group in stats_groups:
618+
group = group.strip()
619+
if "%" in group:
620+
progress.percentage = float(group.strip("%")) / 100
621+
if "ETA" in group:
622+
eta_str = group.strip("ETA ")
623+
# Split on white space to remove byte mark rclone sometimes
624+
# adds to end of stats line
625+
eta_list = eta_str.split(" ")
626+
progress.eta = eta_list[0]
627+
628+
break
629+
except:
630+
continue
631+
632+
return progress
633+
587634
async def list_background_jobs(
588635
self,
589636
org: Organization,
@@ -806,6 +853,17 @@ async def get_background_job(
806853
"""Retrieve information for background job"""
807854
return await ops.get_background_job(job_id, org.id)
808855

856+
@router.get(
857+
"/{job_id}/progress",
858+
response_model=JobProgress,
859+
)
860+
async def get_job_progress(
861+
job_id: str,
862+
org: Organization = Depends(org_crawl_dep),
863+
):
864+
"""Return progress information for background job"""
865+
return await ops.get_job_progress(job_id)
866+
809867
@app.get("/orgs/all/jobs/{job_id}", response_model=AnyJob, tags=["jobs"])
810868
async def get_background_job_all_orgs(job_id: str, user: User = Depends(user_dep)):
811869
"""Get background job from any org"""

backend/btrixcloud/crawlmanager.py

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -389,6 +389,22 @@ async def delete_crawl_config_by_id(self, cid: str) -> None:
389389
"""Delete all crawl configs by id"""
390390
await self._delete_crawl_configs(f"btrix.crawlconfig={cid}")
391391

392+
async def tail_background_job(self, job_id: str) -> str:
393+
"""Tail running background job pod"""
394+
pods = await self.core_api.list_namespaced_pod(
395+
namespace=self.namespace,
396+
label_selector=f"batch.kubernetes.io/job-name={job_id}",
397+
)
398+
399+
if not pods.items:
400+
return ""
401+
402+
pod_name = pods.items[0].metadata.name
403+
404+
return await self.core_api.read_namespaced_pod_log(
405+
pod_name, self.namespace, tail_lines=10
406+
)
407+
392408
# ========================================================================
393409
# Internal Methods
394410
async def _delete_crawl_configs(self, label) -> None:

backend/btrixcloud/models.py

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2454,6 +2454,14 @@ class CopyBucketJob(BackgroundJob):
24542454
]
24552455

24562456

2457+
# ============================================================================
2458+
class JobProgress(BaseModel):
2459+
"""Model for reporting background job progress"""
2460+
2461+
percentage: float
2462+
eta: Optional[str] = None
2463+
2464+
24572465
# ============================================================================
24582466

24592467
### PAGES ###

chart/app-templates/copy_job.yaml

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,13 @@ kind: Job
33
metadata:
44
name: "{{ id }}"
55
labels:
6+
job-id: "{{ id }}"
67
role: "background-job"
78
job_type: {{ job_type }}
89
btrix.org: {{ oid }}
910

1011
spec:
11-
ttlSecondsAfterFinished: 60
12+
ttlSecondsAfterFinished: 30
1213
backoffLimit: 3
1314
template:
1415
spec:
@@ -86,7 +87,7 @@ spec:
8687
- name: RCLONE_CONFIG_NEW_ENDPOINT
8788
value: "{{ new_endpoint }}"
8889

89-
command: ["rclone", "-vv", "--progress", "copy", "--checksum", "--use-mmap", "--transfers=2", "--checkers=2", "prev:{{ prev_bucket }}{{ oid }}", "new:{{ new_bucket }}{{ oid }}"]
90+
command: ["rclone", "-v", "--stats-one-line", "--stats", "10s", "copy", "--checksum", "--use-mmap", "--transfers=2", "--checkers=2", "prev:{{ prev_bucket }}{{ oid }}", "new:{{ new_bucket }}{{ oid }}"]
9091
resources:
9192
limits:
9293
memory: "350Mi"

0 commit comments

Comments
 (0)