Skip to content

Commit 684d410

Browse files
mzouinkmzouink
committed
Daily update 2025-09-29
1 parent 6fb9b2d commit 684d410

File tree

7 files changed

+283
-4
lines changed

7 files changed

+283
-4
lines changed

‎logs/daily_update.log‎

Lines changed: 190 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,190 @@
1+
[2025-09-29 18:20:53] Starting daily update process...
2+
[2025-09-29 18:20:53] Running run.sh...
3+
🔬 Experiment Overview Generator
4+
===============================
5+
📦 Activating conda environment...
6+
📂 Working directory: /groups/cellmap/cellmap/zouinkhim/exp-overview
7+
🔍 Checking directory structure...
8+
📊 Generating experiment overview...
9+
Generating data/processed/auto_generated_overview.csv by scanning experiment directories...
10+
Scanning exp_cell...
11+
Skipping setup_34 - no checkpoints found and not running
12+
Scanning exp_cerebellum...
13+
Scanning exp_pancreas...
14+
Scanning exp_c-elegen...
15+
Skipping 20250725_cell_all_distance_64nm - no checkpoints found and not running
16+
Skipping 20250725_cell_all_lsd_32nm - no checkpoints found and not running
17+
Skipping 20250725_cell_all_lsd_64nm - no checkpoints found and not running
18+
Skipping 20250725_cell_all_mixed_distance_64nm - no checkpoints found and not running
19+
Skipping 20250725_cell_all_mixed_lsd_32nm - no checkpoints found and not running
20+
Skipping 20250725_cell_all_mixed_lsd_64nm - no checkpoints found and not running
21+
Skipping 20250725_ecs_all_lsd_16nm - no checkpoints found and not running
22+
Skipping 20250725_ecs_all_mixed_lsd_16nm - no checkpoints found and not running
23+
Skipping 20250725_er_all_lsd_16nm - no checkpoints found and not running
24+
Skipping 20250725_er_all_mixed_lsd_16nm - no checkpoints found and not running
25+
Skipping 20250725_isg_all_distance_16nm - no checkpoints found and not running
26+
Skipping 20250725_isg_all_distance_32nm - no checkpoints found and not running
27+
Skipping 20250725_isg_all_lsd_16nm - no checkpoints found and not running
28+
Skipping 20250725_isg_all_lsd_32nm - no checkpoints found and not running
29+
Skipping 20250725_isg_all_mixed_lsd_32nm - no checkpoints found and not running
30+
Skipping 20250725_ld_all_lsd_16nm - no checkpoints found and not running
31+
Skipping 20250725_ld_all_lsd_32nm - no checkpoints found and not running
32+
Skipping 20250725_ld_all_mixed_lsd_16nm - no checkpoints found and not running
33+
Skipping 20250725_ld_all_mixed_lsd_32nm - no checkpoints found and not running
34+
Skipping 20250725_lyso_all_lsd_16nm - no checkpoints found and not running
35+
Skipping 20250725_lyso_all_mixed_lsd_16nm - no checkpoints found and not running
36+
Skipping 20250725_mito_all_lsd_16nm - no checkpoints found and not running
37+
Skipping 20250725_mito_all_mixed_lsd_16nm - no checkpoints found and not running
38+
Skipping 20250725_nuc_all_distance_64nm - no checkpoints found and not running
39+
Skipping 20250725_nuc_all_lsd_32nm - no checkpoints found and not running
40+
Skipping 20250725_nuc_all_lsd_64nm - no checkpoints found and not running
41+
Skipping 20250725_nuc_all_mixed_distance_64nm - no checkpoints found and not running
42+
Skipping 20250725_nuc_all_mixed_lsd_32nm - no checkpoints found and not running
43+
Skipping 20250725_nuc_all_mixed_lsd_64nm - no checkpoints found and not running
44+
Skipping 20250806_cell_mouse_distance_32nm - low max iterations (6000) and not running
45+
Skipping 20250806_cell_mouse_distance_64nm - low max iterations (4000) and not running
46+
Skipping 20250806_cell_mouse_lsd_32nm - no checkpoints found and not running
47+
Skipping 20250806_cell_mouse_lsd_64nm - no checkpoints found and not running
48+
Skipping 20250806_ecs_mouse_lsd_16nm - no checkpoints found and not running
49+
Skipping 20250806_er_mouse_distance_16nm - low max iterations (6000) and not running
50+
Skipping 20250806_er_mouse_lsd_16nm - no checkpoints found and not running
51+
Skipping 20250806_isg_mouse_distance_16nm - low max iterations (6000) and not running
52+
Skipping 20250806_isg_mouse_distance_32nm - low max iterations (6000) and not running
53+
Skipping 20250806_isg_mouse_lsd_16nm - low max iterations (4000) and not running
54+
Skipping 20250806_isg_mouse_lsd_32nm - no checkpoints found and not running
55+
Skipping 20250806_ld_mouse_distance_16nm - low max iterations (6000) and not running
56+
Skipping 20250806_ld_mouse_lsd_16nm - low max iterations (6000) and not running
57+
Skipping 20250806_ld_mouse_lsd_32nm - low max iterations (6000) and not running
58+
Skipping 20250806_lyso_mouse_lsd_16nm - no checkpoints found and not running
59+
Skipping 20250806_mito_mouse_lsd_16nm - no checkpoints found and not running
60+
Skipping 20250806_nuc_mouse_lsd_32nm - no checkpoints found and not running
61+
Skipping 20250806_nuc_mouse_lsd_64nm - no checkpoints found and not running
62+
Skipping 20250806_perox_all_distance_16nm - low max iterations (6000) and not running
63+
Skipping 20250806_perox_all_lsd_16nm - no checkpoints found and not running
64+
Skipping 20250806_perox_all_mixed_lsd_16nm - no checkpoints found and not running
65+
Skipping 20250806_yolk_all_distance_16nm - low max iterations (6000) and not running
66+
Skipping 20250806_yolk_all_distance_32nm - low max iterations (2000) and not running
67+
Skipping 20250806_yolk_all_lsd_16nm - no checkpoints found and not running
68+
Skipping 20250806_yolk_all_lsd_32nm - no checkpoints found and not running
69+
Skipping 20250806_yolk_all_mixed_distance_16nm - no checkpoints found and not running
70+
Skipping 20250806_yolk_all_mixed_distance_32nm - no checkpoints found and not running
71+
Skipping 20250806_yolk_all_mixed_lsd_16nm - no checkpoints found and not running
72+
Skipping 20250806_yolk_all_mixed_lsd_32nm - no checkpoints found and not running
73+
Skipping 20250806_yolk_mouse_distance_16nm - no checkpoints found and not running
74+
Skipping 20250806_yolk_mouse_distance_32nm - no checkpoints found and not running
75+
Skipping 20250806_yolk_mouse_lsd_16nm - no checkpoints found and not running
76+
Skipping 20250806_yolk_mouse_lsd_32nm - no checkpoints found and not running
77+
Scanning ex_mito...
78+
Skipping 20250914_all_mitos - appears to be a data config file, not a training experiment
79+
Scanning exp_salivary...
80+
Generated 61 experiment entries in data/processed/auto_generated_overview.csv
81+
82+
Comparing with original overview.csv...
83+
84+
Comparison Results:
85+
Original CSV (data/raw/overview.csv): 41 experiments
86+
Generated CSV (data/processed/auto_generated_overview.csv): 61 experiments
87+
88+
Setup Comparison:
89+
Matched setups: 38
90+
Only in original: 3
91+
Only in generated: 23
92+
93+
Setups only in original CSV:
94+
- [+80 more organelle runs]
95+
- setup_34
96+
- train_fly_model
97+
98+
Setups only in generated CSV:
99+
- 20250725_cell_all_distance_32nm
100+
- 20250725_cell_all_mixed_distance_32nm
101+
- 20250725_ecs_all_distance_16nm
102+
- 20250725_ecs_all_mixed_distance_16nm
103+
- 20250725_er_all_distance_16nm
104+
- 20250725_er_all_mixed_distance_16nm
105+
- 20250725_isg_all_mixed_distance_16nm
106+
- 20250725_isg_all_mixed_distance_32nm
107+
- 20250725_isg_all_mixed_lsd_16nm
108+
- 20250725_ld_all_distance_16nm
109+
- 20250725_ld_all_distance_32nm
110+
- 20250725_ld_all_mixed_distance_16nm
111+
- 20250725_ld_all_mixed_distance_32nm
112+
- 20250725_lyso_all_distance_16nm
113+
- 20250725_lyso_all_mixed_distance_16nm
114+
- 20250725_nuc_all_distance_32nm
115+
- 20250806_ecs_mouse_distance_16nm
116+
- 20250806_ld_mouse_distance_32nm
117+
- 20250806_lyso_mouse_distance_16nm
118+
- 20250806_nuc_mouse_distance_64nm
119+
- 20250806_perox_all_mixed_distance_16nm
120+
- 20250806_perox_mouse_distance_16nm
121+
- 20250806_perox_mouse_lsd_16nm
122+
123+
Detailed comparison of matched setups:
124+
Found differences in 38 matched setups:
125+
20250725_mito_all_distance_16nm:
126+
Starting Checkpoint: 'from scratch' vs 'run08/438000k'
127+
Max Iterations: '80000+' vs '590000'
128+
Resolution (nm): '16' vs 'nan'
129+
20250725_mito_all_mixed_distance_16nm:
130+
Target: 'mito' vs 'er+isg+ld+lyso+mito+nuc'
131+
Starting Checkpoint: 'from scratch' vs 'run08/438000k'
132+
Max Iterations: '80000' vs '568000'
133+
20250725_nuc_all_mixed_distance_32nm:
134+
Target: 'nucleus' vs 'nuc'
135+
Starting Checkpoint: 'from scratch' vs 'run08/438000k'
136+
Max Iterations: '80000+' vs '582000'
137+
20250806_mito_mouse_distance_16nm:
138+
Starting Checkpoint: 'from scratch' vs 'run08/438000k'
139+
Max Iterations: '568000+' vs '362000'
140+
Resolution (nm): '16' vs 'nan'
141+
20250806_nuc_mouse_distance_32nm:
142+
Target: 'nucleus' vs 'nuc'
143+
Starting Checkpoint: 'from scratch' vs 'run08/438000k'
144+
Max Iterations: '342000+' vs '342000'
145+
setup_0:
146+
Starting Checkpoint: '20250725_mito_all_mixed_distance_16nm/568k' vs '20250725_mito_all_mixed_distance_16nm/568000k'
147+
Resolution (nm): '16' vs '16.0'
148+
Learning Rate: '5.0e-06' vs '5e-06'
149+
setup_07:
150+
Starting Checkpoint: '20250806_mito_mouse_distance_16nm/362k' vs '20250806_mito_mouse_distance_16nm/362000k'
151+
Resolution (nm): '16' vs '16.0'
152+
Learning Rate: '5.0e-06' vs '5e-06'
153+
setup_08:
154+
Starting Checkpoint: '20250725_mito_all_mixed_distance_16nm/568k' vs '20250725_mito_all_mixed_distance_16nm/568000k'
155+
Resolution (nm): '16' vs '16.0'
156+
Learning Rate: '5.0e-06' vs '5e-06'
157+
setup_09:
158+
Starting Checkpoint: '20250806_nuc_mouse_distance_32nm/342k' vs '20250806_nuc_mouse_distance_32nm/342000k'
159+
Resolution (nm): '16' vs '32.0'
160+
Learning Rate: '5.0e-06' vs '5e-06'
161+
setup_1:
162+
Starting Checkpoint: '20250725_mito_all_mixed_distance_16nm/80k' vs '20250725_mito_all_mixed_distance_16nm/80000k'
163+
Resolution (nm): '16' vs '16.0'
164+
Learning Rate: '5.0e-06' vs '5e-06'
165+
... and 28 more setups with differences
166+
📈 Generating timeline visualization...
167+
🚀 Generating experiment timeline visualizations...
168+
✅ Main page saved as 'output/visualizations/index.html'
169+
✅ Timeline saved as 'output/visualizations/experiment_timeline.html'
170+
✅ Gantt chart saved as 'output/visualizations/experiment_gantt.html'
171+
✅ Statistics saved as 'output/visualizations/experiment_stats.html'
172+
173+
📈 All visualizations generated successfully!
174+
🌐 Website ready for GitHub Pages deployment!
175+
Files: output/visualizations/index.html, experiment_timeline.html, experiment_gantt.html, experiment_stats.html
176+
✅ Timeline generated: output/visualizations/
177+
178+
🎉 Experiment overview generation complete!
179+
180+
📋 Generated files:
181+
📊 data/processed/auto_generated_overview.csv - Main experiment overview
182+
📝 output/reports/comparison_report.md - Data accuracy report
183+
📈 output/visualizations/ - HTML visualizations
184+
185+
🔗 Quick access:
186+
View data: cat data/processed/auto_generated_overview.csv
187+
View report: cat output/reports/comparison_report.md
188+
Open viz: open output/visualizations/index.html
189+
[2025-09-29 18:20:53] run.sh completed successfully
190+
[2025-09-29 18:20:53] Adding changes to git...

‎output/visualizations/experiment_gantt.html‎

Lines changed: 1 addition & 1 deletion
Large diffs are not rendered by default.

‎output/visualizations/experiment_stats.html‎

Lines changed: 1 addition & 1 deletion
Large diffs are not rendered by default.

‎output/visualizations/experiment_timeline.html‎

Lines changed: 1 addition & 1 deletion
Large diffs are not rendered by default.

‎output/visualizations/index.html‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -402,7 +402,7 @@ <h2>📚 Documentation</h2>
402402
<div class="footer-content">
403403
<div>
404404
<p>&copy; 2025 Howard Hughes Medical Institute | Janelia Research Campus</p>
405-
<p style="font-size: 0.9rem; opacity: 0.8;">Last updated: 2025-09-29 18:17 UTC</p>
405+
<p style="font-size: 0.9rem; opacity: 0.8;">Last updated: 2025-09-29 18:21 UTC</p>
406406
</div>
407407
<div class="footer-links">
408408
<a href="https://www.janelia.org/project-team/cellmap">CellMap Project</a>

‎scripts/daily_update.sh‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
#!/bin/bash
2+
3+
# Daily experiment overview update script
4+
# This script runs run.sh, commits changes, and pushes to repository
5+
6+
# Get current date for commit message
7+
DATE=$(date '+%Y-%m-%d')
8+
TIMESTAMP=$(date '+%Y-%m-%d %H:%M:%S')
9+
10+
# Change to project directory
11+
cd /groups/cellmap/cellmap/zouinkhim/exp-overview
12+
13+
# Log the start of the process
14+
echo "[$TIMESTAMP] Starting daily update process..." >> logs/daily_update.log
15+
16+
# Run the main update script
17+
echo "[$TIMESTAMP] Running run.sh..." >> logs/daily_update.log
18+
./run.sh >> logs/daily_update.log 2>&1
19+
20+
# Check if run.sh executed successfully
21+
if [ $? -eq 0 ]; then
22+
echo "[$TIMESTAMP] run.sh completed successfully" >> logs/daily_update.log
23+
else
24+
echo "[$TIMESTAMP] ERROR: run.sh failed with exit code $?" >> logs/daily_update.log
25+
exit 1
26+
fi
27+
28+
# Add all changes to git
29+
echo "[$TIMESTAMP] Adding changes to git..." >> logs/daily_update.log
30+
git add .
31+
32+
# Check if there are any changes to commit
33+
if git diff --staged --quiet; then
34+
echo "[$TIMESTAMP] No changes to commit" >> logs/daily_update.log
35+
else
36+
# Commit with date as message
37+
echo "[$TIMESTAMP] Committing changes..." >> logs/daily_update.log
38+
git commit -m "Daily update $DATE"
39+
40+
# Push to repository
41+
echo "[$TIMESTAMP] Pushing to repository..." >> logs/daily_update.log
42+
git push >> logs/daily_update.log 2>&1
43+
44+
if [ $? -eq 0 ]; then
45+
echo "[$TIMESTAMP] Successfully pushed changes to repository" >> logs/daily_update.log
46+
else
47+
echo "[$TIMESTAMP] ERROR: Failed to push to repository" >> logs/daily_update.log
48+
exit 1
49+
fi
50+
fi
51+
52+
echo "[$TIMESTAMP] Daily update process completed successfully" >> logs/daily_update.log

‎scripts/setup_cron.sh‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
#!/bin/bash
2+
3+
# Script to set up daily cron job for experiment overview updates
4+
# This script adds a cron job that runs daily at 2:00 AM
5+
6+
SCRIPT_PATH="/groups/cellmap/cellmap/zouinkhim/exp-overview/scripts/daily_update.sh"
7+
CRON_TIME="0 2 * * *" # Run at 2:00 AM every day
8+
9+
# Check if cron job already exists
10+
if crontab -l 2>/dev/null | grep -q "$SCRIPT_PATH"; then
11+
echo "Cron job already exists for daily_update.sh"
12+
echo "Current cron jobs:"
13+
crontab -l | grep "$SCRIPT_PATH"
14+
else
15+
# Add the cron job
16+
echo "Adding daily cron job..."
17+
(crontab -l 2>/dev/null; echo "$CRON_TIME $SCRIPT_PATH") | crontab -
18+
19+
if [ $? -eq 0 ]; then
20+
echo "Successfully added cron job:"
21+
echo "$CRON_TIME $SCRIPT_PATH"
22+
echo ""
23+
echo "This will run daily at 2:00 AM and:"
24+
echo "1. Execute run.sh"
25+
echo "2. Commit all changes with date as message"
26+
echo "3. Push to repository"
27+
echo "4. Log all activities to logs/daily_update.log"
28+
else
29+
echo "Failed to add cron job"
30+
exit 1
31+
fi
32+
fi
33+
34+
echo ""
35+
echo "To view all cron jobs: crontab -l"
36+
echo "To remove this cron job: crontab -l | grep -v '$SCRIPT_PATH' | crontab -"
37+
echo "To check logs: tail -f /groups/cellmap/cellmap/zouinkhim/exp-overview/logs/daily_update.log"

0 commit comments

Comments
 (0)