server update

correspondence.csv

@@ -0,0 +1,13 @@
+JOB ID,Threshold
+123,0.2
+SJFSJFSFKSFKKFKFS,0.2
+"r3J$q""%_@/\o9Us!,qNs7x()|mF0H7",0.5
+rEz7t/ezq2L*\0!AF@a.c.<4/(Oj/p,0.5
+vi>/?t%tC;xSuWI%:)\hK=(k9$\@4r,0.5
+,
+"6rXqZAzP;~CKe,ZL{-CI|(eAdeuIi{",0.5
+F_};];4'vHo0[:DFRo.E.@TH0>|m@Y,0.5
+"hGKYpTjD3i~}3m8_PgF!5X/D""eFA>Q",0.4
+"A9S=q^f{DH+6=d>R""]_!04lK&0{~_L",0.8
+kG\}m;nNR:S43[cHMB0mK|~lc}G7<2,0.9
+ZQ5I*MB&%`2977MyMUO$uV7c{#~@WO,0.8

mopp/_defaults.py

@@ -42,6 +42,7 @@
 DESC_OUTPUT = "Output: Directory name to generate or overwite. [required]"
 DESC_INDEX = "Genome index. [required]"
 DESC_NTHREADS = "Number of threads. [default: 4]."
+DESC_EMAIL = "Email to send calculated coverages to. Will receieve threshold from this address."
 # module: align
 DESC_PATTERN = "File patterns to include for alignment"
 # module: features

mopp/modules/server.py

@@ -0,0 +1,88 @@
+import requests
+import html
+import logging
+import time
+import secrets
+import string
+import csv
+import os
+from pathlib import Path
+
+def generate_random_string(length=30):
+    alphabet = string.ascii_letters + string.digits + string.punctuation
+    random_string = ''.join(secrets.choice(alphabet) for _ in range(length))
+    return random_string
+
+
+def get_url_safe(url, max_retries=5, timeout=5):
+    for attempt in range(max_retries):
+        try:
+            response = requests.get(html.unescape(url), timeout=timeout)
+            if response.status_code == 200:
+                return response
+
+        except requests.RequestException as e:
+            logging.error(f"GET request failed: {e}")
+    return None
+
+def send_data(url, file_path, email, extra_data=None, max_retries=5, retry_delay=2):
+    for attempt in range(0, max_retries):
+        try:
+            with open(file_path, "rb") as file:
+                data = extra_data or {}
+
+                tsv_content = file.read().decode("utf-8")
+                data["tsv"] = tsv_content
+                data["email"] = email
+                data["token"] = os.environ.get("TOKEN")
+
+                response = requests.post(url, data=data)
+                response.raise_for_status() 
+                return response
+
+        except requests.RequestException as e:
+            print(f"Error uploading file (attempt {attempt}/{max_retries}): {e}")
+            if attempt < max_retries:
+                print(f"Retrying in {retry_delay} seconds...")
+                time.sleep(retry_delay)
+            else:
+                print("Max retries reached. Upload failed.")
+                return None
+
+def yield_for_response():
+
+    while True:
+
+        r = requests.get('https://docs.google.com/spreadsheet/ccc?key=1JUhXhmVunPtvMyAU39EDZFKrgTZXnxZAC1M-CHb49io&output=csv')
+        open('correspondence.csv', 'wb').write(r.content)
+
+        tokens = []
+        thresholds = []
+
+        with open('correspondence.csv', 'r') as csv_file:
+            csv_reader = csv.reader(csv_file)
+            for row in csv_reader:
+                tokens.append(row[0])
+                thresholds.append(row[1])
+            try:
+                idx = tokens.index(os.environ.get("TOKEN"))
+                print("VALUE FOUND..........", thresholds[idx])
+                return thresholds[idx]
+            except ValueError:
+                print(f"Checking for response...")
+        time.sleep(3)
+
+
+
+def request_cutoff(email, cov_outdir):
+    path_outdir_cov = Path(cov_outdir) / "coverages.tsv"
+    os.environ["TOKEN"] = generate_random_string()
+    send_data("https://script.google.com/macros/s/AKfycbxVLgVZQnCYdzgNI7QUcwd2JqL56rRETPbtZhmfOpFoN55lr6zcojCXdrSRF_RU8nWP/exec",
+                     path_outdir_cov,
+                     email)
+
+    threshold = yield_for_response()
+
+    return threshold
+
+

mopp/mopp.py

@@ -1,6 +1,7 @@
 import time
 import click
 import logging
+import concurrent.futures
 from pathlib import Path
 from mopp._defaults import (
     MSG_WELCOME,
@@ -21,19 +22,24 @@
     DESC_PREFIX,
     DESC_REFDB,
     DESC_INPUT_COV,
+    DESC_EMAIL
 )
 from mopp.modules.trim import trim_files
 from mopp.modules.align import align_files
 from mopp.modules.coverages import calculate_genome_coverages
 from mopp.modules.index import genome_extraction
 from mopp.modules.features import ft_generation
 from mopp.modules.utils import create_folder_without_clear
+from mopp.modules.server import request_cutoff
 
 
 logger = logging.getLogger("mopp")
 timestamp = time.strftime("%Y-%m-%d_%H-%M-%S")
 formatter = logging.Formatter("%(asctime)s:%(name)s:%(levelname)s:%(message)s")
 
+def get_user_input():
+    return click.prompt("Please enter your desired coverage threshold here")
+
 
 @click.group(help=MSG_WELCOME)
 def mopp():
@@ -47,7 +53,8 @@ def mopp():
 @click.option("-x", "--index", required=True, help=DESC_INDEX)  # wol bt2 index
 @click.option("-t", "--threads", default=4, help=DESC_NTHREADS)
 @click.option("-z", "--zebra", required=True, help=DESC_ZEBRA)
-@click.option("-c", "--cutoff", type=float, required=True, help=DESC_CUTOFF)
+@click.option("-c", "--cutoff", type=float, help=DESC_CUTOFF)
+@click.option("-e", "--email", help=DESC_EMAIL)
 @click.option("-ref", "--refdb", required=True, help=DESC_REFDB)  # index wol.fna
 @click.option("-p", "--prefix", required=True, help=DESC_PREFIX)  # index prefix
 @click.option(
@@ -71,6 +78,7 @@ def workflow(
     threads,
     zebra,
     cutoff,
+    email,
     refdb,
     prefix,
     rank,
@@ -104,7 +112,52 @@ def workflow(
             outdir_trimmed, outdir_aligned_metaG, "*metaG*.fq.gz", index, threads
         )
         calculate_genome_coverages(zebra, outdir_aligned_metaG_samfiles, outdir_cov)
-        genome_extraction(outdir_cov_file, cutoff, refdb, outdir_index, prefix, threads)
+
+        if (cutoff == None):
+            if (email == None):
+                cutoff = get_user_input()
+            else:
+                cutoff = request_cutoff(email, outdir_cov)
+                '''while True:
+                    OK_VALUE = False
+                    with concurrent.futures.ThreadPoolExecutor() as executor:
+                    # Submit both functions for concurrent execution
+                        future1 = executor.submit(get_user_input)
+                        future2 = executor.submit(request_cutoff, email, outdir_cov) 
+
+                        completed, not_done = concurrent.futures.wait([future1, future2], return_when=concurrent.futures.FIRST_COMPLETED)
+
+                        for future in completed:
+                            if future == future1:
+                                print("FIRST")
+                                cutoff = future1.result()
+                                try:
+                                    float_value = float(cutoff)
+                                    if 0 <= float_value <= 1:
+                                        OK_VALUE = True
+                                        executor.shutdown()
+                                except ValueError:
+                                    OK_VALUE = False
+                                    logger.error("Coverage cutoff is not a float or not betwseen 0 and 1. Please re-enter.")
+                            elif future == future2:
+                                print("SECOND", future2.result())
+                                cutoff = future2.result()
+                                OK_VALUE = True
+                                executor.shutdown()
+                                # Sanity check is performed server-side.
+
+                        if OK_VALUE:    
+                            for future in not_done:
+                                print("CANCELLING")
+                                future.cancel()
+         '''           
+
+
+        print("THE CUTOFF WAS RETRIEVED. IT IS: ", cutoff)
+
+
+
+        genome_extraction(outdir_cov_file, float(cutoff), refdb, outdir_index, prefix, threads)
         align_files(
             outdir_trimmed, outdir_aligned, "*.fq.gz", str(outdir_index_path), threads
         )

test/data/out4/aligned_metaG/bowfiles/1-1_t2_metaG_trimmed_wol_subset0.1_index.bow

@@ -0,0 +1,6 @@
+249399 reads; of these:
+  249399 (100.00%) were unpaired; of these:
+    231287 (92.74%) aligned 0 times
+    18022 (7.23%) aligned exactly 1 time
+    90 (0.04%) aligned >1 times
+7.26% overall alignment rate