18
18
19
19
API_URL = "https://oeo.test"
20
20
21
- TIFF_CONTENT = b'T1f7D6t6l0l' * 1000
22
-
23
-
21
+ TIFF_CONTENT = b'T1f7D6t6l0l' * 10000
24
22
25
23
@pytest .fixture
26
24
def con100 (requests_mock ):
@@ -74,7 +72,9 @@ def test_execute_batch(con100, requests_mock, tmpdir):
74
72
}
75
73
},
76
74
)
75
+ requests_mock .head (API_URL + "/jobs/f00ba5/files/output.tiff" , headers = {"Content-Length" : str (len ("tiffdata" ))})
77
76
requests_mock .get (API_URL + "/jobs/f00ba5/files/output.tiff" , text = "tiffdata" )
77
+
78
78
requests_mock .get (API_URL + "/jobs/f00ba5/logs" , json = {'logs' : []})
79
79
80
80
path = tmpdir .join ("tmp.tiff" )
@@ -231,6 +231,7 @@ def test_execute_batch_with_soft_errors(con100, requests_mock, tmpdir, error_res
231
231
}
232
232
},
233
233
)
234
+ requests_mock .head (API_URL + "/jobs/f00ba5/files/output.tiff" , headers = {"Content-Length" : str (len ("tiffdata" ))})
234
235
requests_mock .get (API_URL + "/jobs/f00ba5/files/output.tiff" , text = "tiffdata" )
235
236
requests_mock .get (API_URL + "/jobs/f00ba5/logs" , json = {'logs' : []})
236
237
@@ -536,11 +537,57 @@ def job_with_1_asset(con100, requests_mock, tmp_path) -> BatchJob:
536
537
requests_mock .get (API_URL + "/jobs/jj1/results" , json = {"assets" : {
537
538
"1.tiff" : {"href" : API_URL + "/dl/jjr1.tiff" , "type" : "image/tiff; application=geotiff" },
538
539
}})
540
+ requests_mock .head (API_URL + "/dl/jjr1.tiff" , headers = {"Content-Length" : f"{ len (TIFF_CONTENT )} " })
539
541
requests_mock .get (API_URL + "/dl/jjr1.tiff" , content = TIFF_CONTENT )
542
+
543
+ job = BatchJob ("jj1" , connection = con100 )
544
+ return job
545
+
546
+ @pytest .fixture
547
+ def job_with_chunked_asset_using_head (con100 , requests_mock , tmp_path ) -> BatchJob :
548
+ def handle_content (request , context ):
549
+ range = request .headers .get ("Range" )
550
+ assert range
551
+ search = re .search (r"bytes=(\d+)-(\d+)" , range )
552
+ assert search
553
+ from_bytes = int (search .group (1 ))
554
+ to_bytes = int (search .group (2 ))
555
+ assert from_bytes < to_bytes
556
+ return TIFF_CONTENT [from_bytes : to_bytes + 1 ]
557
+
558
+ requests_mock .get (
559
+ API_URL + "/jobs/jj1/results" ,
560
+ json = {
561
+ "assets" : {
562
+ "1.tiff" : {"href" : API_URL + "/dl/jjr1.tiff" , "type" : "image/tiff; application=geotiff" },
563
+ }
564
+ },
565
+ )
566
+ requests_mock .head (
567
+ API_URL + "/dl/jjr1.tiff" , headers = {"Content-Length" : f"{ len (TIFF_CONTENT )} " , "Accept-Ranges" : "bytes" }
568
+ )
569
+ requests_mock .get (API_URL + "/dl/jjr1.tiff" , content = handle_content )
540
570
job = BatchJob ("jj1" , connection = con100 )
541
571
return job
542
572
543
573
574
+ @pytest .fixture
575
+ def job_with_chunked_asset_using_head_old (con100 , requests_mock , tmp_path ) -> BatchJob :
576
+ requests_mock .get (API_URL + "/jobs/jj1/results" , json = {"assets" : {
577
+ "1.tiff" : {"href" : API_URL + "/dl/jjr1.tiff" , "type" : "image/tiff; application=geotiff" },
578
+ }})
579
+ requests_mock .head (API_URL + "/dl/jjr1.tiff" , headers = {"Content-Length" : f"{ len (TIFF_CONTENT )} " , "Accept-Ranges" : "bytes" })
580
+
581
+ chunk_size = 1000
582
+ for r in range (0 , len (TIFF_CONTENT ), chunk_size ):
583
+ from_bytes = r
584
+ to_bytes = min (r + chunk_size , len (TIFF_CONTENT )) - 1
585
+ requests_mock .get (API_URL + "/dl/jjr1.tiff" , request_headers = {"Range" : f"bytes={ from_bytes } -{ to_bytes } " },
586
+ response_list = [{"status_code" : 500 , "text" : "Server error" },
587
+ {"status_code" : 206 , "content" : TIFF_CONTENT [from_bytes :to_bytes + 1 ]}])
588
+ job = BatchJob ("jj1" , connection = con100 )
589
+ return job
590
+
544
591
@pytest .fixture
545
592
def job_with_2_assets (con100 , requests_mock , tmp_path ) -> BatchJob :
546
593
requests_mock .get (API_URL + "/jobs/jj2/results" , json = {
@@ -551,8 +598,11 @@ def job_with_2_assets(con100, requests_mock, tmp_path) -> BatchJob:
551
598
"2.tiff" : {"href" : API_URL + "/dl/jjr2.tiff" , "type" : "image/tiff; application=geotiff" },
552
599
}
553
600
})
601
+ requests_mock .head (API_URL + "/dl/jjr1.tiff" , headers = {"Content-Length" : f"{ len (TIFF_CONTENT )} " })
554
602
requests_mock .get (API_URL + "/dl/jjr1.tiff" , content = TIFF_CONTENT )
603
+ requests_mock .head (API_URL + "/dl/jjr2.tiff" , headers = {"Content-Length" : f"{ len (TIFF_CONTENT )} " })
555
604
requests_mock .get (API_URL + "/dl/jjr2.tiff" , content = TIFF_CONTENT )
605
+
556
606
job = BatchJob ("jj2" , connection = con100 )
557
607
return job
558
608
@@ -574,6 +624,13 @@ def test_get_results_download_file(job_with_1_asset: BatchJob, tmp_path):
574
624
with target .open ("rb" ) as f :
575
625
assert f .read () == TIFF_CONTENT
576
626
627
+ def test_get_results_download_file_ranged (job_with_chunked_asset_using_head : BatchJob , tmp_path ):
628
+ job = job_with_chunked_asset_using_head
629
+ target = tmp_path / "result.tiff"
630
+ res = job .get_results ().download_file (target , range_size = 1000 )
631
+ assert res == target
632
+ with target .open ("rb" ) as f :
633
+ assert f .read () == TIFF_CONTENT
577
634
578
635
def test_download_result_folder (job_with_1_asset : BatchJob , tmp_path ):
579
636
job = job_with_1_asset
@@ -714,6 +771,7 @@ def test_get_results_download_files_include_stac_metadata(
714
771
715
772
def test_result_asset_download_file (con100 , requests_mock , tmp_path ):
716
773
href = API_URL + "/dl/jjr1.tiff"
774
+ requests_mock .head (href , headers = {"Content-Length" : f"{ len (TIFF_CONTENT )} " })
717
775
requests_mock .get (href , content = TIFF_CONTENT )
718
776
719
777
job = BatchJob ("jj" , connection = con100 )
@@ -729,6 +787,7 @@ def test_result_asset_download_file(con100, requests_mock, tmp_path):
729
787
730
788
def test_result_asset_download_file_error (con100 , requests_mock , tmp_path ):
731
789
href = API_URL + "/dl/jjr1.tiff"
790
+ requests_mock .head (href , status_code = 500 , text = "Nope!" )
732
791
requests_mock .get (href , status_code = 500 , text = "Nope!" )
733
792
734
793
job = BatchJob ("jj" , connection = con100 )
@@ -743,6 +802,7 @@ def test_result_asset_download_file_error(con100, requests_mock, tmp_path):
743
802
744
803
def test_result_asset_download_folder (con100 , requests_mock , tmp_path ):
745
804
href = API_URL + "/dl/jjr1.tiff"
805
+ requests_mock .head (href , headers = {"Content-Length" : f"{ len (TIFF_CONTENT )} " })
746
806
requests_mock .get (href , content = TIFF_CONTENT )
747
807
748
808
job = BatchJob ("jj" , connection = con100 )
@@ -770,6 +830,7 @@ def test_result_asset_load_json(con100, requests_mock):
770
830
771
831
def test_result_asset_load_bytes (con100 , requests_mock ):
772
832
href = API_URL + "/dl/jjr1.tiff"
833
+ requests_mock .head (href , headers = {"Content-Length" : f"{ len (TIFF_CONTENT )} " })
773
834
requests_mock .get (href , content = TIFF_CONTENT )
774
835
775
836
job = BatchJob ("jj" , connection = con100 )
@@ -797,6 +858,7 @@ def download_tiff(request, context):
797
858
return TIFF_CONTENT
798
859
799
860
requests_mock .get (API_URL + "/jobs/jj1/results" , json = get_results )
861
+ requests_mock .head ("https://evilcorp.test/dl/jjr1.tiff" , headers = {"Content-Length" : "666" })
800
862
requests_mock .get ("https://evilcorp.test/dl/jjr1.tiff" , content = download_tiff )
801
863
802
864
con100 .authenticate_basic ("john" , "j0hn" )
0 commit comments