shub_cloudless_LANDSAT.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "81f14008-0c10-41c7-bdc6-998f77a3aa77",
   "metadata": {},
   "source": [
    "# Cloudless images from Sentinel HUB\n",
    "\n",
    "Find cloudless images for a selected AOI in a selected time frame.\n",
    "\n",
    "The conda-forge version of SentinelHub package is not the latest version (problem with access token for Landsat service).\n",
    "\n",
    "Install the latest version (3.4) using pip:\n",
    "\n",
    "`$ pip install sentinelhub --upgrade` or `$ pip install sentinelhub --upgrade --user`"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f51892aa-1304-48b1-8e8b-b635969f9405",
   "metadata": {},
   "source": [
    ">**WARNING**: If setting the credentials manually be careful NOT TO PUBLICALLY SHARE the repository or this script!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "90e3a9f2-a47b-48f5-ae3e-05023c89dc2d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Instance ID: CEOIS2_b\n"
     ]
    }
   ],
   "source": [
    "from sentinelhub import SHConfig\n",
    "import glob\n",
    "\n",
    "config = SHConfig()\n",
    "\n",
    "# Read from file if it exist, otherwise set manually\n",
    "user_creds = glob.glob(\".\\\\userdata\\\\credentials*.txt\")\n",
    "if user_creds:\n",
    "    with open(user_creds[0], \"r\") as txt:\n",
    "        instance_id, shub_id, shub_secret = txt.readline().split(\" \")\n",
    "        config.instance_id = instance_id\n",
    "        config.sh_client_id = shub_id\n",
    "        config.sh_client_secret = shub_secret\n",
    "else:\n",
    "    config.instance_id = '<your instance id>'\n",
    "    config.sh_client_id = '<your client id>'\n",
    "    config.sh_client_secret = '<your client secret>'\n",
    "\n",
    "config.sh_base_url = 'https://services-uswest2.sentinel-hub.com'\n",
    "\n",
    "print(\"Instance ID: \" + config.instance_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "97a673be-4e53-420a-b7c2-8e76dcd0c3c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "9697c296-f82b-4333-ab72-83d153306986",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import os\n",
    "# import datetime\n",
    "# import numpy as np\n",
    "# import matplotlib.pyplot as plt\n",
    "# from utils import plot_image\n",
    "# import osmnx as ox\n",
    "# import glob\n",
    "# import rasterio\n",
    "# import rasterio.plot\n",
    "\n",
    "from sentinelhub import CRS, BBox, bbox_to_dimensions, DataCollection, \\\n",
    "    SentinelHubCatalog, filter_times, \\\n",
    "    SentinelHubRequest, MimeType, SentinelHubDownloadClient\n",
    "\n",
    "import datetime as dt\n",
    "#import tarfile\n",
    "from os.path import join, dirname\n",
    "from os import rename\n",
    "from shutil import rmtree\n",
    "\n",
    "import requests\n",
    "from oauthlib.oauth2 import BackendApplicationClient\n",
    "from requests_oauthlib import OAuth2Session\n",
    "from sentinelhub import SentinelHubCatalog, BBox, Geometry, SHConfig, CRS, DataCollection\n",
    "from sentinelhub.time_utils import serialize_time, parse_time_interval, parse_time\n",
    "from sentinelhub.sh_utils import FeatureIterator, remove_undefined\n",
    "from sentinelhub.sentinelhub_catalog import CatalogSearchIterator\n",
    "\n",
    "import json\n",
    "from dateutil.parser import parse"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "597f1769-88e4-4fc7-a667-01d906bc000d",
   "metadata": {},
   "source": [
    "## Prepare input data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "78d64b21-7085-4319-8fe8-3931a25920ea",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(796, 499)\n",
      "12.44693,41.870072,12.541001,41.917096\n"
     ]
    }
   ],
   "source": [
    "# Data collection\n",
    "data_collection = DataCollection.LANDSAT_OT_L2  # DataCollection.SENTINEL2_L2A\n",
    "\n",
    "# Resolution (pixel size)\n",
    "resolution = 10\n",
    "\n",
    "# # CRS for sentinelhub query getattr() assigns module CRS with UTM_### method ==> CRS.UTM_36N\n",
    "# utm_zone = \"36N\"\n",
    "# sample_crs = getattr(CRS, f\"UTM_{utm_zone}\")\n",
    "# # List of supported CRSs: https://docs.sentinel-hub.com/api/latest/api/process/crs/\n",
    "\n",
    "# ------------------------------------------------------------------------------------------------\n",
    "\n",
    "# ## Build bbox from centrepoint + distance\n",
    "# # centrepoint\n",
    "# utm_x = 512817\n",
    "# utm_y = 269983\n",
    "# # AOI size in meters (x, y)\n",
    "# aoi_size = (5120, 5120)\n",
    "\n",
    "# # Round UTM coords to nearest 10 (so the raster will look nice in UTM - \"target aligned pixels\")\n",
    "# x_y = (round(utm_x, -1), round(utm_y, -1)) \n",
    "\n",
    "# bbox_utm = [x_y[0] - aoi_size[0]/2,\n",
    "#             x_y[1] - aoi_size[1]/2,\n",
    "#             x_y[0] + aoi_size[0]/2,\n",
    "#             x_y[1] + aoi_size[1]/2]\n",
    "\n",
    "\n",
    "# ## Use this to create a bbox directly:\n",
    "# bbox_utm = [510260, 267420, 515380, 272540]\n",
    "\n",
    "# ------------------------------------------------------------------------------------------------\n",
    "\n",
    "# Prepare some data for sentinelhub\n",
    "# sample_bbox = BBox(bbox=bbox_utm, crs=sample_crs)\n",
    "sample_bbox = BBox(bbox=[12.44693, 41.870072, 12.541001, 41.917096], crs=CRS.WGS84)\n",
    "sample_size = bbox_to_dimensions(sample_bbox, resolution=resolution)\n",
    "print(sample_size)\n",
    "print(sample_bbox)\n",
    "# print(sample_crs)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b26c7117-21e2-4c39-bf82-853c8289e412",
   "metadata": {},
   "source": [
    "## Search for all available images in a time frame"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "99c8df9a-9c61-4849-ae92-1617bcdfa81b",
   "metadata": {},
   "source": [
    "Kako pa dobimo vse posnetke (vse datume) znotraj enega četrtletja?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "bfdefc24-9465-4de8-bcfc-64bf03d28cb4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of results: 44\n",
      "Number of unique acquisitions: 44\n",
      "Wall time: 1.16 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Sarch catalog for all images with bbox in a time interval\n",
    "catalog = SentinelHubCatalog(config=config)\n",
    "\n",
    "# SET TIME INTERVAL\n",
    "time = ('2019-01-01', '2019-12-31')\n",
    "collection = data_collection\n",
    "\n",
    "# # ----------------------------------------------------------------------------------------------\n",
    "\n",
    "# # Set up credentials for use with batch\n",
    "# client = BackendApplicationClient(client_id=config.sh_client_id)\n",
    "# oauth = OAuth2Session(client=client)\n",
    "\n",
    "# # Fetch a token\n",
    "# token = oauth.fetch_token(token_url='https://services.sentinel-hub.com/oauth/token',\n",
    "#                           client_id=config.sh_client_id, client_secret=config.sh_client_secret)\n",
    "\n",
    "# # Set-up batch parameters: HERE we change the endpoint\n",
    "# url = 'https://services-uswest2.sentinel-hub.com/api/v1/catalog/search'\n",
    "# headers = {\n",
    "#     \"Authorization\": \"Bearer \" + token['access_token'],\n",
    "#     \"Content-Type\": \"application/json\"\n",
    "# }\n",
    "\n",
    "# collection_id = catalog._parse_collection_id(collection)\n",
    "# start_time, end_time = serialize_time(parse_time_interval(time, allow_undefined=True), use_tz=True)\n",
    "# if sample_bbox and sample_bbox.crs is not CRS.WGS84:\n",
    "#     sample_bbox = sample_bbox.transform_bounds(CRS.WGS84)\n",
    "\n",
    "# payload = {\n",
    "#     'collections': [collection_id],\n",
    "#     'datetime': f'{start_time}/{end_time}' if time else None,\n",
    "#     'bbox': list(sample_bbox) if sample_bbox else None,\n",
    "#     'fields': {\"include\": [\"id\", \"properties.datetime\"]},\n",
    "#     'limit': 60,\n",
    "# }\n",
    "# #print(payload)\n",
    "\n",
    "# response = oauth.request(\"POST\", url, headers=headers, json=payload)\n",
    "\n",
    "# #response.json()\n",
    "\n",
    "# all_timestamps = [parse(feature[\"properties\"]['datetime']) for feature in response.json()[\"features\"]]\n",
    "\n",
    "search_iterator = catalog.search(\n",
    "    data_collection,\n",
    "    bbox=sample_bbox,\n",
    "    time=time,\n",
    "    fields={\n",
    "        \"include\": [\n",
    "            \"id\",\n",
    "            \"properties.datetime\",\n",
    "            \"properties.eo:cloud_cover\"\n",
    "        ],\n",
    "        \"exclude\": []\n",
    "    }\n",
    "\n",
    ")\n",
    "\n",
    "results = list(search_iterator)\n",
    "print('Total number of results:', len(list(search_iterator)))\n",
    "# print(results[:4])\n",
    "\n",
    "# Many timestamps differ only for a few seconds. That is because they are from\n",
    "# tiles in the same orbit acquisition. We want to join them together in a single timestamp.\n",
    "\n",
    "time_difference = dt.timedelta(hours=1)\n",
    "all_timestamps = search_iterator.get_timestamps()\n",
    "\n",
    "unique_acquisitions = filter_times(all_timestamps, time_difference)\n",
    "print(f\"Number of unique acquisitions: {len(unique_acquisitions)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "b9e658e5-d490-4f07-b216-ce36e759f354",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LC08_L2SP_190031_20191228_20200825_02_T1\n",
      "LC08_L2SP_191031_20191219_20201023_02_T2\n",
      "LC08_L2SP_191031_20191203_20200825_02_T1\n",
      "LC08_L2SP_190031_20191126_20200825_02_T1\n",
      "LC08_L2SP_191031_20191117_20200825_02_T2\n",
      "LC08_L2SP_190031_20191110_20200825_02_T1\n",
      "LC08_L2SP_191031_20191101_20200825_02_T1\n",
      "LC08_L2SP_190031_20191025_20200825_02_T1\n",
      "LC08_L2SP_191031_20191016_20200825_02_T1\n",
      "LC08_L2SP_190031_20191009_20200825_02_T1\n",
      "LC08_L2SP_191031_20190930_20200825_02_T1\n",
      "LC08_L2SP_190031_20190923_20200826_02_T1\n",
      "LC08_L2SP_191031_20190914_20200826_02_T1\n",
      "LC08_L2SP_190031_20190907_20200826_02_T1\n",
      "LC08_L2SP_191031_20190829_20200826_02_T1\n",
      "LC08_L2SP_190031_20190822_20200827_02_T1\n",
      "LC08_L2SP_191031_20190813_20200827_02_T1\n",
      "LC08_L2SP_190031_20190806_20200827_02_T1\n",
      "LC08_L2SP_191031_20190728_20200827_02_T1\n",
      "LC08_L2SP_190031_20190721_20200827_02_T1\n",
      "LC08_L2SP_191031_20190712_20200827_02_T1\n",
      "LC08_L2SP_190031_20190705_20200827_02_T1\n",
      "LC08_L2SP_191031_20190626_20200827_02_T1\n",
      "LC08_L2SP_190031_20190619_20200827_02_T1\n",
      "LC08_L2SP_191031_20190610_20200828_02_T1\n",
      "LC08_L2SP_190031_20190603_20200828_02_T1\n",
      "LC08_L2SP_191031_20190525_20200828_02_T1\n",
      "LC08_L2SP_190031_20190518_20200828_02_T1\n",
      "LC08_L2SP_191031_20190509_20200829_02_T1\n",
      "LC08_L2SP_191031_20190423_20200829_02_T1\n",
      "LC08_L2SP_190031_20190416_20200829_02_T1\n",
      "LC08_L2SP_191031_20190407_20200829_02_T1\n",
      "LC08_L2SP_190031_20190331_20200829_02_T1\n",
      "LC08_L2SP_191031_20190322_20200829_02_T1\n",
      "LC08_L2SP_190031_20190315_20200829_02_T1\n",
      "LC08_L2SP_191031_20190306_20200829_02_T1\n",
      "LC08_L2SP_190031_20190227_20200829_02_T1\n",
      "LC08_L2SP_191031_20190218_20200829_02_T1\n",
      "LC08_L2SP_190031_20190211_20200829_02_T1\n",
      "LC08_L2SP_191031_20190202_20200829_02_T2\n",
      "LC08_L2SP_190031_20190126_20200830_02_T1\n",
      "LC08_L2SP_191031_20190117_20200829_02_T2\n",
      "LC08_L2SP_190031_20190110_20200829_02_T1\n",
      "LC08_L2SP_191031_20190101_20200830_02_T1\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None]"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[print(result[\"id\"]) for result in results]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7ad9fce9-6e8e-4b26-a78f-18d08f66a7fd",
   "metadata": {},
   "source": [
    "## Filter by cloud cover"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e4e67d3f-7333-414e-9c75-db67de551225",
   "metadata": {},
   "source": [
    "### Download all CLMs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "44297577-a67e-41c9-8ba4-b300a21f6e19",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloaded: 44\n",
      "Wall time: 5.8 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "# Create a Process API request for each acquisition:\n",
    "cm_evalscript = \"\"\"\n",
    "    //VERSION=3\n",
    "\n",
    "    function setup() {\n",
    "        return {\n",
    "            input: [\"BQA\"],\n",
    "            output: {\n",
    "                bands: 1,\n",
    "                sampleType: \"UINT8\"\n",
    "            }\n",
    "        };\n",
    "    }\n",
    "\n",
    "    function evaluatePixel(sample) {\n",
    "        return [decodeLs8Qa(sample.BQA).cloud];\n",
    "    }\n",
    "\"\"\"\n",
    "\n",
    "# Requests will be appenede to this list\n",
    "process_requests = []\n",
    "\n",
    "for timestamp in unique_acquisitions:\n",
    "    request = SentinelHubRequest(\n",
    "        evalscript=cm_evalscript,\n",
    "        input_data=[\n",
    "            SentinelHubRequest.input_data(\n",
    "                data_collection=data_collection,\n",
    "                time_interval=(timestamp - time_difference, timestamp + time_difference)\n",
    "            )\n",
    "        ],\n",
    "        responses=[\n",
    "            SentinelHubRequest.output_response('default', MimeType.TIFF)\n",
    "        ],\n",
    "        bbox=sample_bbox,\n",
    "        size=sample_size,\n",
    "        config=config\n",
    "    )\n",
    "    process_requests.append(request)\n",
    "    \n",
    "\n",
    "# In order to efficiently download data for all requests in parallel, we extract download information and pass it to a download client.\n",
    "client = SentinelHubDownloadClient(config=config)\n",
    "download_requests = [request.download_list[0] for request in process_requests]\n",
    "data = client.download(download_requests)\n",
    "\n",
    "print(f\"Downloaded: {len(data)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "27630473-2cff-4b0a-a505-605a5bbf6a60",
   "metadata": {},
   "source": [
    "We used the [decodeLs8Qa()](https://docs.sentinel-hub.com/api/latest/evalscript/functions/#decodels8qa) function to evaluate cloud cover of Landsat images.\n",
    "\n",
    "> *cloud:*\n",
    ">- 0  -  this condition doesn't exist\n",
    ">- 1  -  this condition exists"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "f8377f8a-b4c5-4845-b3e7-0e85388313ed",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8804\n",
      "388400\n"
     ]
    }
   ],
   "source": [
    "print((data[0] == 1).sum())  # CLOUD\n",
    "print((data[0] == 0).sum())  # NO CLOUD"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2c8796c5-0e82-4ea1-8926-5c69e6d3ca50",
   "metadata": {},
   "source": [
    "### Filter by cloud cover"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "7775cd30-70fe-4d97-9cd7-3e367b5c8478",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "28\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[(datetime.datetime(2019, 1, 10, 9, 47, 3, 171000, tzinfo=tzutc()), 0.0),\n",
       " (datetime.datetime(2019, 1, 26, 9, 46, 59, 247000, tzinfo=tzutc()), 0.0),\n",
       " (datetime.datetime(2019, 2, 2, 9, 53, 8, 812000, tzinfo=tzutc()), 0.0),\n",
       " (datetime.datetime(2019, 2, 11, 9, 46, 57, 109000, tzinfo=tzutc()), 0.0),\n",
       " (datetime.datetime(2019, 2, 18, 9, 53, 6, 675000, tzinfo=tzutc()), 0.0),\n",
       " (datetime.datetime(2019, 3, 6, 9, 53, 1, 985000, tzinfo=tzutc()), 0.0),\n",
       " (datetime.datetime(2019, 3, 15, 9, 46, 47, 655000, tzinfo=tzutc()), 0.0),\n",
       " (datetime.datetime(2019, 3, 22, 9, 52, 57, 519000, tzinfo=tzutc()), 0.0),\n",
       " (datetime.datetime(2019, 3, 31, 9, 46, 44, 746000, tzinfo=tzutc()), 0.0),\n",
       " (datetime.datetime(2019, 5, 9, 9, 52, 53, 219000, tzinfo=tzutc()), 0.0),\n",
       " (datetime.datetime(2019, 5, 25, 9, 53, 2, 990000, tzinfo=tzutc()), 0.0),\n",
       " (datetime.datetime(2019, 6, 3, 9, 46, 56, 674000, tzinfo=tzutc()), 0.0),\n",
       " (datetime.datetime(2019, 6, 19, 9, 47, 2, 910000, tzinfo=tzutc()), 0.0),\n",
       " (datetime.datetime(2019, 6, 26, 9, 53, 15, 796000, tzinfo=tzutc()), 0.0),\n",
       " (datetime.datetime(2019, 7, 5, 9, 47, 7, 121000, tzinfo=tzutc()), 0.0),\n",
       " (datetime.datetime(2019, 7, 21, 9, 47, 10, 380000, tzinfo=tzutc()), 0.0),\n",
       " (datetime.datetime(2019, 7, 28, 9, 53, 24, 388000, tzinfo=tzutc()), 0.0),\n",
       " (datetime.datetime(2019, 8, 6, 9, 47, 17, 153000, tzinfo=tzutc()), 0.0),\n",
       " (datetime.datetime(2019, 8, 22, 9, 47, 21, 945000, tzinfo=tzutc()), 0.0),\n",
       " (datetime.datetime(2019, 9, 7, 9, 47, 25, 318000, tzinfo=tzutc()), 0.0),\n",
       " (datetime.datetime(2019, 9, 14, 9, 53, 38, 980000, tzinfo=tzutc()), 0.0),\n",
       " (datetime.datetime(2019, 10, 9, 9, 47, 34, 752000, tzinfo=tzutc()), 0.0),\n",
       " (datetime.datetime(2019, 10, 25, 9, 47, 35, 799000, tzinfo=tzutc()), 0.0),\n",
       " (datetime.datetime(2019, 11, 10, 9, 47, 33, 986000, tzinfo=tzutc()), 0.0),\n",
       " (datetime.datetime(2019, 11, 26, 9, 47, 31, 197000, tzinfo=tzutc()), 0.0),\n",
       " (datetime.datetime(2019, 12, 3, 9, 53, 41, 904000, tzinfo=tzutc()), 0.0),\n",
       " (datetime.datetime(2019, 12, 19, 9, 53, 39, 679000, tzinfo=tzutc()), 0.0),\n",
       " (datetime.datetime(2019, 12, 28, 9, 47, 26, 419000, tzinfo=tzutc()), 0.0)]"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Select threshold (e.g. 0.05 for less or equal than 5%)\n",
    "threshold = 0\n",
    "# Calculat cloud ratio for all images\n",
    "cc_ratio = [(cc_image == 1).sum() / cc_image.size for cc_image in data]\n",
    "# Select those below the threshold\n",
    "lt5_cc = [(i, cc) for i, cc in zip(unique_acquisitions, cc_ratio) if cc <= threshold]\n",
    "\n",
    "print(len(lt5_cc))\n",
    "lt5_cc"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dbe50e4a-99a9-4a85-ab61-ac49e710235d",
   "metadata": {},
   "source": [
    "## Download selected images (num)\n",
    "\n",
    "Evalscript parameters: [https://docs.sentinel-hub.com/api/latest/evalscript/v3/#parameters](https://docs.sentinel-hub.com/api/latest/evalscript/v3/#parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "607af2fa-3e8f-4793-a17f-a8e1b949e330",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 1min 22s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "# Double curved brackets, so we can insert variables using .format() method where single brackets are present\n",
    "my_evalscript = \"\"\"\n",
    "    //VERSION=3\n",
    "\n",
    "    function setup() {{\n",
    "        return {{\n",
    "            input: [{{\n",
    "                bands: [\"B02\",\"B03\",\"B04\"],\n",
    "                units: \"reflectance\"  // \"DN\" for Digital Numbers, \"reflectance\" for normalised\n",
    "            }}],\n",
    "            output: {{\n",
    "                id: \"{0}\",\n",
    "                bands: 3,\n",
    "                sampleType: \"FLOAT32\"  // \"UINT16\" for \"DN\", \"FLOAT32\" for \"reflectance\" \n",
    "            }}\n",
    "        }};\n",
    "    }}\n",
    "    \n",
    "    //NOT USED IF ONLY ONE TIFF IS DEFINED IN RESPONSES (uncomment in the SentinelHubRequest below)\n",
    "    function updateOutputMetadata(scenes, inputMetadata, outputMetadata) {{\n",
    "        outputMetadata.userData = {{ \"norm_factor\":  inputMetadata.normalizationFactor }}\n",
    "    }}\n",
    "\n",
    "    function evaluatePixel(sample) {{\n",
    "        return [sample.B04, sample.B03, sample.B02];\n",
    "    }}\n",
    "\"\"\"\n",
    "\n",
    "data_folder = \".\\\\downloaded\"\n",
    "\n",
    "process_requests = []\n",
    "save_locations = []\n",
    "for timestamp, _ in lt5_cc:\n",
    "    # Insert custom name for output file into evalscript\n",
    "    my_string = timestamp.strftime(\"Landsat_%Y%m%dT%H%M%S_cloudless\")\n",
    "    evalscript = my_evalscript.format(my_string)\n",
    "    \n",
    "    # Define custom name for correct file naming\n",
    "    my_folder = join(data_folder, my_string)\n",
    "    \n",
    "    request = SentinelHubRequest(\n",
    "        data_folder=my_folder,\n",
    "        evalscript=evalscript,\n",
    "        input_data=[\n",
    "            SentinelHubRequest.input_data(\n",
    "                data_collection=data_collection,\n",
    "                time_interval=(timestamp - time_difference, timestamp + time_difference)\n",
    "            )\n",
    "        ],\n",
    "        responses=[\n",
    "            SentinelHubRequest.output_response(my_string, MimeType.TIFF),\n",
    "            #SentinelHubRequest.output_response(\"userdata\", MimeType.JSON)\n",
    "        ],\n",
    "        bbox=sample_bbox,\n",
    "        size=sample_size,\n",
    "        config=config\n",
    "    )\n",
    "    process_requests.append(request)\n",
    "    save_locations.append(join(my_folder, request.get_filename_list()[0]))\n",
    "\n",
    "\n",
    "# Download files\n",
    "download_requests = [request.save_data(raise_download_errors=True) for request in process_requests]\n",
    "\n",
    "# Rename filest\n",
    "for file in save_locations:\n",
    "    target_name = dirname(dirname(file))\n",
    "    rename(file, target_name + \".tiff\")\n",
    "    # Remove leftover folder\n",
    "    rmtree(target_name)\n",
    "\n",
    "# # Now extract all the files (only if more than one output was defined)\n",
    "# abc = [request.get_filename_list()[0] for request in process_requests]\n",
    "# for image in abc:\n",
    "#     tar_path = join(data_folder, image)\n",
    "#     with tarfile.open(tar_path) as my_tar:\n",
    "#         my_tar.extractall(data_folder)\n",
    "#     rmtree(dirname(tar_path))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0806233d-710a-4931-869f-ddbc946c6bf3",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Visualize results in Jupyter Notebook\n",
    "\n",
    "TBA"
   ]
  },
  {
   "cell_type": "raw",
   "id": "6cb1f7c3-144f-42df-9ffa-eb2346390947",
   "metadata": {},
   "source": [
    "evalscript_tc = \"\"\"\n",
    "    //VERSION=3\n",
    "    function setup() {\n",
    "        return {\n",
    "            input: [{\n",
    "                bands: [\"B02\",\"B03\",\"B04\",\"CLM\"],\n",
    "            }],\n",
    "            output: {\n",
    "                bands: 3,\n",
    "            }\n",
    "        };\n",
    "    }\n",
    "\n",
    "    function evaluatePixel(sample) {\n",
    "        return [sample.B04, sample.B03, sample.B02];\n",
    "    }\n",
    "\"\"\"\n",
    "\n",
    "def get_true_color_request(time_interval):\n",
    "    return SentinelHubRequest(\n",
    "        evalscript=evalscript_tc,\n",
    "        input_data=[\n",
    "            SentinelHubRequest.input_data(\n",
    "                data_collection=data_collection,\n",
    "                time_interval=time_interval,\n",
    "                mosaicking_order='leastCC',\n",
    "            )\n",
    "        ],\n",
    "        responses=[\n",
    "            SentinelHubRequest.output_response('default', MimeType.PNG)\n",
    "        ],\n",
    "        bbox=sample_bbox,\n",
    "        size=sample_size,\n",
    "        config=config\n",
    "    )\n",
    "\n",
    "# --------------------------------------------------------------------------\n",
    "# Set time intervals\n",
    "from dateutil.relativedelta import relativedelta\n",
    "slots = [(\n",
    "    datetime.datetime(2018,mth,1).date().isoformat(),\n",
    "    (datetime.datetime(2018,mth,1) + relativedelta(months=3)).date().isoformat()\n",
    ") for mth in range(1, 13, 3)]\n",
    "\n",
    "\n",
    "# create a list of requests\n",
    "list_of_requests = [get_true_color_request(slot) for slot in slots]\n",
    "list_of_requests = [request.download_list[0] for request in list_of_requests]\n",
    "\n",
    "# download data with multiple threads\n",
    "data = SentinelHubDownloadClient(config=config).download(list_of_requests, max_threads=5)\n",
    "\n",
    "# some stuff for pretty plots\n",
    "ncols = 2\n",
    "nrows = 2\n",
    "aspect_ratio = sample_size[0] / sample_size[1]\n",
    "subplot_kw = {'xticks': [], 'yticks': [], 'frame_on': False}\n",
    "\n",
    "fig, axs = plt.subplots(ncols=ncols, nrows=nrows, figsize=(5 * ncols * aspect_ratio, 5 * nrows),\n",
    "                        subplot_kw=subplot_kw)\n",
    "\n",
    "for idx, image in enumerate(data):\n",
    "    ax = axs[idx // ncols][idx % ncols]\n",
    "    ax.imshow(np.clip(image * 3.5/255, 0, 1))\n",
    "    ax.set_title(f'{slots[idx][0]}  -  {slots[idx][1]}', fontsize=10)\n",
    "\n",
    "plt.tight_layout()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "76dc4d58-cafb-41ce-ad97-a17c671a756f",
   "metadata": {},
   "source": [
    "## Download numpy and save GeoTIFF with rasterio\n",
    "\n",
    "TBA"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}