added files subpackage to download from the new ENTSOE File Library

Author: fboerman

README.md

@@ -7,7 +7,7 @@ Documentation of the API found on https://transparency.entsoe.eu/content/static_
 `python3 -m pip install entsoe-py`
 
 ## Usage
-The package comes with 2 clients:
+The package comes with 2 clients for the REST API:
 - [`EntsoeRawClient`](#EntsoeRawClient): Returns data in its raw format, usually XML or a ZIP-file containing XML's
 - [`EntsoePandasClient`](#EntsoePandasClient): Returns data parsed as a Pandas Series or DataFrame
 ### <a name="EntsoeRawClient"></a>EntsoeRawClient
@@ -148,6 +148,23 @@ ts = client.query_day_ahead_prices(country_code, start=start, end=end)
 ts.to_csv('outfile.csv')
 ```
 
+### Download from ENTSOE File Library
+To download from the file libary, which replaced the old SFTP use the ```files``` subpackage with the ```EntsoeFileClient```
+
+```python
+from entsoe.files import EntsoeFileClient
+
+client = EntsoeFileClient(username=<YOUR ENTSOE USERNAME>, pwd=<YOUR ENTSOE PASSWORD>)
+# this returns a dict of {filename: unique_id}:
+file_list = client.list_folder('AcceptedAggregatedOffers_17.1.D')
+
+# either download one file by name:
+df = client.download_single_file(folder='AcceptedAggregatedOffers_17.1.D', filename=list(file_list.keys())[0])
+
+# or download multiple by unique_id:
+df = client.download_multiple_files(['a1a82b3f-c453-4181-8d20-ad39c948d4b0', '64e47e15-bac6-4212-b2dd-9667bdf33b5d'])
+```
+
 ### Mappings
 These lists are always evolving, so let us know if something's inaccurate!
 

entsoe/files/__init__.py

@@ -0,0 +1 @@
+from .entsoe_files import EntsoeFileClient

entsoe/files/decorators.py

@@ -0,0 +1,15 @@
+from functools import wraps
+import pandas as pd
+
+
+def check_expired(func):
+    @wraps(func)
+    def check_expired_wrapper(*args, **kwargs):
+        self = args[0]
+
+        if pd.Timestamp.now(tz='europe/amsterdam') >= self.expire:
+            self._update_token()
+
+        return func(*args, **kwargs)
+
+    return check_expired_wrapper

entsoe/files/entsoe_files.py

@@ -0,0 +1,133 @@
+from typing import Optional, Dict
+import requests
+from entsoe import __version__
+import pandas as pd
+import json
+from io import BytesIO
+import zipfile
+from .decorators import check_expired
+
+# DOCS for entsoe file library: https://transparencyplatform.zendesk.com/hc/en-us/articles/35960137882129-File-Library-Guide
+# postman description: https://documenter.getpostman.com/view/28274243/2sB2qgfz3W
+
+
+class EntsoeFileClient:
+    BASEURL = "https://fms.tp.entsoe.eu/"
+
+    def __init__(self, username: str, pwd: str, session: Optional[requests.Session] = None,
+                 proxies: Optional[Dict] = None, timeout: Optional[int] = None
+                 ):
+        self.proxies = proxies
+        self.timeout = timeout
+        self.username = username
+        self.pwd = pwd
+        if session is None:
+            session = requests.Session()
+        self.session = session
+        self.session.headers.update({
+            'user-agent': f'entsoe-py {__version__} (github.com/EnergieID/entsoe-py)'
+        })
+
+        self.access_token = None
+        self.expire = None
+
+        self._update_token()
+
+    def _update_token(self):
+        # different url that other calls so hardcoded new one here
+        r = self.session.post(
+            'https://keycloak.tp.entsoe.eu/realms/tp/protocol/openid-connect/token', data={
+                'client_id': 'tp-fms-public',
+                'grant_type': 'password',
+                'username': self.username,
+                'password': self.pwd
+            },
+            proxies=self.proxies, timeout=self.timeout
+        )
+        r.raise_for_status()
+        data = r.json()
+        self.expire = pd.Timestamp.now(tz='europe/amsterdam') + pd.Timedelta(seconds=data['expires_in'])
+        self.access_token = data['access_token']
+
+    @check_expired
+    def list_folder(self, folder: str) -> dict:
+        """
+        returns a dictionary of filename: unique file id
+        """
+        if not folder.endswith('/'):
+            folder += '/'
+        r = self.session.post(self.BASEURL + "listFolder",
+                              data=json.dumps({
+                                  "path": "/TP_export/" + folder,
+                                  "sorterList": [
+                                      {
+                                          "key": "periodCovered.from",
+                                          "ascending": True
+                                      }
+                                  ],
+                                  "pageInfo": {
+                                      "pageIndex": 0,
+                                      "pageSize": 5000  # this should be enough for basically anything right now
+                                  }
+                              }),
+                              headers={
+                                  'Authorization': f'Bearer {self.access_token}',
+                                  'Content-Type': 'application/json'
+                              },
+                              proxies=self.proxies, timeout=self.timeout)
+        r.raise_for_status()
+        data = r.json()
+        return {x['name']: x['fileId'] for x in data['contentItemList']}
+
+    @check_expired
+    def download_single_file(self, folder, filename) -> pd.DataFrame:
+        """
+        download a file by filename, it is important to split folder and filename here
+        """
+        if not folder.endswith('/'):
+            folder += '/'
+        r = self.session.post(self.BASEURL + "downloadFileContent",
+                              data=json.dumps({
+                                  "folder": "/TP_export/" + folder,
+                                  "filename": filename,
+                                  "downloadAsZip": True,
+                                  "topLevelFolder": "TP_export",
+                              }),
+                              headers={
+                                  'Authorization': f'Bearer {self.access_token}',
+                                  'Content-Type': 'application/json'
+                              })
+        r.raise_for_status()
+        stream = BytesIO(r.content)
+        stream.seek(0)
+        zf = zipfile.ZipFile(stream)
+        with zf.open(zf.filelist[0].filename) as file:
+            return pd.read_csv(file, sep='\t')
+
+    @check_expired
+    def download_multiple_files(self, file_ids: list) -> pd.DataFrame:
+        """
+        for now when downloading multiple files only list of file ids is supported by this package
+        """
+        r = self.session.post(self.BASEURL + "downloadFileContent",
+                              data=json.dumps({
+                                  "fileIdList": file_ids,
+                                  "downloadAsZip": True,
+                                  "topLevelFolder": "TP_export",
+                              }),
+                              headers={
+                                  'Authorization': f'Bearer {self.access_token}',
+                                  'Content-Type': 'application/json'
+                              },
+                              proxies=self.proxies, timeout=self.timeout)
+        r.raise_for_status()
+        stream = BytesIO(r.content)
+        stream.seek(0)
+        zf = zipfile.ZipFile(stream)
+        df = []
+        for fz in zf.filelist:
+            with zf.open(fz.filename) as file:
+                df.append(pd.read_csv(file, sep='\t'))
+
+        return pd.concat(df)
+