Add test ci

.github/workflows/python-app.yml

@@ -0,0 +1,38 @@
+# This workflow will install Python dependencies, run tests and lint with a single version of Python
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
+
+name: Python application
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: create-json
+      id: create-json
+      uses: jsdaniell/create-json@v1.2.3
+      with:
+        name: "credentials.json"
+        json: ${{ secrets.GOOGLE_SHEETS_KEY_FILE }}
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v3
+      with:
+        python-version: "3.10"
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install google-api-python-client google-auth-httplib2 google-auth-oauthlib
+    - name: Run Python script to convert file into token
+      run: |
+        python ci_scripts/set_env_vars_for_tests.py

CMakeLists.txt

@@ -21,6 +21,7 @@ set(EXTENSION_SOURCES
     src/gsheets_requests.cpp
     src/gsheets_read.cpp
     src/gsheets_utils.cpp
+    src/gsheets_get_token.cpp
 )
 
 build_static_extension(${TARGET_NAME} ${EXTENSION_SOURCES})

ci_scripts/set_env_vars_for_tests.py

@@ -0,0 +1,28 @@
+import os
+from google.oauth2 import service_account
+from google.auth.transport.requests import Request
+
+def get_token_from_user_file(user_file_path):
+    SCOPES = ["https://www.googleapis.com/auth/spreadsheets"]
+
+    credentials = service_account.Credentials.from_service_account_file(
+        user_file_path,
+        scopes=SCOPES
+        )
+
+    request = Request()
+    credentials.refresh(request)
+    return credentials.token
+
+key_file_path = "credentials.json"
+token = get_token_from_user_file(key_file_path)
+
+env_file = os.getenv('GITHUB_ENV')
+
+with open(env_file, "a") as myfile:
+    # Set the token as an env var for some tests
+    myfile.write(f"TOKEN={token}\n")
+    # Set the key_file filepath as an env var for other tests
+    myfile.write(f"KEY_FILE_PATH={key_file_path}")
+
+print('It seems to have worked?')

docs/pages/index.md

@@ -40,6 +40,15 @@ CREATE SECRET (
     PROVIDER access_token, 
     TOKEN '<your_token>'
 );
+
+-- OR create a non-expiring JSON secret with your Google API private key 
+-- (This enables use in non-interactive workflows like data pipelines)
+-- (see "Getting a Google API Access Private Key" below)
+CREATE SECRET (
+    TYPE gsheet, 
+    PROVIDER key_file, 
+    FILEPATH '<path_to_JSON_file_with_private_key>'
+);
 ```
 
 ### Read
@@ -114,6 +123,22 @@ To connect DuckDB to Google Sheets via an access token, you’ll need to create
 
 This token will periodically expire - you can re-run the above command again to generate a new one.
 
+## Getting a Google API Access Private Key
+
+Follow steps 1-9 above to get a JSON file with your private key inside.
+
+Include the path to the file as the `FILEPATH` parameter when creating a secret.
+Ex: `CREATE SECRET (TYPE gsheet, PROVIDER key_file, FILEPATH '<path_to_JSON_file_with_private_key>');`
+
+You can skip steps 10, 11, and 12 since this extension will convert from your JSON file to a token on your behalf!
+The contents of the JSON file will be stored in the secret, as will the temporary token.
+
+Follow steps 13 and 14.
+
+This private key by default will not expire. Use caution with it. 
+
+This will also require an additional API request approximately every 30 minutes.
+
 ## Limitations / Known Issues
 
 - DuckDB WASM is not (yet) supported.

src/gsheets_auth.cpp

@@ -1,11 +1,15 @@
 #include "gsheets_auth.hpp"
 #include "gsheets_requests.hpp"
 #include "gsheets_utils.hpp"
+#include "gsheets_get_token.hpp"
 #include "duckdb/common/exception.hpp"
 #include "duckdb/main/secret/secret.hpp"
 #include "duckdb/main/extension_util.hpp"
 #include <fstream>
 #include <cstdlib>
+#include <json.hpp>
+
+using json = nlohmann::json;
 
 namespace duckdb
 {
@@ -78,6 +82,42 @@ namespace duckdb
         return std::move(result);
     }
 
+    // TODO: Maybe this should be a KeyValueSecret
+    static unique_ptr<BaseSecret> CreateGsheetSecretFromKeyFile(ClientContext &context, CreateSecretInput &input) {
+        auto scope = input.scope;
+
+        auto result = make_uniq<KeyValueSecret>(scope, input.type, input.provider, input.name);
+
+        // Want to store the private key and email in case the secret is persisted
+        std::string filepath_key = "filepath";
+        auto filepath = (input.options.find(filepath_key)->second).ToString();
+
+        std::ifstream ifs(filepath);
+        json credentials_file = json::parse(ifs);
+        std::string email = credentials_file["client_email"].get<std::string>();
+        std::string secret = credentials_file["private_key"].get<std::string>();
+
+        // Manage specific secret option
+        (*result).secret_map["email"] = Value(email);
+        (*result).secret_map["secret"] = Value(secret);
+        CopySecret("filepath", input, *result); // Store the filepath anyway
+
+        const auto result_const = *result;
+        TokenDetails token_details = get_token(context, &result_const);
+        std::string token = token_details.token;
+
+        (*result).secret_map["token"] = Value(token);
+        (*result).secret_map["token_expiration"] = Value(token_details.expiration_time);
+
+        // Redact sensible keys
+        RedactCommonKeys(*result);
+        result->redact_keys.insert("secret");
+        result->redact_keys.insert("filepath");
+        result->redact_keys.insert("token");
+
+        return std::move(result);
+    }
+
     void CreateGsheetSecretFunctions::Register(DatabaseInstance &instance)
     {
         string type = "gsheet";
@@ -100,6 +140,12 @@ namespace duckdb
         oauth_function.named_parameters["use_oauth"] = LogicalType::BOOLEAN;
         RegisterCommonSecretParameters(oauth_function);
         ExtensionUtil::RegisterFunction(instance, oauth_function);
+
+        // Register the key_file secret provider
+        CreateSecretFunction key_file_function = {type, "key_file", CreateGsheetSecretFromKeyFile};
+        key_file_function.named_parameters["filepath"] = LogicalType::VARCHAR;
+        RegisterCommonSecretParameters(key_file_function);
+        ExtensionUtil::RegisterFunction(instance, key_file_function);
     }
 
     std::string InitiateOAuthFlow()

src/gsheets_copy.cpp

@@ -7,6 +7,9 @@
 #include "duckdb/common/file_system.hpp"
 #include "duckdb/main/secret/secret_manager.hpp"
 #include <json.hpp>
+#include <regex>
+#include "gsheets_get_token.hpp"
+#include <iostream>
 
 using json = nlohmann::json;
 
@@ -49,12 +52,22 @@ namespace duckdb
             throw InvalidInputException("Invalid secret format for 'gsheet' secret");
         }
 
-        Value token_value;
-        if (!kv_secret->TryGetValue("token", token_value)) {
-            throw InvalidInputException("'token' not found in 'gsheet' secret");
-        }
+        std::string token;
+
+        if (secret.GetProvider() == "key_file") {
+            // If using a private key, retrieve the private key from the secret, but convert it 
+            // into a token before use. This is an extra request per 30 minutes.
+            // The secret is the JSON file that is extracted from Google as per the README
+            token = get_token_and_cache(context, transaction, kv_secret);
 
-        std::string token = token_value.ToString();
+        } else {
+            Value token_value;
+            if (!kv_secret->TryGetValue("token", token_value)) {
+                throw InvalidInputException("'token' not found in 'gsheet' secret");
+            }
+
+            token = token_value.ToString();
+        }
         std::string spreadsheet_id = extract_spreadsheet_id(file_path);
         std::string sheet_id = extract_sheet_id(file_path);
         std::string sheet_name = "Sheet1";