From 9500be5b0d4920a699a4ed5d8257f17934b6f7b7 Mon Sep 17 00:00:00 2001 From: Scott Willeke Date: Tue, 18 Jun 2024 16:17:36 -0400 Subject: [PATCH 01/39] wip: feat: initial logging destination for glide --- .../connectors/destination-glide/.gitignore | 1 + .../connectors/destination-glide/README.md | 103 ++ .../destination_glide/__init__.py | 8 + .../destination_glide/destination.py | 105 ++ .../destination_glide/run.py | 13 + .../destination_glide/spec.json | 27 + .../connectors/destination-glide/dev-check.sh | 5 + .../connectors/destination-glide/dev-spec.sh | 5 + .../connectors/destination-glide/dev-write.sh | 5 + .../integration_tests/integration_test.py | 110 ++ .../connectors/destination-glide/main.py | 11 + .../destination-glide/metadata.yaml | 31 + .../connectors/destination-glide/poetry.lock | 1302 +++++++++++++++++ .../destination-glide/pyproject.toml | 28 + .../configured_catalog-test1.json | 27 + .../scripts/build-docker-image.sh | 6 + .../destination-glide/scripts/dev-check.sh | 5 + .../destination-glide/scripts/dev-spec.sh | 5 + .../destination-glide/scripts/dev-write.sh | 5 + .../scripts/push-docker-image.sh | 9 + .../scripts/test-integration.sh | 5 + .../destination-glide/test-integration.sh | 5 + .../destination-glide/unit_tests/unit_test.py | 7 + 23 files changed, 1828 insertions(+) create mode 100644 airbyte-integrations/connectors/destination-glide/.gitignore create mode 100644 airbyte-integrations/connectors/destination-glide/README.md create mode 100644 airbyte-integrations/connectors/destination-glide/destination_glide/__init__.py create mode 100644 airbyte-integrations/connectors/destination-glide/destination_glide/destination.py create mode 100644 airbyte-integrations/connectors/destination-glide/destination_glide/run.py create mode 100644 airbyte-integrations/connectors/destination-glide/destination_glide/spec.json create mode 100755 airbyte-integrations/connectors/destination-glide/dev-check.sh create mode 100755 airbyte-integrations/connectors/destination-glide/dev-spec.sh create mode 100755 airbyte-integrations/connectors/destination-glide/dev-write.sh create mode 100644 airbyte-integrations/connectors/destination-glide/integration_tests/integration_test.py create mode 100644 airbyte-integrations/connectors/destination-glide/main.py create mode 100644 airbyte-integrations/connectors/destination-glide/metadata.yaml create mode 100644 airbyte-integrations/connectors/destination-glide/poetry.lock create mode 100644 airbyte-integrations/connectors/destination-glide/pyproject.toml create mode 100644 airbyte-integrations/connectors/destination-glide/sample_files/configured_catalog-test1.json create mode 100755 airbyte-integrations/connectors/destination-glide/scripts/build-docker-image.sh create mode 100755 airbyte-integrations/connectors/destination-glide/scripts/dev-check.sh create mode 100755 airbyte-integrations/connectors/destination-glide/scripts/dev-spec.sh create mode 100755 airbyte-integrations/connectors/destination-glide/scripts/dev-write.sh create mode 100755 airbyte-integrations/connectors/destination-glide/scripts/push-docker-image.sh create mode 100755 airbyte-integrations/connectors/destination-glide/scripts/test-integration.sh create mode 100755 airbyte-integrations/connectors/destination-glide/test-integration.sh create mode 100644 airbyte-integrations/connectors/destination-glide/unit_tests/unit_test.py diff --git a/airbyte-integrations/connectors/destination-glide/.gitignore b/airbyte-integrations/connectors/destination-glide/.gitignore new file mode 100644 index 0000000000000..4089e8df247a5 --- /dev/null +++ b/airbyte-integrations/connectors/destination-glide/.gitignore @@ -0,0 +1 @@ +/destination-glide.log \ No newline at end of file diff --git a/airbyte-integrations/connectors/destination-glide/README.md b/airbyte-integrations/connectors/destination-glide/README.md new file mode 100644 index 0000000000000..44afb91a9e47e --- /dev/null +++ b/airbyte-integrations/connectors/destination-glide/README.md @@ -0,0 +1,103 @@ +# Glide Destination + +This is the repository for the Glide destination connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/destinations/glide). + +## Local development + +### Prerequisites + +* Python (`^3.9`) +* Poetry (`^1.7`) - installation instructions [here](https://python-poetry.org/docs/#installation) + + + +### Installing the connector + +From this connector directory, run: +```bash +poetry install --with dev +``` + + +#### Create credentials + +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/destinations/glide) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `destination_glide/spec.json` file. +Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `destination glide test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +poetry run destination-glide spec +poetry run destination-glide check --config secrets/config.json +poetry run destination-glide write --config secrets/config.json --catalog sample_files/configured_catalog.json +``` + +### Running tests + +To run tests locally, from the connector directory run: + +``` +poetry run pytest tests +``` + +### Building the docker image + +1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) +2. Run the following command to build the docker image: +```bash +airbyte-ci connectors --name=destination-glide build +``` + +An image will be available on your host with the tag `airbyte/destination-glide:dev`. + +### Running as a docker container + +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/destination-glide:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-glide:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/destination-glide:dev write --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` + +### Running our CI test suite + +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): + +```bash +airbyte-ci connectors --name=destination-glide test +``` + +### Customizing acceptance Tests + +Customize `acceptance-test-config.yml` file to configure acceptance tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + +### Dependency Management + +All of your dependencies should be managed via Poetry. +To add a new dependency, run: + +```bash +poetry add +``` + +Please commit the changes to `pyproject.toml` and `poetry.lock` files. + +## Publishing a new version of the connector + +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-glide test` +2. Bump the connector version (please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors)): + - bump the `dockerImageTag` value in in `metadata.yaml` + - bump the `version` value in `pyproject.toml` +3. Make sure the `metadata.yaml` content is up to date. +4. Make sure the connector documentation and its changelog is up to date (`docs/integrations/destinations/glide.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry. diff --git a/airbyte-integrations/connectors/destination-glide/destination_glide/__init__.py b/airbyte-integrations/connectors/destination-glide/destination_glide/__init__.py new file mode 100644 index 0000000000000..004353e6ffdea --- /dev/null +++ b/airbyte-integrations/connectors/destination-glide/destination_glide/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +# + + +from .destination import DestinationGlide + +__all__ = ["DestinationGlide"] diff --git a/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py b/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py new file mode 100644 index 0000000000000..274adfce484b5 --- /dev/null +++ b/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py @@ -0,0 +1,105 @@ +# +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +# +from airbyte_cdk.destinations import Destination +from airbyte_cdk.models import ( + AirbyteConnectionStatus, + AirbyteMessage, + ConfiguredAirbyteCatalog, + Status, + DestinationSyncMode, + Type +) +from collections import defaultdict +import datetime +import json +import logging +from typing import Any, Iterable, Mapping +import uuid + + +# Create a logger +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + +# Create a file handler +# TODO REMOVE? +handler = logging.FileHandler('destination-glide.log') +handler.setLevel(logging.DEBUG) + +# Create a logging format +formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +handler.setFormatter(formatter) + +# Add the handlers to the logger +logger.addHandler(handler) + +class DestinationGlide(Destination): + def write( + self, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog, input_messages: Iterable[AirbyteMessage] + ) -> Iterable[AirbyteMessage]: + + """ + TODO + Reads the input stream of messages, config, and catalog to write data to the destination. + + This method returns an iterable (typically a generator of AirbyteMessages via yield) containing state messages received + in the input message stream. Outputting a state message means that every AirbyteRecordMessage which came before it has been + successfully persisted to the destination. This is used to ensure fault tolerance in the case that a sync fails before fully completing, + then the source is given the last state message output from this method as the starting point of the next sync. + + :param config: dict of JSON configuration matching the configuration declared in spec.json + :param configured_catalog: The Configured Catalog describing the schema of the data being received and how it should be persisted in the + destination + :param input_messages: The stream of input messages received from the source + :return: Iterable of AirbyteStateMessages wrapped in AirbyteMessage structs + """ + + stream_names = {s.stream.name for s in configured_catalog.streams} + for configured_stream in configured_catalog.streams: + if configured_stream.destination_sync_mode != DestinationSyncMode.overwrite: + raise Exception(f"Only destination sync mode overwrite it supported, but received '{configured_stream.destination_sync_mode}'.") + # TODO: create a new GBT to prepare for dumping the data into it + table_name = f"_bgt_{configured_stream.stream.name}" + + # stream the records into the GBT: + buffer = defaultdict(list) + for message in input_messages: + if message.type == Type.RECORD: + data = message.record.data + stream = message.record.stream + if stream not in stream_names: + logger.debug(f"Stream {stream} was not present in configured streams, skipping") + continue + + # TODO: Check the columns match the columns that we saw in configured_catalog per https://docs.airbyte.com/understanding-airbyte/airbyte-protocol#destination + + # add to buffer + record_id = str(uuid.uuid4()) + buffer[stream].append((record_id, datetime.datetime.now().isoformat(), json.dumps(data))) + logger.debug(f"Added record to buffer: {buffer[stream][len(buffer[stream])-1]}") + + if message.type == Type.STATE: + # TODO: This is a queue from the source that we should save the buffer of records from message.type == Type.RECORD messages. See https://docs.airbyte.com/understanding-airbyte/airbyte-protocol#state--the-whole-sync + logger.warning(f"TODO: DUMP buffer with {len(buffer.items())} records into the GBT!") + yield message + pass + + def check(self, logger: logging.Logger, config: Mapping[str, Any]) -> AirbyteConnectionStatus: + """ + Tests if the input configuration can be used to successfully connect to the destination with the needed permissions + e.g: if a provided API token or password can be used to connect and write to the destination. + + :param logger: Logging object to display debug/info/error to the logs + (logs will not be accessible via airbyte UI if they are not passed to this logger) + :param config: Json object containing the configuration of this destination, content of this json is as specified in + the properties of the spec.json file + + :return: AirbyteConnectionStatus indicating a Success or Failure + """ + try: + # TODO + + return AirbyteConnectionStatus(status=Status.SUCCEEDED) + except Exception as e: + return AirbyteConnectionStatus(status=Status.FAILED, message=f"An exception occurred: {repr(e)}") diff --git a/airbyte-integrations/connectors/destination-glide/destination_glide/run.py b/airbyte-integrations/connectors/destination-glide/destination_glide/run.py new file mode 100644 index 0000000000000..a304cd3d6aab5 --- /dev/null +++ b/airbyte-integrations/connectors/destination-glide/destination_glide/run.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from .destination import DestinationGlide + +def run(): + destination = DestinationGlide() + destination.run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-glide/destination_glide/spec.json b/airbyte-integrations/connectors/destination-glide/destination_glide/spec.json new file mode 100644 index 0000000000000..ced8c50f41ce3 --- /dev/null +++ b/airbyte-integrations/connectors/destination-glide/destination_glide/spec.json @@ -0,0 +1,27 @@ +{ + "documentationUrl": "https://docs.airbyte.com/integrations/destinations/glide", + "supported_destination_sync_modes": ["overwrite"], + "supportsIncremental": true, + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Destination Glide", + "type": "object", + "required": ["api_host", "api_path_root", "api_key"], + "additionalProperties": false, + "properties": { + "api_host": { + "type": "string", + "description": "The host name of the Glide API destination" + }, + "api_path_root": { + "type": "string", + "description": "The path root of the Glide API destination" + }, + "api_key": { + "type": "string", + "description": "The user's key for the Glide API destination. Needs permission to write to the specified GBT.", + "airbyte_secret": true + } + } + } +} diff --git a/airbyte-integrations/connectors/destination-glide/dev-check.sh b/airbyte-integrations/connectors/destination-glide/dev-check.sh new file mode 100755 index 0000000000000..04498d8de5676 --- /dev/null +++ b/airbyte-integrations/connectors/destination-glide/dev-check.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +this_dir=$(cd $(dirname "$0"); pwd) # this script's directory +this_script=$(basename $0) + +poetry run destination-glide check --config secrets/config.json | jq diff --git a/airbyte-integrations/connectors/destination-glide/dev-spec.sh b/airbyte-integrations/connectors/destination-glide/dev-spec.sh new file mode 100755 index 0000000000000..4714c9da54872 --- /dev/null +++ b/airbyte-integrations/connectors/destination-glide/dev-spec.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +this_dir=$(cd $(dirname "$0"); pwd) # this script's directory +this_script=$(basename $0) + +poetry run destination-glide spec | jq diff --git a/airbyte-integrations/connectors/destination-glide/dev-write.sh b/airbyte-integrations/connectors/destination-glide/dev-write.sh new file mode 100755 index 0000000000000..a8c9732bd9685 --- /dev/null +++ b/airbyte-integrations/connectors/destination-glide/dev-write.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +this_dir=$(cd $(dirname "$0"); pwd) # this script's directory +this_script=$(basename $0) + +poetry run destination-glide write --config secrets/config.json --catalog sample_files/configured_catalog-test1.json | jq diff --git a/airbyte-integrations/connectors/destination-glide/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-glide/integration_tests/integration_test.py new file mode 100644 index 0000000000000..dadeb330414a2 --- /dev/null +++ b/airbyte-integrations/connectors/destination-glide/integration_tests/integration_test.py @@ -0,0 +1,110 @@ +# +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +# +from datetime import datetime +import json +import logging +import pytest +import random +import string +from typing import Any, Mapping + +from airbyte_cdk.models import ( + AirbyteMessage, + AirbyteRecordMessage, + AirbyteStateMessage, + AirbyteStream, + ConfiguredAirbyteCatalog, + ConfiguredAirbyteStream, + DestinationSyncMode, + Status, + SyncMode, + Type +) +from destination_glide import DestinationGlide + + +@pytest.fixture(name="config") +def config_fixture() -> Mapping[str, Any]: + with open("secrets/config.json", "r") as f: + return json.loads(f.read()) + + +@pytest.fixture(scope="module") +def test_table_name() -> str: + letters = string.ascii_lowercase + rand_string = "".join(random.choice(letters) for _ in range(10)) + return f"airbyte_integration_{rand_string}" + + +@pytest.fixture +def table_schema() -> str: + schema = {"type": "object", "properties": {"column1": {"type": ["null", "string"]}}} + return schema + +def AirbyteLogger() -> logging.Logger: + return logging.getLogger('airbyte') + +@pytest.fixture +def configured_catalog(test_table_name: str, table_schema: str) -> ConfiguredAirbyteCatalog: + overwrite_stream = ConfiguredAirbyteStream( + # TODO: I'm not sure if we should expect incoming streams SyncMode.incremental and only the destination to be full_refresh or they should + stream=AirbyteStream(name=test_table_name, json_schema=table_schema, supported_sync_modes=[SyncMode.incremental]), + sync_mode=SyncMode.incremental, + destination_sync_mode=DestinationSyncMode.overwrite, + ) + return ConfiguredAirbyteCatalog(streams=[overwrite_stream]) + +@pytest.fixture +def airbyte_message_record1(test_table_name: str): + return AirbyteMessage( + type=Type.RECORD, + record=AirbyteRecordMessage( + stream=test_table_name, data={"key_str": "value1", "key_int": 3}, emitted_at=int(datetime.now().timestamp()) * 1000 + ), + ) + + +@pytest.fixture +def airbyte_message_record2(test_table_name: str): + return AirbyteMessage( + type=Type.RECORD, + record=AirbyteRecordMessage( + stream=test_table_name, data={"key_str": "value2", "key_int": 2}, emitted_at=int(datetime.now().timestamp()) * 1000 + ), + ) + +@pytest.fixture +def airbyte_message_state(test_table_name: str): + return AirbyteMessage( + type=Type.STATE, + state=AirbyteStateMessage( + data={"opaque": "to destination"} + ) + ) + + +##### Tests Begin Here ##### +def test_check_valid_config(config: Mapping): + outcome = DestinationGlide().check(AirbyteLogger(), config) + assert outcome.status == Status.SUCCEEDED + +def test_write( + config: Mapping, + request, + configured_catalog: ConfiguredAirbyteCatalog, + airbyte_message_record1: AirbyteMessage, + airbyte_message_record2: AirbyteMessage, + airbyte_message_state: AirbyteMessage, + test_table_name: str, +): + destination = DestinationGlide() + generator = destination.write( + config=config, configured_catalog=configured_catalog, input_messages=[airbyte_message_record1, airbyte_message_record1, airbyte_message_state] + ) + + # expecting only to return the state message: + result = list(generator) + assert len(result) == 1 + assert False, "Validate that the data was written to mock API?" + diff --git a/airbyte-integrations/connectors/destination-glide/main.py b/airbyte-integrations/connectors/destination-glide/main.py new file mode 100644 index 0000000000000..59757e151d371 --- /dev/null +++ b/airbyte-integrations/connectors/destination-glide/main.py @@ -0,0 +1,11 @@ +# +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +# + + +import sys + +from destination_glide import DestinationGlide + +if __name__ == "__main__": + DestinationGlide().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-glide/metadata.yaml b/airbyte-integrations/connectors/destination-glide/metadata.yaml new file mode 100644 index 0000000000000..e668d783b3f07 --- /dev/null +++ b/airbyte-integrations/connectors/destination-glide/metadata.yaml @@ -0,0 +1,31 @@ +data: + allowedHosts: + hosts: + - TODO # Please change to the hostname of the source. + registries: + oss: + enabled: true + cloud: + enabled: false + connectorBuildOptions: + # Please update to the latest version of the connector base image. + # Please use the full address with sha256 hash to guarantee build reproducibility. + # https://hub.docker.com/r/airbyte/python-connector-base + baseImage: docker.io/airbyte/python-connector-base:1.2.2 + connectorSubtype: database + connectorType: destination + definitionId: 862fd86f-4d85-443b-8071-85bb8521806b + dockerImageTag: 0.1.0 + #dockerRepository: airbyte/destination-glide + dockerRepository: activescott/destination-glide + githubIssueLabel: destination-glide + icon: glide.svg + license: MIT + name: Glide + releaseDate: TODO + releaseStage: alpha + supportLevel: community + documentationUrl: https://docs.airbyte.com/integrations/destinations/glide + tags: + - language:python +metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-glide/poetry.lock b/airbyte-integrations/connectors/destination-glide/poetry.lock new file mode 100644 index 0000000000000..4cf6df29ae767 --- /dev/null +++ b/airbyte-integrations/connectors/destination-glide/poetry.lock @@ -0,0 +1,1302 @@ +# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. + +[[package]] +name = "airbyte-cdk" +version = "0.90.0" +description = "A framework for writing Airbyte Connectors." +optional = false +python-versions = "<4.0,>=3.9" +files = [ + {file = "airbyte_cdk-0.90.0-py3-none-any.whl", hash = "sha256:bd0aa5843cdc4901f2e482f0e86695ca4e6db83b65c5017799255dd20535cf56"}, + {file = "airbyte_cdk-0.90.0.tar.gz", hash = "sha256:25cefc010718bada5cce3f87e7ae93068630732c0d34ce5145f8ddf7457d4d3c"}, +] + +[package.dependencies] +airbyte-protocol-models = ">=0.9.0,<1.0" +backoff = "*" +cachetools = "*" +cryptography = ">=42.0.5,<43.0.0" +Deprecated = ">=1.2,<1.3" +dpath = ">=2.0.1,<2.1.0" +genson = "1.2.2" +isodate = ">=0.6.1,<0.7.0" +Jinja2 = ">=3.1.2,<3.2.0" +jsonref = ">=0.2,<0.3" +jsonschema = ">=3.2.0,<3.3.0" +langchain_core = "0.1.42" +pendulum = "<3.0.0" +pydantic = ">=1.10.8,<2.0.0" +pyjwt = ">=2.8.0,<3.0.0" +pyrate-limiter = ">=3.1.0,<3.2.0" +python-dateutil = "*" +pytz = "2024.1" +PyYAML = ">=6.0.1,<7.0.0" +requests = "*" +requests_cache = "*" +wcmatch = "8.4" + +[package.extras] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (>=15.0.0,<15.1.0)", "pytesseract (==0.3.10)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +sphinx-docs = ["Sphinx (>=4.2,<4.3)", "sphinx-rtd-theme (>=1.0,<1.1)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.1.16)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] + +[[package]] +name = "airbyte-protocol-models" +version = "0.12.1" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models-0.12.1-py3-none-any.whl", hash = "sha256:f8b8380e4a7573a8a6829d746d5a0c764831a9d411f8ea5a6bd503a39e120ebd"}, + {file = "airbyte_protocol_models-0.12.1.tar.gz", hash = "sha256:da070bf2b30fa406f8a2f559ba65c6d59affd7a8a7b3a02177bb03a52ef3ac3a"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "bracex" +version = "2.4" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, + {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, +] + +[[package]] +name = "cachetools" +version = "5.3.3" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.3-py3-none-any.whl", hash = "sha256:0abad1021d3f8325b2fc1d2e9c8b9c9d57b04c3932657a72465447332c24d945"}, + {file = "cachetools-5.3.3.tar.gz", hash = "sha256:ba29e2dfa0b8b556606f097407ed1aa62080ee108ab0dc5ec9d6a723a007d105"}, +] + +[[package]] +name = "cattrs" +version = "23.2.3" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2024.6.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.6.2-py3-none-any.whl", hash = "sha256:ddc6c8ce995e6987e7faf5e3f1b02b302836a0e5d98ece18392cb1a36c72ad56"}, + {file = "certifi-2024.6.2.tar.gz", hash = "sha256:3cd43f1c6fa7dedc5899d69d3ad0398fd018ad1a17fba83ddaf78aa46c747516"}, +] + +[[package]] +name = "cffi" +version = "1.16.0" +description = "Foreign Function Interface for Python calling C code." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cffi-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088"}, + {file = "cffi-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614"}, + {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743"}, + {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d"}, + {file = "cffi-1.16.0-cp310-cp310-win32.whl", hash = "sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a"}, + {file = "cffi-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1"}, + {file = "cffi-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404"}, + {file = "cffi-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e"}, + {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc"}, + {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb"}, + {file = "cffi-1.16.0-cp311-cp311-win32.whl", hash = "sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab"}, + {file = "cffi-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba"}, + {file = "cffi-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956"}, + {file = "cffi-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969"}, + {file = "cffi-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520"}, + {file = "cffi-1.16.0-cp312-cp312-win32.whl", hash = "sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b"}, + {file = "cffi-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235"}, + {file = "cffi-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324"}, + {file = "cffi-1.16.0-cp38-cp38-win32.whl", hash = "sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a"}, + {file = "cffi-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36"}, + {file = "cffi-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed"}, + {file = "cffi-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098"}, + {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000"}, + {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe"}, + {file = "cffi-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4"}, + {file = "cffi-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8"}, + {file = "cffi-1.16.0.tar.gz", hash = "sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0"}, +] + +[package.dependencies] +pycparser = "*" + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "cryptography" +version = "42.0.8" +description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +optional = false +python-versions = ">=3.7" +files = [ + {file = "cryptography-42.0.8-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:81d8a521705787afe7a18d5bfb47ea9d9cc068206270aad0b96a725022e18d2e"}, + {file = "cryptography-42.0.8-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:961e61cefdcb06e0c6d7e3a1b22ebe8b996eb2bf50614e89384be54c48c6b63d"}, + {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e3ec3672626e1b9e55afd0df6d774ff0e953452886e06e0f1eb7eb0c832e8902"}, + {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e599b53fd95357d92304510fb7bda8523ed1f79ca98dce2f43c115950aa78801"}, + {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:5226d5d21ab681f432a9c1cf8b658c0cb02533eece706b155e5fbd8a0cdd3949"}, + {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:6b7c4f03ce01afd3b76cf69a5455caa9cfa3de8c8f493e0d3ab7d20611c8dae9"}, + {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:2346b911eb349ab547076f47f2e035fc8ff2c02380a7cbbf8d87114fa0f1c583"}, + {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:ad803773e9df0b92e0a817d22fd8a3675493f690b96130a5e24f1b8fabbea9c7"}, + {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2f66d9cd9147ee495a8374a45ca445819f8929a3efcd2e3df6428e46c3cbb10b"}, + {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:d45b940883a03e19e944456a558b67a41160e367a719833c53de6911cabba2b7"}, + {file = "cryptography-42.0.8-cp37-abi3-win32.whl", hash = "sha256:a0c5b2b0585b6af82d7e385f55a8bc568abff8923af147ee3c07bd8b42cda8b2"}, + {file = "cryptography-42.0.8-cp37-abi3-win_amd64.whl", hash = "sha256:57080dee41209e556a9a4ce60d229244f7a66ef52750f813bfbe18959770cfba"}, + {file = "cryptography-42.0.8-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:dea567d1b0e8bc5764b9443858b673b734100c2871dc93163f58c46a97a83d28"}, + {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c4783183f7cb757b73b2ae9aed6599b96338eb957233c58ca8f49a49cc32fd5e"}, + {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0608251135d0e03111152e41f0cc2392d1e74e35703960d4190b2e0f4ca9c70"}, + {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:dc0fdf6787f37b1c6b08e6dfc892d9d068b5bdb671198c72072828b80bd5fe4c"}, + {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:9c0c1716c8447ee7dbf08d6db2e5c41c688544c61074b54fc4564196f55c25a7"}, + {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:fff12c88a672ab9c9c1cf7b0c80e3ad9e2ebd9d828d955c126be4fd3e5578c9e"}, + {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:cafb92b2bc622cd1aa6a1dce4b93307792633f4c5fe1f46c6b97cf67073ec961"}, + {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:31f721658a29331f895a5a54e7e82075554ccfb8b163a18719d342f5ffe5ecb1"}, + {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b297f90c5723d04bcc8265fc2a0f86d4ea2e0f7ab4b6994459548d3a6b992a14"}, + {file = "cryptography-42.0.8-cp39-abi3-win32.whl", hash = "sha256:2f88d197e66c65be5e42cd72e5c18afbfae3f741742070e3019ac8f4ac57262c"}, + {file = "cryptography-42.0.8-cp39-abi3-win_amd64.whl", hash = "sha256:fa76fbb7596cc5839320000cdd5d0955313696d9511debab7ee7278fc8b5c84a"}, + {file = "cryptography-42.0.8-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:ba4f0a211697362e89ad822e667d8d340b4d8d55fae72cdd619389fb5912eefe"}, + {file = "cryptography-42.0.8-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:81884c4d096c272f00aeb1f11cf62ccd39763581645b0812e99a91505fa48e0c"}, + {file = "cryptography-42.0.8-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c9bb2ae11bfbab395bdd072985abde58ea9860ed84e59dbc0463a5d0159f5b71"}, + {file = "cryptography-42.0.8-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:7016f837e15b0a1c119d27ecd89b3515f01f90a8615ed5e9427e30d9cdbfed3d"}, + {file = "cryptography-42.0.8-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5a94eccb2a81a309806027e1670a358b99b8fe8bfe9f8d329f27d72c094dde8c"}, + {file = "cryptography-42.0.8-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:dec9b018df185f08483f294cae6ccac29e7a6e0678996587363dc352dc65c842"}, + {file = "cryptography-42.0.8-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:343728aac38decfdeecf55ecab3264b015be68fc2816ca800db649607aeee648"}, + {file = "cryptography-42.0.8-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:013629ae70b40af70c9a7a5db40abe5d9054e6f4380e50ce769947b73bf3caad"}, + {file = "cryptography-42.0.8.tar.gz", hash = "sha256:8d09d05439ce7baa8e9e95b07ec5b6c886f548deb7e0f69ef25f64b3bce842f2"}, +] + +[package.dependencies] +cffi = {version = ">=1.12", markers = "platform_python_implementation != \"PyPy\""} + +[package.extras] +docs = ["sphinx (>=5.3.0)", "sphinx-rtd-theme (>=1.1.1)"] +docstest = ["pyenchant (>=1.6.11)", "readme-renderer", "sphinxcontrib-spelling (>=4.0.1)"] +nox = ["nox"] +pep8test = ["check-sdist", "click", "mypy", "ruff"] +sdist = ["build"] +ssh = ["bcrypt (>=3.1.5)"] +test = ["certifi", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"] +test-randomorder = ["pytest-randomly"] + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "dpath" +version = "2.0.8" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, + {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.1" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.1-py3-none-any.whl", hash = "sha256:5258b9ed329c5bbdd31a309f53cbfb0b155341807f6ff7606a1e801a891b29ad"}, + {file = "exceptiongroup-1.2.1.tar.gz", hash = "sha256:a4785e48b045528f5bfe627b6ad554ff32def154f42372786903b7abcfe1aa16"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "idna" +version = "3.7" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.7-py3-none-any.whl", hash = "sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0"}, + {file = "idna-3.7.tar.gz", hash = "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.4" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "jinja2-3.1.4-py3-none-any.whl", hash = "sha256:bc5dd2abb727a5319567b7a813e6a2e7318c39f4f487cfe6c89c6f9c7d25197d"}, + {file = "jinja2-3.1.4.tar.gz", hash = "sha256:4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jsonpatch" +version = "1.33" +description = "Apply JSON-Patches (RFC 6902)" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*" +files = [ + {file = "jsonpatch-1.33-py2.py3-none-any.whl", hash = "sha256:0ae28c0cd062bbd8b8ecc26d7d164fbbea9652a1a3693f3b956c1eae5145dade"}, + {file = "jsonpatch-1.33.tar.gz", hash = "sha256:9fcd4009c41e6d12348b4a0ff2563ba56a2923a7dfee731d004e212e1ee5030c"}, +] + +[package.dependencies] +jsonpointer = ">=1.9" + +[[package]] +name = "jsonpointer" +version = "3.0.0" +description = "Identify specific nodes in a JSON document (RFC 6901)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "jsonpointer-3.0.0-py2.py3-none-any.whl", hash = "sha256:13e088adc14fca8b6aa8177c044e12701e6ad4b28ff10e65f2267a90109c9942"}, + {file = "jsonpointer-3.0.0.tar.gz", hash = "sha256:2b2d729f2091522d61c3b31f82e11870f60b68f43fbc705cb76bf4b832af59ef"}, +] + +[[package]] +name = "jsonref" +version = "0.2" +description = "An implementation of JSON Reference for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonref-0.2-py3-none-any.whl", hash = "sha256:b1e82fa0b62e2c2796a13e5401fe51790b248f6d9bf9d7212a3e31a3501b291f"}, + {file = "jsonref-0.2.tar.gz", hash = "sha256:f3c45b121cf6257eafabdc3a8008763aed1cd7da06dbabc59a9e4d2a5e4e6697"}, +] + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "langchain-core" +version = "0.1.42" +description = "Building applications with LLMs through composability" +optional = false +python-versions = "<4.0,>=3.8.1" +files = [ + {file = "langchain_core-0.1.42-py3-none-any.whl", hash = "sha256:c5653ffa08a44f740295c157a24c0def4a753333f6a2c41f76bf431cd00be8b5"}, + {file = "langchain_core-0.1.42.tar.gz", hash = "sha256:40751bf60ea5d8e2b2efe65290db434717ee3834870c002e40e2811f09d814e6"}, +] + +[package.dependencies] +jsonpatch = ">=1.33,<2.0" +langsmith = ">=0.1.0,<0.2.0" +packaging = ">=23.2,<24.0" +pydantic = ">=1,<3" +PyYAML = ">=5.3" +tenacity = ">=8.1.0,<9.0.0" + +[package.extras] +extended-testing = ["jinja2 (>=3,<4)"] + +[[package]] +name = "langsmith" +version = "0.1.79" +description = "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform." +optional = false +python-versions = "<4.0,>=3.8.1" +files = [ + {file = "langsmith-0.1.79-py3-none-any.whl", hash = "sha256:c7f2c23981917713b5515b773f37c84ff68a7adf803476e2ebb5adcb36a04202"}, + {file = "langsmith-0.1.79.tar.gz", hash = "sha256:d215718cfdcdf4a011126b7a3d4a37eee96d887e59ac1e628a57e24b2bfa3163"}, +] + +[package.dependencies] +orjson = ">=3.9.14,<4.0.0" +pydantic = ">=1,<3" +requests = ">=2,<3" + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "orjson" +version = "3.10.5" +description = "Fast, correct Python JSON library supporting dataclasses, datetimes, and numpy" +optional = false +python-versions = ">=3.8" +files = [ + {file = "orjson-3.10.5-cp310-cp310-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:545d493c1f560d5ccfc134803ceb8955a14c3fcb47bbb4b2fee0232646d0b932"}, + {file = "orjson-3.10.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f4324929c2dd917598212bfd554757feca3e5e0fa60da08be11b4aa8b90013c1"}, + {file = "orjson-3.10.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8c13ca5e2ddded0ce6a927ea5a9f27cae77eee4c75547b4297252cb20c4d30e6"}, + {file = "orjson-3.10.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b6c8e30adfa52c025f042a87f450a6b9ea29649d828e0fec4858ed5e6caecf63"}, + {file = "orjson-3.10.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:338fd4f071b242f26e9ca802f443edc588fa4ab60bfa81f38beaedf42eda226c"}, + {file = "orjson-3.10.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6970ed7a3126cfed873c5d21ece1cd5d6f83ca6c9afb71bbae21a0b034588d96"}, + {file = "orjson-3.10.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:235dadefb793ad12f7fa11e98a480db1f7c6469ff9e3da5e73c7809c700d746b"}, + {file = "orjson-3.10.5-cp310-none-win32.whl", hash = "sha256:be79e2393679eda6a590638abda16d167754393f5d0850dcbca2d0c3735cebe2"}, + {file = "orjson-3.10.5-cp310-none-win_amd64.whl", hash = "sha256:c4a65310ccb5c9910c47b078ba78e2787cb3878cdded1702ac3d0da71ddc5228"}, + {file = "orjson-3.10.5-cp311-cp311-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:cdf7365063e80899ae3a697def1277c17a7df7ccfc979990a403dfe77bb54d40"}, + {file = "orjson-3.10.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b68742c469745d0e6ca5724506858f75e2f1e5b59a4315861f9e2b1df77775a"}, + {file = "orjson-3.10.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7d10cc1b594951522e35a3463da19e899abe6ca95f3c84c69e9e901e0bd93d38"}, + {file = "orjson-3.10.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dcbe82b35d1ac43b0d84072408330fd3295c2896973112d495e7234f7e3da2e1"}, + {file = "orjson-3.10.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10c0eb7e0c75e1e486c7563fe231b40fdd658a035ae125c6ba651ca3b07936f5"}, + {file = "orjson-3.10.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:53ed1c879b10de56f35daf06dbc4a0d9a5db98f6ee853c2dbd3ee9d13e6f302f"}, + {file = "orjson-3.10.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:099e81a5975237fda3100f918839af95f42f981447ba8f47adb7b6a3cdb078fa"}, + {file = "orjson-3.10.5-cp311-none-win32.whl", hash = "sha256:1146bf85ea37ac421594107195db8bc77104f74bc83e8ee21a2e58596bfb2f04"}, + {file = "orjson-3.10.5-cp311-none-win_amd64.whl", hash = "sha256:36a10f43c5f3a55c2f680efe07aa93ef4a342d2960dd2b1b7ea2dd764fe4a37c"}, + {file = "orjson-3.10.5-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:68f85ecae7af14a585a563ac741b0547a3f291de81cd1e20903e79f25170458f"}, + {file = "orjson-3.10.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28afa96f496474ce60d3340fe8d9a263aa93ea01201cd2bad844c45cd21f5268"}, + {file = "orjson-3.10.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9cd684927af3e11b6e754df80b9ffafd9fb6adcaa9d3e8fdd5891be5a5cad51e"}, + {file = "orjson-3.10.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3d21b9983da032505f7050795e98b5d9eee0df903258951566ecc358f6696969"}, + {file = "orjson-3.10.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ad1de7fef79736dde8c3554e75361ec351158a906d747bd901a52a5c9c8d24b"}, + {file = "orjson-3.10.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2d97531cdfe9bdd76d492e69800afd97e5930cb0da6a825646667b2c6c6c0211"}, + {file = "orjson-3.10.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d69858c32f09c3e1ce44b617b3ebba1aba030e777000ebdf72b0d8e365d0b2b3"}, + {file = "orjson-3.10.5-cp312-none-win32.whl", hash = "sha256:64c9cc089f127e5875901ac05e5c25aa13cfa5dbbbd9602bda51e5c611d6e3e2"}, + {file = "orjson-3.10.5-cp312-none-win_amd64.whl", hash = "sha256:b2efbd67feff8c1f7728937c0d7f6ca8c25ec81373dc8db4ef394c1d93d13dc5"}, + {file = "orjson-3.10.5-cp38-cp38-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:03b565c3b93f5d6e001db48b747d31ea3819b89abf041ee10ac6988886d18e01"}, + {file = "orjson-3.10.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:584c902ec19ab7928fd5add1783c909094cc53f31ac7acfada817b0847975f26"}, + {file = "orjson-3.10.5-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5a35455cc0b0b3a1eaf67224035f5388591ec72b9b6136d66b49a553ce9eb1e6"}, + {file = "orjson-3.10.5-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1670fe88b116c2745a3a30b0f099b699a02bb3482c2591514baf5433819e4f4d"}, + {file = "orjson-3.10.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:185c394ef45b18b9a7d8e8f333606e2e8194a50c6e3c664215aae8cf42c5385e"}, + {file = "orjson-3.10.5-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:ca0b3a94ac8d3886c9581b9f9de3ce858263865fdaa383fbc31c310b9eac07c9"}, + {file = "orjson-3.10.5-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:dfc91d4720d48e2a709e9c368d5125b4b5899dced34b5400c3837dadc7d6271b"}, + {file = "orjson-3.10.5-cp38-none-win32.whl", hash = "sha256:c05f16701ab2a4ca146d0bca950af254cb7c02f3c01fca8efbbad82d23b3d9d4"}, + {file = "orjson-3.10.5-cp38-none-win_amd64.whl", hash = "sha256:8a11d459338f96a9aa7f232ba95679fc0c7cedbd1b990d736467894210205c09"}, + {file = "orjson-3.10.5-cp39-cp39-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:85c89131d7b3218db1b24c4abecea92fd6c7f9fab87441cfc342d3acc725d807"}, + {file = "orjson-3.10.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb66215277a230c456f9038d5e2d84778141643207f85336ef8d2a9da26bd7ca"}, + {file = "orjson-3.10.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:51bbcdea96cdefa4a9b4461e690c75ad4e33796530d182bdd5c38980202c134a"}, + {file = "orjson-3.10.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dbead71dbe65f959b7bd8cf91e0e11d5338033eba34c114f69078d59827ee139"}, + {file = "orjson-3.10.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5df58d206e78c40da118a8c14fc189207fffdcb1f21b3b4c9c0c18e839b5a214"}, + {file = "orjson-3.10.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:c4057c3b511bb8aef605616bd3f1f002a697c7e4da6adf095ca5b84c0fd43595"}, + {file = "orjson-3.10.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:b39e006b00c57125ab974362e740c14a0c6a66ff695bff44615dcf4a70ce2b86"}, + {file = "orjson-3.10.5-cp39-none-win32.whl", hash = "sha256:eded5138cc565a9d618e111c6d5c2547bbdd951114eb822f7f6309e04db0fb47"}, + {file = "orjson-3.10.5-cp39-none-win_amd64.whl", hash = "sha256:cc28e90a7cae7fcba2493953cff61da5a52950e78dc2dacfe931a317ee3d8de7"}, + {file = "orjson-3.10.5.tar.gz", hash = "sha256:7a5baef8a4284405d96c90c7c62b755e9ef1ada84c2406c24a9ebec86b89f46d"}, +] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pendulum" +version = "2.1.2" +description = "Python datetimes made easy" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, + {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, + {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, + {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, + {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, + {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, + {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, + {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, + {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, + {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, + {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, +] + +[package.dependencies] +python-dateutil = ">=2.6,<3.0" +pytzdata = ">=2020.1" + +[[package]] +name = "platformdirs" +version = "4.2.2" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.2-py3-none-any.whl", hash = "sha256:2d7a1657e36a80ea911db832a8a6ece5ee53d8de21edd5cc5879af6530b1bfee"}, + {file = "platformdirs-4.2.2.tar.gz", hash = "sha256:38b7b51f512eed9e84a22788b4bce1de17c0adb134d6becb09836e37d8654cd3"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] +type = ["mypy (>=1.8)"] + +[[package]] +name = "pluggy" +version = "1.5.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"}, + {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "pycparser" +version = "2.22" +description = "C parser in Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc"}, + {file = "pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6"}, +] + +[[package]] +name = "pydantic" +version = "1.10.16" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.16-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1a539ac40551b01a85e899829aa43ca8036707474af8d74b48be288d4d2d2846"}, + {file = "pydantic-1.10.16-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8a4fcc7b0b8038dbda2dda642cff024032dfae24a7960cc58e57a39eb1949b9b"}, + {file = "pydantic-1.10.16-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4660dd697de1ae2d4305a85161312611f64d5360663a9ba026cd6ad9e3fe14c3"}, + {file = "pydantic-1.10.16-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:900a787c574f903a97d0bf52a43ff3b6cf4fa0119674bcfc0e5fd1056d388ad9"}, + {file = "pydantic-1.10.16-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:d30192a63e6d3334c3f0c0506dd6ae9f1dce7b2f8845518915291393a5707a22"}, + {file = "pydantic-1.10.16-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:16cf23ed599ca5ca937e37ba50ab114e6b5c387eb43a6cc533701605ad1be611"}, + {file = "pydantic-1.10.16-cp310-cp310-win_amd64.whl", hash = "sha256:8d23111f41d1e19334edd51438fd57933f3eee7d9d2fa8cc3f5eda515a272055"}, + {file = "pydantic-1.10.16-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ef287b8d7fc0e86a8bd1f902c61aff6ba9479c50563242fe88ba39692e98e1e0"}, + {file = "pydantic-1.10.16-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b9ded699bfd3b3912d796ff388b0c607e6d35d41053d37aaf8fd6082c660de9a"}, + {file = "pydantic-1.10.16-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:daeb199814333e4426c5e86d7fb610f4e230289f28cab90eb4de27330bef93cf"}, + {file = "pydantic-1.10.16-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5973843f1fa99ec6c3ac8d1a8698ac9340b35e45cca6c3e5beb5c3bd1ef15de6"}, + {file = "pydantic-1.10.16-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c6b8a7788a8528a558828fe4a48783cafdcf2612d13c491594a8161dc721629c"}, + {file = "pydantic-1.10.16-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:8abaecf54dacc9d991dda93c3b880d41092a8924cde94eeb811d7d9ab55df7d8"}, + {file = "pydantic-1.10.16-cp311-cp311-win_amd64.whl", hash = "sha256:ddc7b682fbd23f051edc419dc6977e11dd2dbdd0cef9d05f0e15d1387862d230"}, + {file = "pydantic-1.10.16-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:067c2b5539f7839653ad8c3d1fc2f1343338da8677b7b2172abf3cd3fdc8f719"}, + {file = "pydantic-1.10.16-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4d1fc943583c046ecad0ff5d6281ee571b64e11b5503d9595febdce54f38b290"}, + {file = "pydantic-1.10.16-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:18548b30ccebe71d380b0886cc44ea5d80afbcc155e3518792f13677ad06097d"}, + {file = "pydantic-1.10.16-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:4e92292f9580fc5ea517618580fac24e9f6dc5657196e977c194a8e50e14f5a9"}, + {file = "pydantic-1.10.16-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:5da8bc4bb4f85b8c97cc7f11141fddbbd29eb25e843672e5807e19cc3d7c1b7f"}, + {file = "pydantic-1.10.16-cp37-cp37m-win_amd64.whl", hash = "sha256:a04ee1ea34172b87707a6ecfcdb120d7656892206b7c4dbdb771a73e90179fcb"}, + {file = "pydantic-1.10.16-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:4fa86469fd46e732242c7acb83282d33f83591a7e06f840481327d5bf6d96112"}, + {file = "pydantic-1.10.16-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:89c2783dc261726fe7a5ce1121bce29a2f7eb9b1e704c68df2b117604e3b346f"}, + {file = "pydantic-1.10.16-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:78e59fa919fa7a192f423d190d8660c35dd444efa9216662273f36826765424b"}, + {file = "pydantic-1.10.16-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b7e82a80068c77f4b074032e031e642530b6d45cb8121fc7c99faa31fb6c6b72"}, + {file = "pydantic-1.10.16-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:d82d5956cee27a30e26a5b88d00a6a2a15a4855e13c9baf50175976de0dc282c"}, + {file = "pydantic-1.10.16-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4b7b99424cc0970ff08deccb549b5a6ec1040c0b449eab91723e64df2bd8fdca"}, + {file = "pydantic-1.10.16-cp38-cp38-win_amd64.whl", hash = "sha256:d97a35e1ba59442775201657171f601a2879e63517a55862a51f8d67cdfc0017"}, + {file = "pydantic-1.10.16-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9d91f6866fd3e303c632207813ef6bc4d86055e21c5e5a0a311983a9ac5f0192"}, + {file = "pydantic-1.10.16-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d8d3c71d14c8bd26d2350c081908dbf59d5a6a8f9596d9ef2b09cc1e61c8662b"}, + {file = "pydantic-1.10.16-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b73e6386b439b4881d79244e9fc1e32d1e31e8d784673f5d58a000550c94a6c0"}, + {file = "pydantic-1.10.16-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5f039881fb2ef86f6de6eacce6e71701b47500355738367413ccc1550b2a69cf"}, + {file = "pydantic-1.10.16-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:3895ddb26f22bdddee7e49741486aa7b389258c6f6771943e87fc00eabd79134"}, + {file = "pydantic-1.10.16-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:55b945da2756b5cef93d792521ad0d457fdf2f69fd5a2d10a27513f5281717dd"}, + {file = "pydantic-1.10.16-cp39-cp39-win_amd64.whl", hash = "sha256:22dd265c77c3976a34be78409b128cb84629284dfd1b69d2fa1507a36f84dc8b"}, + {file = "pydantic-1.10.16-py3-none-any.whl", hash = "sha256:aa2774ba5412fd1c5cb890d08e8b0a3bb5765898913ba1f61a65a4810f03cf29"}, + {file = "pydantic-1.10.16.tar.gz", hash = "sha256:8bb388f6244809af69ee384900b10b677a69f1980fdc655ea419710cffcb5610"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyjwt" +version = "2.8.0" +description = "JSON Web Token implementation in Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "PyJWT-2.8.0-py3-none-any.whl", hash = "sha256:59127c392cc44c2da5bb3192169a91f429924e17aff6534d70fdc02ab3e04320"}, + {file = "PyJWT-2.8.0.tar.gz", hash = "sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de"}, +] + +[package.extras] +crypto = ["cryptography (>=3.4.0)"] +dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.4.0)", "pre-commit", "pytest (>=6.0.0,<7.0.0)", "sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] +docs = ["sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] +tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] + +[[package]] +name = "pyrate-limiter" +version = "3.1.1" +description = "Python Rate-Limiter using Leaky-Bucket Algorithm" +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "pyrate_limiter-3.1.1-py3-none-any.whl", hash = "sha256:c51906f1d51d56dc992ff6c26e8300e32151bc6cfa3e6559792e31971dfd4e2b"}, + {file = "pyrate_limiter-3.1.1.tar.gz", hash = "sha256:2f57eda712687e6eccddf6afe8f8a15b409b97ed675fe64a626058f12863b7b7"}, +] + +[package.extras] +all = ["filelock (>=3.0)", "redis (>=5.0.0,<6.0.0)"] +docs = ["furo (>=2022.3.4,<2023.0.0)", "myst-parser (>=0.17)", "sphinx (>=4.3.0,<5.0.0)", "sphinx-autodoc-typehints (>=1.17,<2.0)", "sphinx-copybutton (>=0.5)", "sphinxcontrib-apidoc (>=0.3,<0.4)"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + +[[package]] +name = "pytest" +version = "8.2.2" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-8.2.2-py3-none-any.whl", hash = "sha256:c434598117762e2bd304e526244f67bf66bbd7b5d6cf22138be51ff661980343"}, + {file = "pytest-8.2.2.tar.gz", hash = "sha256:de4bb8104e201939ccdc688b27a89a7be2079b22e2bd2b07f806b6ba71117977"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=1.5,<2.0" +tomli = {version = ">=1", markers = "python_version < \"3.11\""} + +[package.extras] +dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] + +[[package]] +name = "pytest-mock" +version = "3.14.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.14.0.tar.gz", hash = "sha256:2719255a1efeceadbc056d6bf3df3d1c5015530fb40cf347c0f9afac88410bd0"}, + {file = "pytest_mock-3.14.0-py3-none-any.whl", hash = "sha256:0b72c38033392a5f4621342fe11e9219ac11ec9d375f8e2a0c164539e0d70f6f"}, +] + +[package.dependencies] +pytest = ">=6.2.5" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, + {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytz" +version = "2024.1" +description = "World timezone definitions, modern and historical" +optional = false +python-versions = "*" +files = [ + {file = "pytz-2024.1-py2.py3-none-any.whl", hash = "sha256:328171f4e3623139da4983451950b28e95ac706e13f3f2630a879749e7a8b319"}, + {file = "pytz-2024.1.tar.gz", hash = "sha256:2a29735ea9c18baf14b448846bde5a48030ed267578472d8955cd0e7443a9812"}, +] + +[[package]] +name = "pytzdata" +version = "2020.1" +description = "The Olson timezone database for Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, + {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.32.3" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.8" +files = [ + {file = "requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6"}, + {file = "requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-cache" +version = "1.2.0" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.8" +files = [ + {file = "requests_cache-1.2.0-py3-none-any.whl", hash = "sha256:490324301bf0cb924ff4e6324bd2613453e7e1f847353928b08adb0fdfb7f722"}, + {file = "requests_cache-1.2.0.tar.gz", hash = "sha256:db1c709ca343cc1cd5b6c8b1a5387298eceed02306a6040760db538c885e3838"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=6.0.1)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.9)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=6.0.1)"] + +[[package]] +name = "requests-mock" +version = "1.12.1" +description = "Mock out responses from the requests package" +optional = false +python-versions = ">=3.5" +files = [ + {file = "requests-mock-1.12.1.tar.gz", hash = "sha256:e9e12e333b525156e82a3c852f22016b9158220d2f47454de9cae8a77d371401"}, + {file = "requests_mock-1.12.1-py2.py3-none-any.whl", hash = "sha256:b1e37054004cdd5e56c84454cc7df12b25f90f382159087f4b6915aaeef39563"}, +] + +[package.dependencies] +requests = ">=2.22,<3" + +[package.extras] +fixture = ["fixtures"] + +[[package]] +name = "setuptools" +version = "70.0.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-70.0.0-py3-none-any.whl", hash = "sha256:54faa7f2e8d2d11bcd2c07bed282eef1046b5c080d1c32add737d7b5817b1ad4"}, + {file = "setuptools-70.0.0.tar.gz", hash = "sha256:f211a66637b8fa059bb28183da127d4e86396c991a942b028c6650d4319c3fd0"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mypy (==1.9)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.1)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "tenacity" +version = "8.4.1" +description = "Retry code until it succeeds" +optional = false +python-versions = ">=3.8" +files = [ + {file = "tenacity-8.4.1-py3-none-any.whl", hash = "sha256:28522e692eda3e1b8f5e99c51464efcc0b9fc86933da92415168bc1c4e2308fa"}, + {file = "tenacity-8.4.1.tar.gz", hash = "sha256:54b1412b878ddf7e1f1577cd49527bad8cdef32421bd599beac0c6c3f10582fd"}, +] + +[package.extras] +doc = ["reno", "sphinx"] +test = ["pytest", "tornado (>=4.5)", "typeguard"] + +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.12.2" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, + {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, +] + +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "urllib3" +version = "2.2.2" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.2-py3-none-any.whl", hash = "sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472"}, + {file = "urllib3-2.2.2.tar.gz", hash = "sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "wcmatch" +version = "8.4" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, + {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9,<3.12" +content-hash = "acd5908c82765b55ec5859799db1bcbb616d044db689a3ba94346d8b1d2f9b5c" diff --git a/airbyte-integrations/connectors/destination-glide/pyproject.toml b/airbyte-integrations/connectors/destination-glide/pyproject.toml new file mode 100644 index 0000000000000..b47059962cf2c --- /dev/null +++ b/airbyte-integrations/connectors/destination-glide/pyproject.toml @@ -0,0 +1,28 @@ +[build-system] +requires = [ "poetry-core>=1.0.0",] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +version = "0.1.0" +name = "destination-glide" +description = "Destination implementation for glide." +authors = [ "Airbyte ",] +license = "MIT" +readme = "README.md" +documentation = "https://docs.airbyte.com/integrations/destinations/glide" +homepage = "https://airbyte.com" +repository = "https://github.com/airbytehq/airbyte" +packages = [ { include = "destination_glide" }, {include = "main.py" } ] + +[tool.poetry.dependencies] +python = "^3.9,<3.12" +airbyte-cdk = "^0" + +[tool.poetry.scripts] +destination-glide = "destination_glide.run:run" + +[tool.poetry.group.dev.dependencies] +requests-mock = "*" +pytest-mock = "*" +pytest = "*" + diff --git a/airbyte-integrations/connectors/destination-glide/sample_files/configured_catalog-test1.json b/airbyte-integrations/connectors/destination-glide/sample_files/configured_catalog-test1.json new file mode 100644 index 0000000000000..9ac002e358d35 --- /dev/null +++ b/airbyte-integrations/connectors/destination-glide/sample_files/configured_catalog-test1.json @@ -0,0 +1,27 @@ +{ + "streams": [ + { + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite", + "stream": { + "name": "ab-airbyte-testing", + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": false, + "json_schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "body": { + "type": "string" + }, + "attributes": { + "type": ["null", "object"] + } + } + } + } + } + ] +} diff --git a/airbyte-integrations/connectors/destination-glide/scripts/build-docker-image.sh b/airbyte-integrations/connectors/destination-glide/scripts/build-docker-image.sh new file mode 100755 index 0000000000000..bdebc21ea7412 --- /dev/null +++ b/airbyte-integrations/connectors/destination-glide/scripts/build-docker-image.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +this_dir=$(cd $(dirname "$0"); pwd) # this script's directory +this_script=$(basename $0) + +airbyte-ci connectors --name=destination-glide build + diff --git a/airbyte-integrations/connectors/destination-glide/scripts/dev-check.sh b/airbyte-integrations/connectors/destination-glide/scripts/dev-check.sh new file mode 100755 index 0000000000000..04498d8de5676 --- /dev/null +++ b/airbyte-integrations/connectors/destination-glide/scripts/dev-check.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +this_dir=$(cd $(dirname "$0"); pwd) # this script's directory +this_script=$(basename $0) + +poetry run destination-glide check --config secrets/config.json | jq diff --git a/airbyte-integrations/connectors/destination-glide/scripts/dev-spec.sh b/airbyte-integrations/connectors/destination-glide/scripts/dev-spec.sh new file mode 100755 index 0000000000000..4714c9da54872 --- /dev/null +++ b/airbyte-integrations/connectors/destination-glide/scripts/dev-spec.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +this_dir=$(cd $(dirname "$0"); pwd) # this script's directory +this_script=$(basename $0) + +poetry run destination-glide spec | jq diff --git a/airbyte-integrations/connectors/destination-glide/scripts/dev-write.sh b/airbyte-integrations/connectors/destination-glide/scripts/dev-write.sh new file mode 100755 index 0000000000000..a8c9732bd9685 --- /dev/null +++ b/airbyte-integrations/connectors/destination-glide/scripts/dev-write.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +this_dir=$(cd $(dirname "$0"); pwd) # this script's directory +this_script=$(basename $0) + +poetry run destination-glide write --config secrets/config.json --catalog sample_files/configured_catalog-test1.json | jq diff --git a/airbyte-integrations/connectors/destination-glide/scripts/push-docker-image.sh b/airbyte-integrations/connectors/destination-glide/scripts/push-docker-image.sh new file mode 100755 index 0000000000000..a12c4897009de --- /dev/null +++ b/airbyte-integrations/connectors/destination-glide/scripts/push-docker-image.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +this_dir=$(cd $(dirname "$0"); pwd) # this script's directory +this_script=$(basename $0) + + +docker image tag airbyte/destination-glide:dev activescott/destination-glide:dev + +docker push activescott/destination-glide:dev + diff --git a/airbyte-integrations/connectors/destination-glide/scripts/test-integration.sh b/airbyte-integrations/connectors/destination-glide/scripts/test-integration.sh new file mode 100755 index 0000000000000..b2c0b792415ab --- /dev/null +++ b/airbyte-integrations/connectors/destination-glide/scripts/test-integration.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +this_dir=$(cd $(dirname "$0"); pwd) # this script's directory +this_script=$(basename $0) + +poetry run pytest integration_tests diff --git a/airbyte-integrations/connectors/destination-glide/test-integration.sh b/airbyte-integrations/connectors/destination-glide/test-integration.sh new file mode 100755 index 0000000000000..b2c0b792415ab --- /dev/null +++ b/airbyte-integrations/connectors/destination-glide/test-integration.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +this_dir=$(cd $(dirname "$0"); pwd) # this script's directory +this_script=$(basename $0) + +poetry run pytest integration_tests diff --git a/airbyte-integrations/connectors/destination-glide/unit_tests/unit_test.py b/airbyte-integrations/connectors/destination-glide/unit_tests/unit_test.py new file mode 100644 index 0000000000000..8da866052b812 --- /dev/null +++ b/airbyte-integrations/connectors/destination-glide/unit_tests/unit_test.py @@ -0,0 +1,7 @@ +# +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +# + + +def test_example_method(): + assert True From 7f1649f85c08e0b9233621eb409d25406847788c Mon Sep 17 00:00:00 2001 From: Scott Willeke Date: Tue, 18 Jun 2024 18:57:09 -0400 Subject: [PATCH 02/39] wip: glide api client --- .../destination_glide/destination.py | 69 +++++++---- .../destination_glide/glide.py | 117 ++++++++++++++++++ .../destination_glide/log.py | 21 ++++ .../destination_glide/spec.json | 5 + .../connectors/destination-glide/dev-check.sh | 5 - .../connectors/destination-glide/dev-spec.sh | 5 - .../connectors/destination-glide/dev-write.sh | 5 - .../integration_tests/integration_test.py | 56 ++++++--- .../destination-glide/scripts/dev-check.sh | 1 + .../destination-glide/scripts/dev-write.sh | 2 +- 10 files changed, 225 insertions(+), 61 deletions(-) create mode 100644 airbyte-integrations/connectors/destination-glide/destination_glide/glide.py create mode 100644 airbyte-integrations/connectors/destination-glide/destination_glide/log.py delete mode 100755 airbyte-integrations/connectors/destination-glide/dev-check.sh delete mode 100755 airbyte-integrations/connectors/destination-glide/dev-spec.sh delete mode 100755 airbyte-integrations/connectors/destination-glide/dev-write.sh diff --git a/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py b/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py index 274adfce484b5..96b2b7ed85ce9 100644 --- a/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py +++ b/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py @@ -12,35 +12,28 @@ ) from collections import defaultdict import datetime + +from .glide import GlideBigTable import json +from .log import getLogger import logging +import requests from typing import Any, Iterable, Mapping import uuid - -# Create a logger -logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) - -# Create a file handler -# TODO REMOVE? -handler = logging.FileHandler('destination-glide.log') -handler.setLevel(logging.DEBUG) - -# Create a logging format -formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') -handler.setFormatter(formatter) - -# Add the handlers to the logger -logger.addHandler(handler) +logger = getLogger() class DestinationGlide(Destination): + # GlideBigTable optional for tests to inject mock + def __init__(self, glide: GlideBigTable = None): + if glide is None: + glide = GlideBigTable() + self.glide = glide + def write( self, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog, input_messages: Iterable[AirbyteMessage] ) -> Iterable[AirbyteMessage]: - """ - TODO Reads the input stream of messages, config, and catalog to write data to the destination. This method returns an iterable (typically a generator of AirbyteMessages via yield) containing state messages received @@ -54,35 +47,59 @@ def write( :param input_messages: The stream of input messages received from the source :return: Iterable of AirbyteStateMessages wrapped in AirbyteMessage structs """ + # get config: + api_host = config['api_host'] + api_path_root = config['api_path_root'] + api_key = config['api_key'] + table_id = config['table_id'] + # TODO: hardcoded for now using old api + app_id = "Ix9CEuP6DiFugfjhSG5t" + + self.glide.init(api_host, api_key, api_path_root, app_id, table_id) + # go through each stream and add it as needed: stream_names = {s.stream.name for s in configured_catalog.streams} for configured_stream in configured_catalog.streams: if configured_stream.destination_sync_mode != DestinationSyncMode.overwrite: - raise Exception(f"Only destination sync mode overwrite it supported, but received '{configured_stream.destination_sync_mode}'.") + raise Exception(f'Only destination sync mode overwrite it supported, but received "{configured_stream.destination_sync_mode}".') # nopep8 because https://github.com/hhatto/autopep8/issues/712 + # TODO: create a new GBT to prepare for dumping the data into it - table_name = f"_bgt_{configured_stream.stream.name}" + # NOTE: for now using an existing GBT in old API + logger.debug("deleting all rows...") + self.glide.delete_all() + logger.debug("deleting all rows complete.") # stream the records into the GBT: buffer = defaultdict(list) + logger.debug(f"buffer created.") for message in input_messages: + logger.debug(f"processing message {message.type}...") if message.type == Type.RECORD: + logger.debug("buffering record...") data = message.record.data stream = message.record.stream if stream not in stream_names: - logger.debug(f"Stream {stream} was not present in configured streams, skipping") + logger.debug( + f"Stream {stream} was not present in configured streams, skipping") continue - + # TODO: Check the columns match the columns that we saw in configured_catalog per https://docs.airbyte.com/understanding-airbyte/airbyte-protocol#destination # add to buffer record_id = str(uuid.uuid4()) - buffer[stream].append((record_id, datetime.datetime.now().isoformat(), json.dumps(data))) - logger.debug(f"Added record to buffer: {buffer[stream][len(buffer[stream])-1]}") + buffer[stream].append( + (record_id, datetime.datetime.now().isoformat(), json.dumps(data))) + logger.debug(f"buffering record complete: {buffer[stream][len(buffer[stream])-1]}") # nopep8 because https://github.com/hhatto/autopep8/issues/712 - if message.type == Type.STATE: + elif message.type == Type.STATE: # TODO: This is a queue from the source that we should save the buffer of records from message.type == Type.RECORD messages. See https://docs.airbyte.com/understanding-airbyte/airbyte-protocol#state--the-whole-sync - logger.warning(f"TODO: DUMP buffer with {len(buffer.items())} records into the GBT!") + logger.debug(f"Writing '{len(buffer.items())}' buffered records to GBT...") # nopep8 because https://github.com/hhatto/autopep8/issues/712 + self.glide.add_rows(buffer.items()) + logger.debug(f"Writing '{len(buffer.items())}' buffered records to GBT complete.") # nopep8 because https://github.com/hhatto/autopep8/issues/712 + yield message + else: + logger.warn(f"Ignoring unknown Airbyte input message type: {message.type}") pass def check(self, logger: logging.Logger, config: Mapping[str, Any]) -> AirbyteConnectionStatus: diff --git a/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py b/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py new file mode 100644 index 0000000000000..5e9bae0bfb34d --- /dev/null +++ b/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py @@ -0,0 +1,117 @@ +import requests +from typing import Dict, Any, Iterator + +from .log import getLogger + +logger = getLogger() + +BigTableRow = Dict[str, Any] + + +class GlideBigTable: + """ + An API client for interacting with a Glide Big Table. + """ + + def init(self, api_host, api_key, api_path_root, app_id, table_id): + self.api_host = api_host + self.api_key = api_key + self.api_path_root = api_path_root + self.app_id = app_id + self.table_id = table_id + # todo: validate args + pass + + def headers(self) -> Dict[str, str]: + return { + "Content-Type": "application/json", + f"Authorization": f"Bearer {self.api_key}" + } + + def url(self, path: str) -> str: + return f"https://{self.api_host}/{self.api_path_root}/{path}" + + # todo: add type + def rows(self) -> Iterator[BigTableRow]: + """ + Gets the rows as of the Glide Big Table. + """ + + r = requests.post( + self.url("function/queryTables"), + headers=self.headers(), + json={ + "appID": self.app_id, + "queries": [ + { + "tableName": self.table_id, + "utc": True + } + ] + } + ) + if r.status_code != 200: + logger.error(f"get rows request failed with status {r.status_code}: {r.text}.") # nopep8 because https://github.com/hhatto/autopep8/issues/712 + r.raise_for_status() # This will raise an HTTPError if the status is 4xx or 5xx + + result = r.json() + + # the result looks like an array of results; each result has a rows member that has an array or JSON rows: + for row in result: + for r in row['rows']: + yield r + + def delete_all(self) -> None: + # TODO: perf: don't put these in a list + rows = list(self.rows()) + logger.debug(f"Iterating over {len(rows)} rows to delete") + + for row in rows: + # TODO: lame. batch these into one request with multiple mutations + r = requests.post( + self.url("function/mutateTables"), + headers=self.headers(), + json={ + "appID": self.app_id, + "mutations": [ + { + "kind": "delete-row", + "tableName": self.table_id, + "rowID": row['$rowID'] + } + ] + } + ) + if r.status_code != 200: + logger.error(f"delete request failed with status {r.status_code}: {r.text} trying to delete row id {row['$rowID']} with row: {row}") # nopep8 because https://github.com/hhatto/autopep8/issues/712 + r.raise_for_status() # This will raise an HTTPError if the status is 4xx or 5xx + + def add_rows(self, row: BigTableRow) -> None: + # TODO: lame. need to batch mutations/requests + mutations = [] + for row in rows: + # row is columnName -> value, but glide's mutate is value -> columnName so we fix that here: + mutated_row = {v: k for k, v in row.items()} + + mutations.append({ + { + "kind": "add-row-to-table", + "tableName": self.table_id, + "columnValues": { + # TODO: kinda praying this row is the right shape 😅 + mutated_row + } + } + }) + + r = requests.post( + self.url("function/mutateTables"), + headers=self.headers(), + json={ + "appID": self.app_id, + "mutations": mutations + } + ) + if r.status_code != 200: + logger.error(f"add rows failed with status {r.status_code}: {r.text}") # nopep8 because https://github.com/hhatto/autopep8/issues/712 + r.raise_for_status() # This will raise an HTTPError if the status is 4xx or 5xx diff --git a/airbyte-integrations/connectors/destination-glide/destination_glide/log.py b/airbyte-integrations/connectors/destination-glide/destination_glide/log.py new file mode 100644 index 0000000000000..69ca6ac655c29 --- /dev/null +++ b/airbyte-integrations/connectors/destination-glide/destination_glide/log.py @@ -0,0 +1,21 @@ + +import logging + +# Create a logger + +logger = logging.getLogger("destination-glide") +logger.setLevel(logging.DEBUG) + +# Create a file handler +# TODO REMOVE? +handler = logging.FileHandler('destination-glide.log') +handler.setLevel(logging.DEBUG) +# Create a logging format +formatter = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s - %(message)s') +handler.setFormatter(formatter) +# Add the handlers to the logger +logger.addHandler(handler) + +def getLogger() -> logging.Logger: + return logger \ No newline at end of file diff --git a/airbyte-integrations/connectors/destination-glide/destination_glide/spec.json b/airbyte-integrations/connectors/destination-glide/destination_glide/spec.json index ced8c50f41ce3..66f093c73c456 100644 --- a/airbyte-integrations/connectors/destination-glide/destination_glide/spec.json +++ b/airbyte-integrations/connectors/destination-glide/destination_glide/spec.json @@ -21,6 +21,11 @@ "type": "string", "description": "The user's key for the Glide API destination. Needs permission to write to the specified GBT.", "airbyte_secret": true + }, + "table_id": { + "type": "string", + "description": "The ID of the table that the Glide destination should use. If it does not exist it will be created.", + "airbyte_secret": true } } } diff --git a/airbyte-integrations/connectors/destination-glide/dev-check.sh b/airbyte-integrations/connectors/destination-glide/dev-check.sh deleted file mode 100755 index 04498d8de5676..0000000000000 --- a/airbyte-integrations/connectors/destination-glide/dev-check.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash -this_dir=$(cd $(dirname "$0"); pwd) # this script's directory -this_script=$(basename $0) - -poetry run destination-glide check --config secrets/config.json | jq diff --git a/airbyte-integrations/connectors/destination-glide/dev-spec.sh b/airbyte-integrations/connectors/destination-glide/dev-spec.sh deleted file mode 100755 index 4714c9da54872..0000000000000 --- a/airbyte-integrations/connectors/destination-glide/dev-spec.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash -this_dir=$(cd $(dirname "$0"); pwd) # this script's directory -this_script=$(basename $0) - -poetry run destination-glide spec | jq diff --git a/airbyte-integrations/connectors/destination-glide/dev-write.sh b/airbyte-integrations/connectors/destination-glide/dev-write.sh deleted file mode 100755 index a8c9732bd9685..0000000000000 --- a/airbyte-integrations/connectors/destination-glide/dev-write.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash -this_dir=$(cd $(dirname "$0"); pwd) # this script's directory -this_script=$(basename $0) - -poetry run destination-glide write --config secrets/config.json --catalog sample_files/configured_catalog-test1.json | jq diff --git a/airbyte-integrations/connectors/destination-glide/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-glide/integration_tests/integration_test.py index dadeb330414a2..6791b69faf459 100644 --- a/airbyte-integrations/connectors/destination-glide/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-glide/integration_tests/integration_test.py @@ -1,27 +1,30 @@ # # Copyright (c) 2024 Airbyte, Inc., all rights reserved. # -from datetime import datetime -import json -import logging -import pytest -import random -import string -from typing import Any, Mapping - from airbyte_cdk.models import ( AirbyteMessage, AirbyteRecordMessage, AirbyteStateMessage, - AirbyteStream, - ConfiguredAirbyteCatalog, - ConfiguredAirbyteStream, - DestinationSyncMode, - Status, + AirbyteStream, + ConfiguredAirbyteCatalog, + ConfiguredAirbyteStream, + DestinationSyncMode, + Status, SyncMode, Type ) +from datetime import datetime from destination_glide import DestinationGlide +# for mock: +from destination_glide.glide import GlideBigTable + +import json +import logging +import pytest +import random +import string +from typing import Any, Mapping +from unittest.mock import create_autospec @pytest.fixture(name="config") @@ -39,22 +42,27 @@ def test_table_name() -> str: @pytest.fixture def table_schema() -> str: - schema = {"type": "object", "properties": {"column1": {"type": ["null", "string"]}}} + schema = {"type": "object", "properties": { + "column1": {"type": ["null", "string"]}}} return schema + def AirbyteLogger() -> logging.Logger: return logging.getLogger('airbyte') + @pytest.fixture def configured_catalog(test_table_name: str, table_schema: str) -> ConfiguredAirbyteCatalog: overwrite_stream = ConfiguredAirbyteStream( # TODO: I'm not sure if we should expect incoming streams SyncMode.incremental and only the destination to be full_refresh or they should - stream=AirbyteStream(name=test_table_name, json_schema=table_schema, supported_sync_modes=[SyncMode.incremental]), + stream=AirbyteStream(name=test_table_name, json_schema=table_schema, + supported_sync_modes=[SyncMode.incremental]), sync_mode=SyncMode.incremental, destination_sync_mode=DestinationSyncMode.overwrite, ) return ConfiguredAirbyteCatalog(streams=[overwrite_stream]) + @pytest.fixture def airbyte_message_record1(test_table_name: str): return AirbyteMessage( @@ -74,6 +82,7 @@ def airbyte_message_record2(test_table_name: str): ), ) + @pytest.fixture def airbyte_message_state(test_table_name: str): return AirbyteMessage( @@ -83,12 +92,14 @@ def airbyte_message_state(test_table_name: str): ) ) - ##### Tests Begin Here ##### + + def test_check_valid_config(config: Mapping): outcome = DestinationGlide().check(AirbyteLogger(), config) assert outcome.status == Status.SUCCEEDED + def test_write( config: Mapping, request, @@ -98,13 +109,20 @@ def test_write( airbyte_message_state: AirbyteMessage, test_table_name: str, ): - destination = DestinationGlide() + mock_gbt = create_autospec(GlideBigTable) + + destination = DestinationGlide(mock_gbt) generator = destination.write( - config=config, configured_catalog=configured_catalog, input_messages=[airbyte_message_record1, airbyte_message_record1, airbyte_message_state] + config=config, configured_catalog=configured_catalog, input_messages=[ + airbyte_message_record1, airbyte_message_record1, airbyte_message_state] ) # expecting only to return the state message: result = list(generator) assert len(result) == 1 - assert False, "Validate that the data was written to mock API?" + # expect the API was called: + # todo: validate args on these calls + + mock_gbt.delete_all.assert_called_once() + mock_gbt.add_rows.assert_called_once() diff --git a/airbyte-integrations/connectors/destination-glide/scripts/dev-check.sh b/airbyte-integrations/connectors/destination-glide/scripts/dev-check.sh index 04498d8de5676..a844f0ac30dae 100755 --- a/airbyte-integrations/connectors/destination-glide/scripts/dev-check.sh +++ b/airbyte-integrations/connectors/destination-glide/scripts/dev-check.sh @@ -3,3 +3,4 @@ this_dir=$(cd $(dirname "$0"); pwd) # this script's directory this_script=$(basename $0) poetry run destination-glide check --config secrets/config.json | jq + diff --git a/airbyte-integrations/connectors/destination-glide/scripts/dev-write.sh b/airbyte-integrations/connectors/destination-glide/scripts/dev-write.sh index a8c9732bd9685..7ce2c738de0f3 100755 --- a/airbyte-integrations/connectors/destination-glide/scripts/dev-write.sh +++ b/airbyte-integrations/connectors/destination-glide/scripts/dev-write.sh @@ -2,4 +2,4 @@ this_dir=$(cd $(dirname "$0"); pwd) # this script's directory this_script=$(basename $0) -poetry run destination-glide write --config secrets/config.json --catalog sample_files/configured_catalog-test1.json | jq +poetry run destination-glide write --config secrets/config.json --catalog sample_files/configured_catalog-test1.json From 73f4a86d036257be1227aea49f9e9931572ac28d Mon Sep 17 00:00:00 2001 From: Scott Willeke Date: Wed, 19 Jun 2024 11:57:01 -0400 Subject: [PATCH 03/39] wip: gbt strategy progress --- .../destination_glide/destination.py | 66 +++-- .../destination_glide/glide.py | 226 +++++++++++++++--- .../integration_tests/integration_test.py | 24 +- .../scripts/push-docker-image.sh | 34 ++- .../destination-glide/scripts/test-all.sh | 14 ++ .../destination-glide/scripts/test-unit.sh | 5 + .../connectors/destination-glide/todo.md | 3 + .../GlideBigTableRestStrategy_test.py | 58 +++++ .../destination-glide/unit_tests/unit_test.py | 7 - 9 files changed, 374 insertions(+), 63 deletions(-) create mode 100755 airbyte-integrations/connectors/destination-glide/scripts/test-all.sh create mode 100755 airbyte-integrations/connectors/destination-glide/scripts/test-unit.sh create mode 100644 airbyte-integrations/connectors/destination-glide/todo.md create mode 100644 airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableRestStrategy_test.py delete mode 100644 airbyte-integrations/connectors/destination-glide/unit_tests/unit_test.py diff --git a/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py b/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py index 96b2b7ed85ce9..0dc2e7af82896 100644 --- a/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py +++ b/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py @@ -13,7 +13,7 @@ from collections import defaultdict import datetime -from .glide import GlideBigTable +from .glide import GlideBigTableBase, CreateBigTableDefaultImpl, Column import json from .log import getLogger import logging @@ -25,9 +25,9 @@ class DestinationGlide(Destination): # GlideBigTable optional for tests to inject mock - def __init__(self, glide: GlideBigTable = None): + def __init__(self, glide: GlideBigTableBase = None): if glide is None: - glide = GlideBigTable() + glide = CreateBigTableDefaultImpl() self.glide = glide def write( @@ -52,18 +52,46 @@ def write( api_path_root = config['api_path_root'] api_key = config['api_key'] table_id = config['table_id'] - # TODO: hardcoded for now using old api - app_id = "Ix9CEuP6DiFugfjhSG5t" - self.glide.init(api_host, api_key, api_path_root, app_id, table_id) + self.glide.init(api_host, api_key, api_path_root, table_id) # go through each stream and add it as needed: stream_names = {s.stream.name for s in configured_catalog.streams} for configured_stream in configured_catalog.streams: if configured_stream.destination_sync_mode != DestinationSyncMode.overwrite: raise Exception(f'Only destination sync mode overwrite it supported, but received "{configured_stream.destination_sync_mode}".') # nopep8 because https://github.com/hhatto/autopep8/issues/712 - - # TODO: create a new GBT to prepare for dumping the data into it + + # TODO: upsert the GBT with schema to prepare for dumping the data into it + def mapJsonSchemaTypeToGlideType(json_type: str) -> str: + jsonSchemaTypeToGlideType = { + "string":"string", + "number": "number", + "integer": "number", + "boolean":"boolean", + } + if isinstance(json_type, list): + logger.debug(f"Found list type '{json_type}' in stream {configured_stream.stream.name}. Attempting to map to a primitive type.") # nopep8 because + # find the first type that is not 'null' and use that instead: + for t in json_type: + if t != "null" and t in jsonSchemaTypeToGlideType: + logger.debug(f"Mapped json schema list type of '{json_type}' to '{t}' in stream {configured_stream.stream.name}.") # nopep8 because + json_type = t + + if json_type in jsonSchemaTypeToGlideType: + return jsonSchemaTypeToGlideType[json_type] + raise ValueError(f"Unsupported JSON schema type for glide '{json_type}'") + + columns = [] + properties = configured_stream.stream.json_schema["properties"] + for prop_name in properties.keys(): + prop = properties[prop_name] + prop_type = prop["type"] + prop_format = prop["format"] if "format" in prop else "" + logger.debug(f"Found column/property '{prop_name}' with type '{prop_type}' and format '{prop_format}' in stream {configured_stream.stream.name}.") + columns.append(Column(prop_name, mapJsonSchemaTypeToGlideType(prop_type))) + + self.glide.prepare_table(columns) + # NOTE: for now using an existing GBT in old API logger.debug("deleting all rows...") self.glide.delete_all() @@ -71,7 +99,7 @@ def write( # stream the records into the GBT: buffer = defaultdict(list) - logger.debug(f"buffer created.") + logger.debug("Processing messages...") for message in input_messages: logger.debug(f"processing message {message.type}...") if message.type == Type.RECORD: @@ -87,15 +115,21 @@ def write( # add to buffer record_id = str(uuid.uuid4()) - buffer[stream].append( - (record_id, datetime.datetime.now().isoformat(), json.dumps(data))) - logger.debug(f"buffering record complete: {buffer[stream][len(buffer[stream])-1]}") # nopep8 because https://github.com/hhatto/autopep8/issues/712 + stream_buffer = buffer[stream] + stream_buffer.append( + (record_id, datetime.datetime.now().isoformat(), data)) + + logger.debug(f"buffering record complete: {stream_buffer[len(stream_buffer)-1]}") # nopep8 because https://github.com/hhatto/autopep8/issues/712 elif message.type == Type.STATE: - # TODO: This is a queue from the source that we should save the buffer of records from message.type == Type.RECORD messages. See https://docs.airbyte.com/understanding-airbyte/airbyte-protocol#state--the-whole-sync - logger.debug(f"Writing '{len(buffer.items())}' buffered records to GBT...") # nopep8 because https://github.com/hhatto/autopep8/issues/712 - self.glide.add_rows(buffer.items()) - logger.debug(f"Writing '{len(buffer.items())}' buffered records to GBT complete.") # nopep8 because https://github.com/hhatto/autopep8/issues/712 + # This is a queue from the source that we should save the buffer of records from message.type == Type.RECORD messages. See https://docs.airbyte.com/understanding-airbyte/airbyte-protocol#state--the-whole-sync + for stream_name in buffer.keys(): + stream_records = buffer[stream_name] + logger.debug(f"Saving buffered records to Glide API (stream: '{stream_name}' count: '{len(stream_records)}')...") # nopep8 because https://github.com/hhatto/autopep8/issues/712 + DATA_INDEX = 2 + data_rows = [row_tuple[DATA_INDEX] for row_tuple in stream_records] + self.glide.add_rows(data_rows) + logger.debug(f"Saving buffered records to Glide API complete.") # nopep8 because https://github.com/hhatto/autopep8/issues/712 yield message else: diff --git a/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py b/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py index 5e9bae0bfb34d..1d9903a01441b 100644 --- a/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py +++ b/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py @@ -1,5 +1,6 @@ +from abc import ABC, abstractmethod import requests -from typing import Dict, Any, Iterator +from typing import Dict, Any, Iterator, List from .log import getLogger @@ -7,21 +8,179 @@ BigTableRow = Dict[str, Any] +ALLOWED_COLUMN_TYPES = [ + "string", + "number", + "boolean", + "url", + "dateTime", + "json", +] + +class Column: + def __init__(self, name: str, type: str): + if type not in ALLOWED_COLUMN_TYPES: + raise ValueError(f"Column type {type} not allowed. Must be one of {ALLOWED_COLUMN_TYPES}") # nopep8 + self._name = name + self._type = type + + def name() -> str: + return self._name + + def type() -> str: + return self._type + + def to_json(self) -> Dict[str, Any]: + return { + 'name': self._name, + 'displayName': self._name, + 'type': self._type, + } + + +class GlideBigTableBase(ABC): + def headers(self) -> Dict[str, str]: + return { + "Content-Type": "application/json", + f"Authorization": f"Bearer {self.api_key}" + } + + def url(self, path: str) -> str: + return f"https://{self.api_host}/{self.api_path_root}/{path}" -class GlideBigTable: """ An API client for interacting with a Glide Big Table. """ - def init(self, api_host, api_key, api_path_root, app_id, table_id): + def init(self, api_host, api_key, api_path_root, table_id): self.api_host = api_host self.api_key = api_key self.api_path_root = api_path_root - self.app_id = app_id self.table_id = table_id # todo: validate args pass + @abstractmethod + def prepare_table(self, columns: List[Column]) -> None: + """ + Prepares the table with the given columns. + Each column is a json-schema property where the key is the column name and the type is the . + """ + pass + + @abstractmethod + def rows(self) -> Iterator[BigTableRow]: + """ + Gets the rows as of the Glide Big Table. + """ + pass + + @abstractmethod + def delete_all(self) -> None: + """ + Deletes all rows in the table. + """ + pass + + def add_rows(self, rows: Iterator[BigTableRow]) -> None: + """ + Adds rows to the table. + """ + pass + + +def CreateBigTableDefaultImpl() -> GlideBigTableBase: + """ + Creates a new instance of the default implementation for the GlideBigTable API client. + """ + return GlideBigTableMutationsStrategy() + + +class GlideBigTableRestStrategy(GlideBigTableBase): + + def prepare_table(self, columns: List[Column]) -> None: + logger.debug(f"prepare_table columns: {columns}") + # update the table: + r = requests.put( + self.url(f"/tables/{self.table_id}"), + headers=self.headers(), + json={ + "name": self.table_id, + "schema": { + "columns": [ + { + "name": col.name, + "type": col.type + } for col in columns + ], + }, + "rows": [] + } + ) + if r.status_code != 200: + logger.error(f"prepare table request failed with status {r.status_code}: {r.text}.") # nopep8 + + def rows(self) -> Iterator[BigTableRow]: + r = requests.get( + self.url(f"/tables/{self.table_id}/rows"), + headers=self.headers(), + ) + if r.status_code != 200: + logger.error(f"get rows request failed with status {r.status_code}: {r.text}.") # nopep8 because https://github.com/hhatto/autopep8/issues/712 + r.raise_for_status() # This will raise an HTTPError if the status is 4xx or 5xx + + result = r.json() + + # the result looks like an array of results; each result has a rows member that has an array or JSON rows: + for row in result: + for r in row['rows']: + yield r + + def delete_all(self) -> None: + logger.warning(f"delete_all call is ignored in {type(self).__class__.__name__}") # nopep8 + pass + + def add_rows(self, rows: Iterator[BigTableRow]) -> None: + r = requests.post( + self.url(f"/tables/{self.table_id}/rows"), + headers=self.headers(), + json={ + "rows": list(rows) + } + ) + if r.status_code != 200: + logger.error(f"get rows request failed with status {r.status_code}: {r.text}.") # nopep8 because https://github.com/hhatto/autopep8/issues/712 + r.raise_for_status() # This will raise an HTTPError if the status is 4xx or 5xx + + +class GlideBigTableMutationsStrategy(GlideBigTableBase): + def __init__(self): + # TODO: hardcoded for now using old api + self.hardcoded_app_id = "Ix9CEuP6DiFugfjhSG5t" + self.hardcoded_column_lookup = { + '_airtable_id': {'type': "string", 'name': "Name"}, + '_airtable_created_time': {'type': "date-time", 'name': "AwkFL"}, + '_airtable_table_name': {'type': "string", 'name': "QF0zI"}, + 'id': {'type': "string", 'name': "tLPjZ"}, + 'name': {'type': "string", 'name': "1ZqF1"}, + 'host_id': {'type': "string", 'name': "B7fYe"}, + 'host_name': {'type': "string", 'name': "oyVzO"}, + 'neighbourhood_group': {'type': "string", 'name': "15J8U"}, + 'neighbourhood': {'type': "string", 'name': "Fy28U"}, + 'latitude': {'type': "number", 'name': "TLpMC"}, + 'longitude': {'type': "number", 'name': "oazQO"}, + 'room_type': {'type': "string", 'name': "TPJDZ"}, + 'price': {'type': "number", 'name': "7xzlG"}, + 'minimum_nights': {'type': "number", 'name': "usoY5"}, + 'number_of_reviews': {'type': "number", 'name': "XFXmR"}, + 'last_review': {'type': "date-time", 'name': "oseZl"}, + 'reviews_per_month': {'type': "number", 'name': "alw2R"}, + 'calculated_host_listings_count': {'type': "number", 'name': "hKws0"}, + 'availability_365': {'type': "number", 'name': "qZsgl"}, + 'number_of_reviews_ltm': {'type': "number", 'name': "rWisS"}, + 'license': {'type': "string", 'name': "7PVig"} + } + def headers(self) -> Dict[str, str]: return { "Content-Type": "application/json", @@ -31,17 +190,21 @@ def headers(self) -> Dict[str, str]: def url(self, path: str) -> str: return f"https://{self.api_host}/{self.api_path_root}/{path}" - # todo: add type + def prepare_table(self, columns: List[Column]) -> None: + for col in columns: + if col.name not in self.hardcoded_column_lookup: + logger.warning( + f"Column '{col.name}' not found in hardcoded column lookup. Will be ignored.") + def rows(self) -> Iterator[BigTableRow]: """ Gets the rows as of the Glide Big Table. """ - r = requests.post( self.url("function/queryTables"), headers=self.headers(), json={ - "appID": self.app_id, + "appID": self.hardcoded_app_id, "queries": [ { "tableName": self.table_id, @@ -51,7 +214,7 @@ def rows(self) -> Iterator[BigTableRow]: } ) if r.status_code != 200: - logger.error(f"get rows request failed with status {r.status_code}: {r.text}.") # nopep8 because https://github.com/hhatto/autopep8/issues/712 + logger.error(f"get rows request failed with status {r.status_code}: {r.text}.") # nopep8 because https://github.com/hhatto/autopep8/issues/712 r.raise_for_status() # This will raise an HTTPError if the status is 4xx or 5xx result = r.json() @@ -72,7 +235,7 @@ def delete_all(self) -> None: self.url("function/mutateTables"), headers=self.headers(), json={ - "appID": self.app_id, + "appID": self.hardcoded_app_id, "mutations": [ { "kind": "delete-row", @@ -83,35 +246,40 @@ def delete_all(self) -> None: } ) if r.status_code != 200: - logger.error(f"delete request failed with status {r.status_code}: {r.text} trying to delete row id {row['$rowID']} with row: {row}") # nopep8 because https://github.com/hhatto/autopep8/issues/712 + logger.error(f"delete request failed with status {r.status_code}: {r.text} trying to delete row id {row['$rowID']} with row: {row}") # nopep8 because https://github.com/hhatto/autopep8/issues/712 r.raise_for_status() # This will raise an HTTPError if the status is 4xx or 5xx - def add_rows(self, row: BigTableRow) -> None: + def add_rows(self, rows: Iterator[BigTableRow]) -> None: # TODO: lame. need to batch mutations/requests mutations = [] for row in rows: - # row is columnName -> value, but glide's mutate is value -> columnName so we fix that here: - mutated_row = {v: k for k, v in row.items()} + # row is columnLabel -> value, but glide's mutate uses a column "name". We hard-code the lookup for our table here: + + logger.debug(f"Mutating row: {row}") + mutated_row = dict() + for k, v in row.items(): + if k in self.hardcoded_column_lookup: + col_info = self.hardcoded_column_lookup[k] + mutated_row[col_info["name"]] = v + else: + logger.error( + f"Column {k} not found in column lookup. Ignoring column") - mutations.append({ + mutations.append( { "kind": "add-row-to-table", "tableName": self.table_id, - "columnValues": { - # TODO: kinda praying this row is the right shape 😅 - mutated_row - } + "columnValues": mutated_row } - }) - - r = requests.post( - self.url("function/mutateTables"), - headers=self.headers(), - json={ - "appID": self.app_id, - "mutations": mutations - } - ) + ) + r = requests.post( + self.url("function/mutateTables"), + headers=self.headers(), + json={ + "appID": self.hardcoded_app_id, + "mutations": mutations + } + ) if r.status_code != 200: - logger.error(f"add rows failed with status {r.status_code}: {r.text}") # nopep8 because https://github.com/hhatto/autopep8/issues/712 + logger.error(f"add rows failed with status {r.status_code}: {r.text}") # nopep8 because https://github.com/hhatto/autopep8/issues/712 r.raise_for_status() # This will raise an HTTPError if the status is 4xx or 5xx diff --git a/airbyte-integrations/connectors/destination-glide/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-glide/integration_tests/integration_test.py index 6791b69faf459..fb3912cb32029 100644 --- a/airbyte-integrations/connectors/destination-glide/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-glide/integration_tests/integration_test.py @@ -16,7 +16,7 @@ from datetime import datetime from destination_glide import DestinationGlide # for mock: -from destination_glide.glide import GlideBigTable +from destination_glide.glide import GlideBigTableBase import json import logging @@ -26,7 +26,6 @@ from typing import Any, Mapping from unittest.mock import create_autospec - @pytest.fixture(name="config") def config_fixture() -> Mapping[str, Any]: with open("secrets/config.json", "r") as f: @@ -42,15 +41,21 @@ def test_table_name() -> str: @pytest.fixture def table_schema() -> str: - schema = {"type": "object", "properties": { - "column1": {"type": ["null", "string"]}}} - return schema + stream_schema = { + "type": "object", + "properties": { + "string_col": {"type": "string"}, + "int_col": {"type": "integer"}, + "date_col": {"type": "string", "format": "date-time"}, + "other_col": {"type": ["null", "string"]} + }, + } + return stream_schema def AirbyteLogger() -> logging.Logger: return logging.getLogger('airbyte') - @pytest.fixture def configured_catalog(test_table_name: str, table_schema: str) -> ConfiguredAirbyteCatalog: overwrite_stream = ConfiguredAirbyteStream( @@ -62,7 +67,6 @@ def configured_catalog(test_table_name: str, table_schema: str) -> ConfiguredAir ) return ConfiguredAirbyteCatalog(streams=[overwrite_stream]) - @pytest.fixture def airbyte_message_record1(test_table_name: str): return AirbyteMessage( @@ -72,7 +76,6 @@ def airbyte_message_record1(test_table_name: str): ), ) - @pytest.fixture def airbyte_message_record2(test_table_name: str): return AirbyteMessage( @@ -92,6 +95,9 @@ def airbyte_message_state(test_table_name: str): ) ) + +#configured_stream.stream.json_schema["properties"] + ##### Tests Begin Here ##### @@ -109,7 +115,7 @@ def test_write( airbyte_message_state: AirbyteMessage, test_table_name: str, ): - mock_gbt = create_autospec(GlideBigTable) + mock_gbt = create_autospec(GlideBigTableBase) destination = DestinationGlide(mock_gbt) generator = destination.write( diff --git a/airbyte-integrations/connectors/destination-glide/scripts/push-docker-image.sh b/airbyte-integrations/connectors/destination-glide/scripts/push-docker-image.sh index a12c4897009de..37054bc1500c0 100755 --- a/airbyte-integrations/connectors/destination-glide/scripts/push-docker-image.sh +++ b/airbyte-integrations/connectors/destination-glide/scripts/push-docker-image.sh @@ -2,8 +2,38 @@ this_dir=$(cd $(dirname "$0"); pwd) # this script's directory this_script=$(basename $0) +#!/usr/bin/env bash + +# Fetch the list of tags from Docker Hub +tags=$(wget -q -O - "https://hub.docker.com/v2/namespaces/activescott/repositories/destination-glide/tags?page_size=10" | grep -o '"name": *"[^"]*' | grep -o '[^"]*$' | grep -E '([0-9]+\.)+[0-9]+' ) +`` +echo "Found tags: $tags" +# Sort the tags and get the highest one +highest_tag=$(echo "$tags" | sort -V | tail -n 1) + +echo "found highest tag: $highest_tag" + +# Increment the version +IFS='.' read -ra ADDR <<< "$highest_tag" +new_version="${ADDR[0]}.${ADDR[1]}.$((ADDR[2]+1))" + +# Show the user the new version and ask if they want to continue +echo "The new version will be $new_version. Do you want to continue? (y/n)" +read answer + +if [ "$answer" != "${answer#[Yy]}" ] ; then + echo "Continuing..." +else + echo "Exiting..." + exit 1 +fi + + +# Tag the local image with the new version +# TODO: airbyte/destination-glide is the local docker image that airbyte's CI builds it and names it as locally. Can't we change this ?? +docker image tag airbyte/destination-glide:dev activescott/destination-glide:$new_version -docker image tag airbyte/destination-glide:dev activescott/destination-glide:dev +# Push the image with the new tag +docker push activescott/destination-glide:$new_version -docker push activescott/destination-glide:dev diff --git a/airbyte-integrations/connectors/destination-glide/scripts/test-all.sh b/airbyte-integrations/connectors/destination-glide/scripts/test-all.sh new file mode 100755 index 0000000000000..49d8dfecff924 --- /dev/null +++ b/airbyte-integrations/connectors/destination-glide/scripts/test-all.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +this_dir=$(cd $(dirname "$0"); pwd) # this script's directory +this_script=$(basename $0) + +"$this_dir/test-unit.sh" +if [ $? -ne 0 ]; then + echo "Unit tests failed" + exit 1 +fi +"$this_dir/test-integration.sh" +if [ $? -ne 0 ]; then + echo "Integration tests failed" + exit 1 +fi \ No newline at end of file diff --git a/airbyte-integrations/connectors/destination-glide/scripts/test-unit.sh b/airbyte-integrations/connectors/destination-glide/scripts/test-unit.sh new file mode 100755 index 0000000000000..488421db4323b --- /dev/null +++ b/airbyte-integrations/connectors/destination-glide/scripts/test-unit.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +this_dir=$(cd $(dirname "$0"); pwd) # this script's directory +this_script=$(basename $0) + +poetry run pytest unit_tests diff --git a/airbyte-integrations/connectors/destination-glide/todo.md b/airbyte-integrations/connectors/destination-glide/todo.md new file mode 100644 index 0000000000000..760caa1aa41cb --- /dev/null +++ b/airbyte-integrations/connectors/destination-glide/todo.md @@ -0,0 +1,3 @@ +- [ ] unify test framework across unit/integration tests (remove pytest?) +- [ ] clean up todos +- [ ] figure out why the dev-write script stopped working (freezes) diff --git a/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableRestStrategy_test.py b/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableRestStrategy_test.py new file mode 100644 index 0000000000000..8782d345b6e95 --- /dev/null +++ b/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableRestStrategy_test.py @@ -0,0 +1,58 @@ +import unittest +from unittest.mock import patch +import requests # for mocking it +from destination_glide.glide import GlideBigTableRestStrategy, Column + + +class TestGlideBigTableRestStrategy(unittest.TestCase): + + api_host = 'https://test-api-host.com' + api_key = 'test-api-key' + api_path_root = '/test/api/path/root' + table_id = 'test-table-id' + + def setUp(self): + self.gbt = GlideBigTableRestStrategy() + self.gbt.init(self.api_host, self.api_key, + self.api_path_root, self.table_id) + + @patch.object(requests, 'put') + def test_prepare_table_valid(self, mock_put): + mock_put.return_value.status_code = 200 + mock_put.return_value.json.return_value = {'data': 'test'} + + self.gbt.prepare_table([ + Column('test-str', 'string'), + Column('test-num', 'number') + ]) + + mock_put.assert_called_once() + + @patch.object(requests, 'put') + def test_prepare_table_invalid_col_type(self, mock_put): + mock_put.return_value.status_code = 200 + mock_put.return_value.json.return_value = {'data': 'test'} + + with self.assertRaises(ValueError): + self.gbt.prepare_table([ + Column('test-str', 'string'), + Column('test-num', 'invalid-type') + ]) + + @patch.object(requests, 'post') + def test_add_rows(self, mock_post): + mock_post.return_value.status_code = 200 + mock_post.return_value.json.return_value = {'data': 'test'} + + test_rows = [ + Column('strcol', "string"), + Column('numcol', "number") + ] + self.gbt.add_rows(test_rows) + + mock_post.assert_called_once() + assert mock_post.call_args[1]['json']['rows'] == test_rows + + +if __name__ == '__main__': + unittest.main() diff --git a/airbyte-integrations/connectors/destination-glide/unit_tests/unit_test.py b/airbyte-integrations/connectors/destination-glide/unit_tests/unit_test.py deleted file mode 100644 index 8da866052b812..0000000000000 --- a/airbyte-integrations/connectors/destination-glide/unit_tests/unit_test.py +++ /dev/null @@ -1,7 +0,0 @@ -# -# Copyright (c) 2024 Airbyte, Inc., all rights reserved. -# - - -def test_example_method(): - assert True From dc74465aed551914d0716a08ea3aac01ac238da2 Mon Sep 17 00:00:00 2001 From: Scott Willeke Date: Wed, 19 Jun 2024 12:26:04 -0400 Subject: [PATCH 04/39] chore: minor cleanup, move delete_all decision into the strategy --- .../destination_glide/destination.py | 51 +++++++++---------- .../destination_glide/glide.py | 5 +- .../integration_tests/integration_test.py | 2 +- .../connectors/destination-glide/todo.md | 2 + 4 files changed, 30 insertions(+), 30 deletions(-) diff --git a/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py b/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py index 0dc2e7af82896..2c4e3fc161ffd 100644 --- a/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py +++ b/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py @@ -23,6 +23,25 @@ logger = getLogger() +def mapJsonSchemaTypeToGlideType(json_type: str) -> str: + jsonSchemaTypeToGlideType = { + "string":"string", + "number": "number", + "integer": "number", + "boolean":"boolean", + } + if isinstance(json_type, list): + logger.debug(f"Found list type '{json_type}'. Attempting to map to a primitive type.") # nopep8 because + # find the first type that is not 'null' and use that instead: + for t in json_type: + if t != "null" and t in jsonSchemaTypeToGlideType: + logger.debug(f"Mapped json schema list type of '{json_type}' to '{t}'.") # nopep8 because + json_type = t + + if json_type in jsonSchemaTypeToGlideType: + return jsonSchemaTypeToGlideType[json_type] + raise ValueError(f"Unsupported JSON schema type for glide '{json_type}'") + class DestinationGlide(Destination): # GlideBigTable optional for tests to inject mock def __init__(self, glide: GlideBigTableBase = None): @@ -53,34 +72,16 @@ def write( api_key = config['api_key'] table_id = config['table_id'] + # TODO: choose a strategy based on config self.glide.init(api_host, api_key, api_path_root, table_id) # go through each stream and add it as needed: stream_names = {s.stream.name for s in configured_catalog.streams} for configured_stream in configured_catalog.streams: if configured_stream.destination_sync_mode != DestinationSyncMode.overwrite: - raise Exception(f'Only destination sync mode overwrite it supported, but received "{configured_stream.destination_sync_mode}".') # nopep8 because https://github.com/hhatto/autopep8/issues/712 + raise Exception(f'Only destination sync mode overwrite is supported, but received "{configured_stream.destination_sync_mode}".') # nopep8 because https://github.com/hhatto/autopep8/issues/712 - # TODO: upsert the GBT with schema to prepare for dumping the data into it - def mapJsonSchemaTypeToGlideType(json_type: str) -> str: - jsonSchemaTypeToGlideType = { - "string":"string", - "number": "number", - "integer": "number", - "boolean":"boolean", - } - if isinstance(json_type, list): - logger.debug(f"Found list type '{json_type}' in stream {configured_stream.stream.name}. Attempting to map to a primitive type.") # nopep8 because - # find the first type that is not 'null' and use that instead: - for t in json_type: - if t != "null" and t in jsonSchemaTypeToGlideType: - logger.debug(f"Mapped json schema list type of '{json_type}' to '{t}' in stream {configured_stream.stream.name}.") # nopep8 because - json_type = t - - if json_type in jsonSchemaTypeToGlideType: - return jsonSchemaTypeToGlideType[json_type] - raise ValueError(f"Unsupported JSON schema type for glide '{json_type}'") - + # upsert the GBT with schema to prepare for dumping the data into it columns = [] properties = configured_stream.stream.json_schema["properties"] for prop_name in properties.keys(): @@ -91,11 +92,6 @@ def mapJsonSchemaTypeToGlideType(json_type: str) -> str: columns.append(Column(prop_name, mapJsonSchemaTypeToGlideType(prop_type))) self.glide.prepare_table(columns) - - # NOTE: for now using an existing GBT in old API - logger.debug("deleting all rows...") - self.glide.delete_all() - logger.debug("deleting all rows complete.") # stream the records into the GBT: buffer = defaultdict(list) @@ -107,7 +103,7 @@ def mapJsonSchemaTypeToGlideType(json_type: str) -> str: data = message.record.data stream = message.record.stream if stream not in stream_names: - logger.debug( + logger.warning( f"Stream {stream} was not present in configured streams, skipping") continue @@ -154,3 +150,4 @@ def check(self, logger: logging.Logger, config: Mapping[str, Any]) -> AirbyteCon return AirbyteConnectionStatus(status=Status.SUCCEEDED) except Exception as e: return AirbyteConnectionStatus(status=Status.FAILED, message=f"An exception occurred: {repr(e)}") + diff --git a/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py b/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py index 1d9903a01441b..2aed09b243374 100644 --- a/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py +++ b/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py @@ -32,8 +32,7 @@ def type() -> str: def to_json(self) -> Dict[str, Any]: return { - 'name': self._name, - 'displayName': self._name, + 'id': self._name, 'type': self._type, } @@ -191,6 +190,8 @@ def url(self, path: str) -> str: return f"https://{self.api_host}/{self.api_path_root}/{path}" def prepare_table(self, columns: List[Column]) -> None: + self.delete_all() + for col in columns: if col.name not in self.hardcoded_column_lookup: logger.warning( diff --git a/airbyte-integrations/connectors/destination-glide/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-glide/integration_tests/integration_test.py index fb3912cb32029..a06fc8b99ba14 100644 --- a/airbyte-integrations/connectors/destination-glide/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-glide/integration_tests/integration_test.py @@ -130,5 +130,5 @@ def test_write( # expect the API was called: # todo: validate args on these calls - mock_gbt.delete_all.assert_called_once() + mock_gbt.prepare_table.assert_called_once() mock_gbt.add_rows.assert_called_once() diff --git a/airbyte-integrations/connectors/destination-glide/todo.md b/airbyte-integrations/connectors/destination-glide/todo.md index 760caa1aa41cb..f8300434a6ec3 100644 --- a/airbyte-integrations/connectors/destination-glide/todo.md +++ b/airbyte-integrations/connectors/destination-glide/todo.md @@ -1,3 +1,5 @@ +- [ ] choose a strategy based on config - [ ] unify test framework across unit/integration tests (remove pytest?) - [ ] clean up todos +- [ ] update readme - [ ] figure out why the dev-write script stopped working (freezes) From 47fdd7499f8bb81a9cb84663849c61995bfeca58 Mon Sep 17 00:00:00 2001 From: Scott Willeke Date: Wed, 19 Jun 2024 16:51:11 -0400 Subject: [PATCH 05/39] chore: cleanup logging --- .../destination_glide/destination.py | 3 +- .../destination_glide/glide.py | 34 +++++++++++-------- .../connectors/destination-glide/todo.md | 12 ++++--- .../GlideBigTableRestStrategy_test.py | 6 ++-- 4 files changed, 31 insertions(+), 24 deletions(-) diff --git a/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py b/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py index 2c4e3fc161ffd..564ca762ac16f 100644 --- a/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py +++ b/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py @@ -114,8 +114,7 @@ def write( stream_buffer = buffer[stream] stream_buffer.append( (record_id, datetime.datetime.now().isoformat(), data)) - - logger.debug(f"buffering record complete: {stream_buffer[len(stream_buffer)-1]}") # nopep8 because https://github.com/hhatto/autopep8/issues/712 + logger.debug(f"buffering record complete.") # nopep8 because https://github.com/hhatto/autopep8/issues/712 elif message.type == Type.STATE: # This is a queue from the source that we should save the buffer of records from message.type == Type.RECORD messages. See https://docs.airbyte.com/understanding-airbyte/airbyte-protocol#state--the-whole-sync diff --git a/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py b/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py index 2aed09b243374..481b7e6fde8cc 100644 --- a/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py +++ b/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py @@ -18,24 +18,32 @@ ] class Column: - def __init__(self, name: str, type: str): + def __init__(self, id: str, type: str): if type not in ALLOWED_COLUMN_TYPES: raise ValueError(f"Column type {type} not allowed. Must be one of {ALLOWED_COLUMN_TYPES}") # nopep8 - self._name = name + self._id = id self._type = type - def name() -> str: - return self._name + def id() -> str: + return self._id def type() -> str: return self._type def to_json(self) -> Dict[str, Any]: return { - 'id': self._name, + 'id': self._id, 'type': self._type, + 'displayName': self._id } + def __eq__(self, other): + if isinstance(other, Column): + return self._id == other._id and self._type == other._type + return False + + def __repr__(self): + return f"Column(id='{self._id}', type='{self._type}')" class GlideBigTableBase(ABC): def headers(self) -> Dict[str, str]: @@ -106,12 +114,7 @@ def prepare_table(self, columns: List[Column]) -> None: json={ "name": self.table_id, "schema": { - "columns": [ - { - "name": col.name, - "type": col.type - } for col in columns - ], + "columns": columns, }, "rows": [] } @@ -190,12 +193,13 @@ def url(self, path: str) -> str: return f"https://{self.api_host}/{self.api_path_root}/{path}" def prepare_table(self, columns: List[Column]) -> None: + logger.debug(f"prepare_table for table '{self.table_id}. Expecting columns: '{[c.id for c in columns]}'.") self.delete_all() for col in columns: if col.name not in self.hardcoded_column_lookup: logger.warning( - f"Column '{col.name}' not found in hardcoded column lookup. Will be ignored.") + f"Column '{col.id}' not found in hardcoded column lookup. Will be ignored.") def rows(self) -> Iterator[BigTableRow]: """ @@ -248,15 +252,15 @@ def delete_all(self) -> None: ) if r.status_code != 200: logger.error(f"delete request failed with status {r.status_code}: {r.text} trying to delete row id {row['$rowID']} with row: {row}") # nopep8 because https://github.com/hhatto/autopep8/issues/712 - r.raise_for_status() # This will raise an HTTPError if the status is 4xx or 5xx + r.raise_for_status() # This will raise an HTTPError if the status is 4xx or 5xx + else: + logger.debug(f"Deleted row successfully (rowID:'{row['$rowID']}'") def add_rows(self, rows: Iterator[BigTableRow]) -> None: # TODO: lame. need to batch mutations/requests mutations = [] for row in rows: # row is columnLabel -> value, but glide's mutate uses a column "name". We hard-code the lookup for our table here: - - logger.debug(f"Mutating row: {row}") mutated_row = dict() for k, v in row.items(): if k in self.hardcoded_column_lookup: diff --git a/airbyte-integrations/connectors/destination-glide/todo.md b/airbyte-integrations/connectors/destination-glide/todo.md index f8300434a6ec3..d7bc7c83dc711 100644 --- a/airbyte-integrations/connectors/destination-glide/todo.md +++ b/airbyte-integrations/connectors/destination-glide/todo.md @@ -1,5 +1,7 @@ -- [ ] choose a strategy based on config -- [ ] unify test framework across unit/integration tests (remove pytest?) -- [ ] clean up todos -- [ ] update readme -- [ ] figure out why the dev-write script stopped working (freezes) +- [ ] feat: choose a strategy based on config +- [+] chore: stop writing any record data to logs +- [+] chore: cleanup logs generally +- [ ] chore: unify test framework across unit/integration tests (remove pytest?) +- [ ] chore: clean up todos +- [ ] chore: update readme +- [ ] chore: figure out why the dev-write script stopped working (freezes) diff --git a/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableRestStrategy_test.py b/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableRestStrategy_test.py index 8782d345b6e95..f3ec91fa6e954 100644 --- a/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableRestStrategy_test.py +++ b/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableRestStrategy_test.py @@ -21,12 +21,14 @@ def test_prepare_table_valid(self, mock_put): mock_put.return_value.status_code = 200 mock_put.return_value.json.return_value = {'data': 'test'} - self.gbt.prepare_table([ + test_columns = [ Column('test-str', 'string'), Column('test-num', 'number') - ]) + ] + self.gbt.prepare_table(test_columns) mock_put.assert_called_once() + self.assertListEqual(mock_put.call_args[1]['json']['schema']['columns'], test_columns) @patch.object(requests, 'put') def test_prepare_table_invalid_col_type(self, mock_put): From 49830626efb3306746120278e8210eae5efd2b23 Mon Sep 17 00:00:00 2001 From: Scott Willeke Date: Wed, 19 Jun 2024 17:17:32 -0400 Subject: [PATCH 06/39] fix: AttributeError: 'Column' object has no attribute 'name' --- .../destination_glide/glide.py | 2 +- .../GlideBigTableMutationsStrategy_test.py | 34 +++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableMutationsStrategy_test.py diff --git a/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py b/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py index 481b7e6fde8cc..2c0736c8ec250 100644 --- a/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py +++ b/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py @@ -197,7 +197,7 @@ def prepare_table(self, columns: List[Column]) -> None: self.delete_all() for col in columns: - if col.name not in self.hardcoded_column_lookup: + if col.id not in self.hardcoded_column_lookup: logger.warning( f"Column '{col.id}' not found in hardcoded column lookup. Will be ignored.") diff --git a/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableMutationsStrategy_test.py b/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableMutationsStrategy_test.py new file mode 100644 index 0000000000000..951cb19d541d2 --- /dev/null +++ b/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableMutationsStrategy_test.py @@ -0,0 +1,34 @@ +from destination_glide.glide import GlideBigTableMutationsStrategy, Column +import requests +import unittest +from unittest.mock import patch + +class TestGlideBigTableMutationsStrategy(unittest.TestCase): + api_host = 'https://test-api-host.com' + api_key = 'test-api-key' + api_path_root = '/test/api/path/root' + table_id = 'test-table-id' + + def setUp(self): + self.gbt = GlideBigTableMutationsStrategy() + self.gbt.init(self.api_host, self.api_key, + self.api_path_root, self.table_id) + + @patch.object(requests, 'post') + def test_prepare_table_valid(self, mock_post): + mock_post.return_value.status_code = 200 + test_columns = [ + Column('id', 'string'), + Column('name', 'string') + ] + self.gbt.prepare_table(test_columns) + + @patch.object(requests, 'post') + def test_prepare_table_invalid_column(self, mock_post): + mock_post.return_value.status_code = 200 + test_columns = [ + Column('id', 'string'), + Column('this column wont be found', 'string') + ] + self.gbt.prepare_table(test_columns) + From 388df6ed8af0a8413a90755ed2f891eacd93f910 Mon Sep 17 00:00:00 2001 From: Scott Willeke Date: Wed, 19 Jun 2024 17:48:35 -0400 Subject: [PATCH 07/39] fix: was adding duplicate rows to gbt --- .../destination_glide/glide.py | 16 ++++++------ .../GlideBigTableMutationsStrategy_test.py | 25 +++++++++++++++++++ 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py b/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py index 2c0736c8ec250..9d01b7bc9f083 100644 --- a/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py +++ b/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py @@ -277,14 +277,14 @@ def add_rows(self, rows: Iterator[BigTableRow]) -> None: "columnValues": mutated_row } ) - r = requests.post( - self.url("function/mutateTables"), - headers=self.headers(), - json={ - "appID": self.hardcoded_app_id, - "mutations": mutations - } - ) + r = requests.post( + self.url("function/mutateTables"), + headers=self.headers(), + json={ + "appID": self.hardcoded_app_id, + "mutations": mutations + } + ) if r.status_code != 200: logger.error(f"add rows failed with status {r.status_code}: {r.text}") # nopep8 because https://github.com/hhatto/autopep8/issues/712 r.raise_for_status() # This will raise an HTTPError if the status is 4xx or 5xx diff --git a/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableMutationsStrategy_test.py b/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableMutationsStrategy_test.py index 951cb19d541d2..0c2e9821b2279 100644 --- a/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableMutationsStrategy_test.py +++ b/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableMutationsStrategy_test.py @@ -31,4 +31,29 @@ def test_prepare_table_invalid_column(self, mock_post): Column('this column wont be found', 'string') ] self.gbt.prepare_table(test_columns) + + @patch.object(requests, 'post') + def test_add_rows(self, mock_post): + test_columns = [ + Column('id', 'string'), + Column('this column wont be found', 'string') + ] + + mock_post.return_value.status_code = 200 + + self.gbt.prepare_table(test_columns) + + mock_post.reset_mock() + test_data = [ + { + 'id': '1', + 'col2': 'test name' + }, + { + 'id': '2', + 'col2': 'test name2' + } + ] + self.gbt.add_rows(test_data) + self.assertEqual(1, mock_post.call_count) From 71cf51e5522f4d98432e111756b55003aff4c7b9 Mon Sep 17 00:00:00 2001 From: Scott Willeke Date: Thu, 20 Jun 2024 09:25:24 -0400 Subject: [PATCH 08/39] feat: choose a strategy based on config; unify tests --- .../destination_glide/destination.py | 34 ++-- .../destination_glide/glide.py | 48 +++--- .../destination_glide/spec.json | 5 + .../integration_tests/integration_test.py | 134 --------------- .../destination-glide/scripts/test-all.sh | 14 -- .../scripts/test-integration.sh | 5 - .../destination-glide/test-integration.sh | 5 - .../connectors/destination-glide/todo.md | 5 +- .../unit_tests/destination_test.py | 160 ++++++++++++++++++ 9 files changed, 214 insertions(+), 196 deletions(-) delete mode 100644 airbyte-integrations/connectors/destination-glide/integration_tests/integration_test.py delete mode 100755 airbyte-integrations/connectors/destination-glide/scripts/test-all.sh delete mode 100755 airbyte-integrations/connectors/destination-glide/scripts/test-integration.sh delete mode 100755 airbyte-integrations/connectors/destination-glide/test-integration.sh create mode 100644 airbyte-integrations/connectors/destination-glide/unit_tests/destination_test.py diff --git a/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py b/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py index 564ca762ac16f..f7bcfc85474e2 100644 --- a/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py +++ b/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py @@ -11,9 +11,10 @@ Type ) from collections import defaultdict +from collections.abc import Hashable import datetime -from .glide import GlideBigTableBase, CreateBigTableDefaultImpl, Column +from .glide import Column, GlideBigTableFactory import json from .log import getLogger import logging @@ -21,6 +22,8 @@ from typing import Any, Iterable, Mapping import uuid +CONFIG_GLIDE_API_STRATEGY_DEFAULT = "tables" + logger = getLogger() def mapJsonSchemaTypeToGlideType(json_type: str) -> str: @@ -32,23 +35,21 @@ def mapJsonSchemaTypeToGlideType(json_type: str) -> str: } if isinstance(json_type, list): logger.debug(f"Found list type '{json_type}'. Attempting to map to a primitive type.") # nopep8 because - # find the first type that is not 'null' and use that instead: + # find the first type that is not 'null' and supported and use that instead: for t in json_type: if t != "null" and t in jsonSchemaTypeToGlideType: logger.debug(f"Mapped json schema list type of '{json_type}' to '{t}'.") # nopep8 because json_type = t - - if json_type in jsonSchemaTypeToGlideType: + break + + # NOTE: if json_type is still a list, it won't be Hashable and we can't use it as a key in the dict + if isinstance(json_type, Hashable) and json_type in jsonSchemaTypeToGlideType: return jsonSchemaTypeToGlideType[json_type] - raise ValueError(f"Unsupported JSON schema type for glide '{json_type}'") -class DestinationGlide(Destination): - # GlideBigTable optional for tests to inject mock - def __init__(self, glide: GlideBigTableBase = None): - if glide is None: - glide = CreateBigTableDefaultImpl() - self.glide = glide + logger.warning(f"Unsupported JSON schema type for glide '{json_type}'. Will use string.") + return "string" +class DestinationGlide(Destination): def write( self, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog, input_messages: Iterable[AirbyteMessage] ) -> Iterable[AirbyteMessage]: @@ -66,14 +67,17 @@ def write( :param input_messages: The stream of input messages received from the source :return: Iterable of AirbyteStateMessages wrapped in AirbyteMessage structs """ - # get config: + # load user-specified config: api_host = config['api_host'] api_path_root = config['api_path_root'] api_key = config['api_key'] table_id = config['table_id'] + glide_api_strategy = config.get('glide_api_strategy', CONFIG_GLIDE_API_STRATEGY_DEFAULT) # TODO: choose a strategy based on config - self.glide.init(api_host, api_key, api_path_root, table_id) + glide = GlideBigTableFactory.create(glide_api_strategy) + logger.debug(f"Using glide api strategy '{glide.__class__.__name__}'.") + glide.init(api_host, api_key, api_path_root, table_id) # go through each stream and add it as needed: stream_names = {s.stream.name for s in configured_catalog.streams} @@ -91,7 +95,7 @@ def write( logger.debug(f"Found column/property '{prop_name}' with type '{prop_type}' and format '{prop_format}' in stream {configured_stream.stream.name}.") columns.append(Column(prop_name, mapJsonSchemaTypeToGlideType(prop_type))) - self.glide.prepare_table(columns) + glide.prepare_table(columns) # stream the records into the GBT: buffer = defaultdict(list) @@ -123,7 +127,7 @@ def write( logger.debug(f"Saving buffered records to Glide API (stream: '{stream_name}' count: '{len(stream_records)}')...") # nopep8 because https://github.com/hhatto/autopep8/issues/712 DATA_INDEX = 2 data_rows = [row_tuple[DATA_INDEX] for row_tuple in stream_records] - self.glide.add_rows(data_rows) + glide.add_rows(data_rows) logger.debug(f"Saving buffered records to Glide API complete.") # nopep8 because https://github.com/hhatto/autopep8/issues/712 yield message diff --git a/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py b/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py index 9d01b7bc9f083..49c1e87e62a6d 100644 --- a/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py +++ b/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py @@ -16,34 +16,30 @@ "dateTime", "json", ] - -class Column: + +class Column(dict): + """ + Represents a Column in the glide API. + NOTE: inherits from dict to be serializable to json. + """ def __init__(self, id: str, type: str): if type not in ALLOWED_COLUMN_TYPES: raise ValueError(f"Column type {type} not allowed. Must be one of {ALLOWED_COLUMN_TYPES}") # nopep8 - self._id = id - self._type = type + dict.__init__(self, id=id, type=type, displayName=id) - def id() -> str: - return self._id + def id(self) -> str: + return self['id'] - def type() -> str: - return self._type - - def to_json(self) -> Dict[str, Any]: - return { - 'id': self._id, - 'type': self._type, - 'displayName': self._id - } + def type(self) -> str: + return self['type'] def __eq__(self, other): if isinstance(other, Column): - return self._id == other._id and self._type == other._type + return dict(self) == dict(other) return False def __repr__(self): - return f"Column(id='{self._id}', type='{self._type}')" + return f"Column(id='{self.id()}', type='{self.type()}')" class GlideBigTableBase(ABC): def headers(self) -> Dict[str, str]: @@ -96,15 +92,25 @@ def add_rows(self, rows: Iterator[BigTableRow]) -> None: pass -def CreateBigTableDefaultImpl() -> GlideBigTableBase: +class GlideBigTableFactory: """ - Creates a new instance of the default implementation for the GlideBigTable API client. + Factory for creating a GlideBigTableBase API client. """ - return GlideBigTableMutationsStrategy() + @classmethod + def create(cls, strategy: str) -> GlideBigTableBase: + """ + Creates a new instance of the default implementation for the GlideBigTable API client. + """ + implementation_map = { + "tables": GlideBigTableRestStrategy(), + "mutations": GlideBigTableMutationsStrategy() + } + if strategy not in implementation_map: + raise ValueError(f"Strategy '{strategy}' not found. Expected one of '{implmap.keys()}'.") + return implementation_map[strategy] class GlideBigTableRestStrategy(GlideBigTableBase): - def prepare_table(self, columns: List[Column]) -> None: logger.debug(f"prepare_table columns: {columns}") # update the table: diff --git a/airbyte-integrations/connectors/destination-glide/destination_glide/spec.json b/airbyte-integrations/connectors/destination-glide/destination_glide/spec.json index 66f093c73c456..6d034121b5ede 100644 --- a/airbyte-integrations/connectors/destination-glide/destination_glide/spec.json +++ b/airbyte-integrations/connectors/destination-glide/destination_glide/spec.json @@ -26,6 +26,11 @@ "type": "string", "description": "The ID of the table that the Glide destination should use. If it does not exist it will be created.", "airbyte_secret": true + }, + "glide_api_version": { + "type": "string", + "description": "The Glide API version to use. Options are 'tables' or 'mutations'. Default is 'tables'.", + "enum": ["tables", "mutations"] } } } diff --git a/airbyte-integrations/connectors/destination-glide/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-glide/integration_tests/integration_test.py deleted file mode 100644 index a06fc8b99ba14..0000000000000 --- a/airbyte-integrations/connectors/destination-glide/integration_tests/integration_test.py +++ /dev/null @@ -1,134 +0,0 @@ -# -# Copyright (c) 2024 Airbyte, Inc., all rights reserved. -# -from airbyte_cdk.models import ( - AirbyteMessage, - AirbyteRecordMessage, - AirbyteStateMessage, - AirbyteStream, - ConfiguredAirbyteCatalog, - ConfiguredAirbyteStream, - DestinationSyncMode, - Status, - SyncMode, - Type -) -from datetime import datetime -from destination_glide import DestinationGlide -# for mock: -from destination_glide.glide import GlideBigTableBase - -import json -import logging -import pytest -import random -import string -from typing import Any, Mapping -from unittest.mock import create_autospec - -@pytest.fixture(name="config") -def config_fixture() -> Mapping[str, Any]: - with open("secrets/config.json", "r") as f: - return json.loads(f.read()) - - -@pytest.fixture(scope="module") -def test_table_name() -> str: - letters = string.ascii_lowercase - rand_string = "".join(random.choice(letters) for _ in range(10)) - return f"airbyte_integration_{rand_string}" - - -@pytest.fixture -def table_schema() -> str: - stream_schema = { - "type": "object", - "properties": { - "string_col": {"type": "string"}, - "int_col": {"type": "integer"}, - "date_col": {"type": "string", "format": "date-time"}, - "other_col": {"type": ["null", "string"]} - }, - } - return stream_schema - - -def AirbyteLogger() -> logging.Logger: - return logging.getLogger('airbyte') - -@pytest.fixture -def configured_catalog(test_table_name: str, table_schema: str) -> ConfiguredAirbyteCatalog: - overwrite_stream = ConfiguredAirbyteStream( - # TODO: I'm not sure if we should expect incoming streams SyncMode.incremental and only the destination to be full_refresh or they should - stream=AirbyteStream(name=test_table_name, json_schema=table_schema, - supported_sync_modes=[SyncMode.incremental]), - sync_mode=SyncMode.incremental, - destination_sync_mode=DestinationSyncMode.overwrite, - ) - return ConfiguredAirbyteCatalog(streams=[overwrite_stream]) - -@pytest.fixture -def airbyte_message_record1(test_table_name: str): - return AirbyteMessage( - type=Type.RECORD, - record=AirbyteRecordMessage( - stream=test_table_name, data={"key_str": "value1", "key_int": 3}, emitted_at=int(datetime.now().timestamp()) * 1000 - ), - ) - -@pytest.fixture -def airbyte_message_record2(test_table_name: str): - return AirbyteMessage( - type=Type.RECORD, - record=AirbyteRecordMessage( - stream=test_table_name, data={"key_str": "value2", "key_int": 2}, emitted_at=int(datetime.now().timestamp()) * 1000 - ), - ) - - -@pytest.fixture -def airbyte_message_state(test_table_name: str): - return AirbyteMessage( - type=Type.STATE, - state=AirbyteStateMessage( - data={"opaque": "to destination"} - ) - ) - - -#configured_stream.stream.json_schema["properties"] - -##### Tests Begin Here ##### - - -def test_check_valid_config(config: Mapping): - outcome = DestinationGlide().check(AirbyteLogger(), config) - assert outcome.status == Status.SUCCEEDED - - -def test_write( - config: Mapping, - request, - configured_catalog: ConfiguredAirbyteCatalog, - airbyte_message_record1: AirbyteMessage, - airbyte_message_record2: AirbyteMessage, - airbyte_message_state: AirbyteMessage, - test_table_name: str, -): - mock_gbt = create_autospec(GlideBigTableBase) - - destination = DestinationGlide(mock_gbt) - generator = destination.write( - config=config, configured_catalog=configured_catalog, input_messages=[ - airbyte_message_record1, airbyte_message_record1, airbyte_message_state] - ) - - # expecting only to return the state message: - result = list(generator) - assert len(result) == 1 - - # expect the API was called: - # todo: validate args on these calls - - mock_gbt.prepare_table.assert_called_once() - mock_gbt.add_rows.assert_called_once() diff --git a/airbyte-integrations/connectors/destination-glide/scripts/test-all.sh b/airbyte-integrations/connectors/destination-glide/scripts/test-all.sh deleted file mode 100755 index 49d8dfecff924..0000000000000 --- a/airbyte-integrations/connectors/destination-glide/scripts/test-all.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash -this_dir=$(cd $(dirname "$0"); pwd) # this script's directory -this_script=$(basename $0) - -"$this_dir/test-unit.sh" -if [ $? -ne 0 ]; then - echo "Unit tests failed" - exit 1 -fi -"$this_dir/test-integration.sh" -if [ $? -ne 0 ]; then - echo "Integration tests failed" - exit 1 -fi \ No newline at end of file diff --git a/airbyte-integrations/connectors/destination-glide/scripts/test-integration.sh b/airbyte-integrations/connectors/destination-glide/scripts/test-integration.sh deleted file mode 100755 index b2c0b792415ab..0000000000000 --- a/airbyte-integrations/connectors/destination-glide/scripts/test-integration.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash -this_dir=$(cd $(dirname "$0"); pwd) # this script's directory -this_script=$(basename $0) - -poetry run pytest integration_tests diff --git a/airbyte-integrations/connectors/destination-glide/test-integration.sh b/airbyte-integrations/connectors/destination-glide/test-integration.sh deleted file mode 100755 index b2c0b792415ab..0000000000000 --- a/airbyte-integrations/connectors/destination-glide/test-integration.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash -this_dir=$(cd $(dirname "$0"); pwd) # this script's directory -this_script=$(basename $0) - -poetry run pytest integration_tests diff --git a/airbyte-integrations/connectors/destination-glide/todo.md b/airbyte-integrations/connectors/destination-glide/todo.md index d7bc7c83dc711..0b0915b6cb43b 100644 --- a/airbyte-integrations/connectors/destination-glide/todo.md +++ b/airbyte-integrations/connectors/destination-glide/todo.md @@ -1,7 +1,8 @@ -- [ ] feat: choose a strategy based on config +- [+] feat: choose a strategy based on config - [+] chore: stop writing any record data to logs - [+] chore: cleanup logs generally -- [ ] chore: unify test framework across unit/integration tests (remove pytest?) +- [ ] fix: use friendly names for the properties on the config page +- [+] chore: unify test framework across unit/integration tests (remove pytest?) - [ ] chore: clean up todos - [ ] chore: update readme - [ ] chore: figure out why the dev-write script stopped working (freezes) diff --git a/airbyte-integrations/connectors/destination-glide/unit_tests/destination_test.py b/airbyte-integrations/connectors/destination-glide/unit_tests/destination_test.py new file mode 100644 index 0000000000000..1a6d67bccdeb4 --- /dev/null +++ b/airbyte-integrations/connectors/destination-glide/unit_tests/destination_test.py @@ -0,0 +1,160 @@ +from airbyte_cdk.models import ( + AirbyteMessage, + AirbyteRecordMessage, + AirbyteStateMessage, + AirbyteStream, + ConfiguredAirbyteCatalog, + ConfiguredAirbyteStream, + DestinationSyncMode, + Status, + SyncMode, + Type +) +from datetime import datetime +from destination_glide import DestinationGlide +from destination_glide.glide import GlideBigTableBase, GlideBigTableFactory +import json +import logging +import random +import string +from typing import Any, Mapping, Callable +import unittest +from unittest.mock import patch, create_autospec + + +def create_config() -> Mapping[str, Any]: + with open("secrets/config.json", "r") as f: + return json.loads(f.read()) + + +def create_test_table_name() -> str: + letters = string.ascii_lowercase + rand_string = "".join(random.choice(letters) for _ in range(10)) + return f"test_table_{rand_string}" + + +def table_schema() -> str: + stream_schema = { + "type": "object", + "properties": { + "string_col": {"type": "string"}, + "int_col": {"type": "integer"}, + "date_col": {"type": "string", "format": "date-time"}, + "other_col": {"type": ["null", "string"]} + }, + } + return stream_schema + + +def AirbyteLogger() -> logging.Logger: + return logging.getLogger('airbyte') + + +def configured_catalog(test_table_name: str, table_schema: str) -> ConfiguredAirbyteCatalog: + overwrite_stream = ConfiguredAirbyteStream( + # TODO: I'm not sure if we should expect incoming streams SyncMode.incremental and only the destination to be full_refresh or they should + stream=AirbyteStream(name=test_table_name, json_schema=table_schema, + supported_sync_modes=[SyncMode.incremental]), + sync_mode=SyncMode.incremental, + destination_sync_mode=DestinationSyncMode.overwrite, + ) + return ConfiguredAirbyteCatalog(streams=[overwrite_stream]) + + +def airbyte_message_record1(test_table_name: str): + return AirbyteMessage( + type=Type.RECORD, + record=AirbyteRecordMessage( + stream=test_table_name, data={"key_str": "value1", "key_int": 3}, emitted_at=int(datetime.now().timestamp()) * 1000 + ), + ) + + +def airbyte_message_record2(test_table_name: str): + return AirbyteMessage( + type=Type.RECORD, + record=AirbyteRecordMessage( + stream=test_table_name, data={"key_str": "value2", "key_int": 2}, emitted_at=int(datetime.now().timestamp()) * 1000 + ), + ) + + +def airbyte_message_state(test_table_name: str): + return AirbyteMessage( + type=Type.STATE, + state=AirbyteStateMessage( + data={"opaque": "to destination"} + ) + ) + + +def CreateMockGlideBigTable(): + return create_autospec(GlideBigTableBase) + + +class TestDestinationGlide(unittest.TestCase): + api_host = 'https://test-api-host.com' + api_key = 'test-api-key' + api_path_root = '/test/api/path/root' + table_id = 'test-table-id' + + def setUp(self): + self.test_table_name = create_test_table_name() + + def test_check_with_valid_config(config: Mapping): + outcome = DestinationGlide().check(AirbyteLogger(), config) + assert outcome.status == Status.SUCCEEDED + + @patch.object(GlideBigTableFactory, "create") + def test_write(self, mock_factory: Callable): + mock_bigtable = CreateMockGlideBigTable() + mock_factory.return_value = mock_bigtable + + destination = DestinationGlide() + + generator = destination.write( + config=create_config(), configured_catalog=configured_catalog(self.test_table_name, table_schema=table_schema()), input_messages=[ + airbyte_message_record1(self.test_table_name), airbyte_message_record2(self.test_table_name), airbyte_message_state(self.test_table_name)] + ) + + # expecting only to return the state message: + result = list(generator) + assert len(result) == 1 + + # todo: validate args on these calls + mock_bigtable.prepare_table.assert_called_once() + mock_bigtable.add_rows.assert_called_once() + + @patch.object(GlideBigTableFactory, "create") + def test_with_invalid_column_types(self, mock_factory: Callable): + mock_bigtable = CreateMockGlideBigTable() + mock_factory.return_value = mock_bigtable + + destination = DestinationGlide() + + test_schema = { + "type": "object", + "properties": { + "int_col": {"type": "integer"}, + # NOTE: null and object not supported by glide so this fails + "obj_null_col": {"type": ["null", "object"]}, + "date_col": {"type": "string", "format": "date-time"}, + }, + } + + generator = destination.write( + config=create_config(), configured_catalog=configured_catalog(self.test_table_name, table_schema=test_schema), input_messages=[ + airbyte_message_record1(self.test_table_name), airbyte_message_record2(self.test_table_name), airbyte_message_state(self.test_table_name)] + ) + + # expecting only to return the state message: + result = list(generator) + assert len(result) == 1 + + mock_bigtable.prepare_table.assert_called_once() + mock_bigtable.add_rows.assert_called_once() + # get the columns we passed into teh API and verify the type defaulted to string: + prepared_cols = mock_bigtable.prepare_table.call_args[0][0] + null_column = [col for col in prepared_cols if col.id() + == "obj_null_col"][0] + self.assertEqual(null_column.type(), "string") From 94ca0bc909db8eb7bd229ae1793c1eba60a814fb Mon Sep 17 00:00:00 2001 From: Scott Willeke Date: Thu, 20 Jun 2024 10:09:07 -0400 Subject: [PATCH 09/39] fix: glide_api_version configuration was not honored --- .../destination_glide/destination.py | 8 ++--- .../unit_tests/destination_test.py | 34 ++++++++++++++++--- 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py b/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py index f7bcfc85474e2..9ac45943db2d7 100644 --- a/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py +++ b/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py @@ -22,7 +22,7 @@ from typing import Any, Iterable, Mapping import uuid -CONFIG_GLIDE_API_STRATEGY_DEFAULT = "tables" +CONFIG_GLIDE_API_VERSION_DEFAULT = "tables" logger = getLogger() @@ -72,11 +72,11 @@ def write( api_path_root = config['api_path_root'] api_key = config['api_key'] table_id = config['table_id'] - glide_api_strategy = config.get('glide_api_strategy', CONFIG_GLIDE_API_STRATEGY_DEFAULT) + glide_api_version = config.get('glide_api_version', CONFIG_GLIDE_API_VERSION_DEFAULT) # TODO: choose a strategy based on config - glide = GlideBigTableFactory.create(glide_api_strategy) - logger.debug(f"Using glide api strategy '{glide.__class__.__name__}'.") + glide = GlideBigTableFactory.create(glide_api_version) + logger.debug(f"Using glide api strategy '{glide.__class__.__name__}' for glide_api_version '{glide_api_version}'.") glide.init(api_host, api_key, api_path_root, table_id) # go through each stream and add it as needed: diff --git a/airbyte-integrations/connectors/destination-glide/unit_tests/destination_test.py b/airbyte-integrations/connectors/destination-glide/unit_tests/destination_test.py index 1a6d67bccdeb4..4bb3dee1aa465 100644 --- a/airbyte-integrations/connectors/destination-glide/unit_tests/destination_test.py +++ b/airbyte-integrations/connectors/destination-glide/unit_tests/destination_test.py @@ -19,10 +19,10 @@ import string from typing import Any, Mapping, Callable import unittest -from unittest.mock import patch, create_autospec +from unittest.mock import patch, create_autospec, Mock -def create_config() -> Mapping[str, Any]: +def create_default_config() -> Mapping[str, Any]: with open("secrets/config.json", "r") as f: return json.loads(f.read()) @@ -113,7 +113,7 @@ def test_write(self, mock_factory: Callable): destination = DestinationGlide() generator = destination.write( - config=create_config(), configured_catalog=configured_catalog(self.test_table_name, table_schema=table_schema()), input_messages=[ + config=create_default_config(), configured_catalog=configured_catalog(self.test_table_name, table_schema=table_schema()), input_messages=[ airbyte_message_record1(self.test_table_name), airbyte_message_record2(self.test_table_name), airbyte_message_state(self.test_table_name)] ) @@ -143,7 +143,7 @@ def test_with_invalid_column_types(self, mock_factory: Callable): } generator = destination.write( - config=create_config(), configured_catalog=configured_catalog(self.test_table_name, table_schema=test_schema), input_messages=[ + config=create_default_config(), configured_catalog=configured_catalog(self.test_table_name, table_schema=test_schema), input_messages=[ airbyte_message_record1(self.test_table_name), airbyte_message_record2(self.test_table_name), airbyte_message_state(self.test_table_name)] ) @@ -158,3 +158,29 @@ def test_with_invalid_column_types(self, mock_factory: Callable): null_column = [col for col in prepared_cols if col.id() == "obj_null_col"][0] self.assertEqual(null_column.type(), "string") + + @patch.object(GlideBigTableFactory, "create") + def test_api_version_passes_correct_strategy(self, mock_factory: Mock): + mock_bigtable = CreateMockGlideBigTable() + mock_factory.return_value = mock_bigtable + + + + config = { + "api_host": "foo", + "api_path_root": "bar", + "api_key": "baz", + "table_id": "buz", + "glide_api_version": "mutations" + } + + destination = DestinationGlide() + generator = destination.write( + config=config, configured_catalog=configured_catalog(self.test_table_name, table_schema=table_schema()), input_messages=[ + airbyte_message_record1(self.test_table_name), airbyte_message_record2(self.test_table_name), airbyte_message_state(self.test_table_name)] + ) + # expecting only to return the state message: + result = list(generator) + + passed_strategy = mock_factory.call_args[0][0] + self.assertEqual("mutations", passed_strategy) From 2fdcd4f7e3e4e644dee602a6cdda9a28d14b88bc Mon Sep 17 00:00:00 2001 From: Scott Willeke Date: Thu, 20 Jun 2024 12:10:31 -0400 Subject: [PATCH 10/39] fix: batch row adds in rest api --- .../destination_glide/glide.py | 24 +++++++++++++++---- .../connectors/destination-glide/todo.md | 4 ++++ .../GlideBigTableRestStrategy_test.py | 17 ++++++++++--- 3 files changed, 38 insertions(+), 7 deletions(-) diff --git a/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py b/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py index 49c1e87e62a6d..06e20536858a9 100644 --- a/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py +++ b/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py @@ -139,6 +139,8 @@ def rows(self) -> Iterator[BigTableRow]: result = r.json() + # TODO: Paging?? + # the result looks like an array of results; each result has a rows member that has an array or JSON rows: for row in result: for r in row['rows']: @@ -148,18 +150,32 @@ def delete_all(self) -> None: logger.warning(f"delete_all call is ignored in {type(self).__class__.__name__}") # nopep8 pass - def add_rows(self, rows: Iterator[BigTableRow]) -> None: + def _add_row_batch(self, rows: List[BigTableRow]) -> None: + logger.debug(f"Adding rows batch with size {len(rows)}") r = requests.post( self.url(f"/tables/{self.table_id}/rows"), headers=self.headers(), json={ - "rows": list(rows) + "rows": rows } ) if r.status_code != 200: - logger.error(f"get rows request failed with status {r.status_code}: {r.text}.") # nopep8 because https://github.com/hhatto/autopep8/issues/712 - r.raise_for_status() # This will raise an HTTPError if the status is 4xx or 5xx + logger.error(f"add rows batch failed with status {r.status_code}: {r.text}") + else: + logger.debug(f"add rows batch succeeded") + def add_rows(self, rows: Iterator[BigTableRow]) -> None: + BATCH_SIZE = 100 + + batch = [] + for row in rows: + batch.append(row) + if len(batch) >= BATCH_SIZE: + self._add_row_batch(batch) + batch = [] + + if len(batch) > 0: + self._add_row_batch(batch) class GlideBigTableMutationsStrategy(GlideBigTableBase): def __init__(self): diff --git a/airbyte-integrations/connectors/destination-glide/todo.md b/airbyte-integrations/connectors/destination-glide/todo.md index 0b0915b6cb43b..dc72ca98771bf 100644 --- a/airbyte-integrations/connectors/destination-glide/todo.md +++ b/airbyte-integrations/connectors/destination-glide/todo.md @@ -1,8 +1,12 @@ - [+] feat: choose a strategy based on config - [+] chore: stop writing any record data to logs - [+] chore: cleanup logs generally +- [ ] fix: "add rows failed with status 400: {"message":"More than 500 mutations"}" in mutation api (batch them to ~100 or something) +- [+] fix: batch row adds in rest api - [ ] fix: use friendly names for the properties on the config page +- [ ] fix: ensure table_id is not optional (or come up with a way to consistently name tables to find it again and use that?) - [+] chore: unify test framework across unit/integration tests (remove pytest?) - [ ] chore: clean up todos - [ ] chore: update readme - [ ] chore: figure out why the dev-write script stopped working (freezes) +- [ ] feat: make the check verify api connection and config settings diff --git a/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableRestStrategy_test.py b/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableRestStrategy_test.py index f3ec91fa6e954..4178b0ba39ad0 100644 --- a/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableRestStrategy_test.py +++ b/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableRestStrategy_test.py @@ -28,7 +28,8 @@ def test_prepare_table_valid(self, mock_put): self.gbt.prepare_table(test_columns) mock_put.assert_called_once() - self.assertListEqual(mock_put.call_args[1]['json']['schema']['columns'], test_columns) + self.assertListEqual( + mock_put.call_args[1]['json']['schema']['columns'], test_columns) @patch.object(requests, 'put') def test_prepare_table_invalid_col_type(self, mock_put): @@ -47,14 +48,24 @@ def test_add_rows(self, mock_post): mock_post.return_value.json.return_value = {'data': 'test'} test_rows = [ - Column('strcol', "string"), - Column('numcol', "number") + {"strcol": "one", "numcol": 1}, + {"strcol": "two", "numcol": 2} ] self.gbt.add_rows(test_rows) mock_post.assert_called_once() assert mock_post.call_args[1]['json']['rows'] == test_rows + @patch.object(requests, 'post') + def test_add_rows_batching(self, mock_post): + mock_post.return_value.status_code = 200 + mock_post.return_value.json.return_value = {'data': 'test'} + + test_rows = list([{"strcol": f"{i}", "numcol": i} for i in range(1000)]) + + self.gbt.add_rows(test_rows) + + self.assertEqual(10, mock_post.call_count) if __name__ == '__main__': unittest.main() From 5ea2963e22d33ad370133d688fa76d8d28ef393b Mon Sep 17 00:00:00 2001 From: Scott Willeke Date: Thu, 20 Jun 2024 12:19:41 -0400 Subject: [PATCH 11/39] fix: batching row adds in mutations api to prevent error: add rows failed with status 400: {"message":"More than 500 mutations"}" in mutation api --- .../destination_glide/glide.py | 16 ++++++++++++++-- .../GlideBigTableMutationsStrategy_test.py | 17 +++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py b/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py index 06e20536858a9..e2cc10e652ecd 100644 --- a/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py +++ b/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py @@ -278,8 +278,7 @@ def delete_all(self) -> None: else: logger.debug(f"Deleted row successfully (rowID:'{row['$rowID']}'") - def add_rows(self, rows: Iterator[BigTableRow]) -> None: - # TODO: lame. need to batch mutations/requests + def add_rows_batch(self, rows: Iterator[BigTableRow]) -> None: mutations = [] for row in rows: # row is columnLabel -> value, but glide's mutate uses a column "name". We hard-code the lookup for our table here: @@ -310,3 +309,16 @@ def add_rows(self, rows: Iterator[BigTableRow]) -> None: if r.status_code != 200: logger.error(f"add rows failed with status {r.status_code}: {r.text}") # nopep8 because https://github.com/hhatto/autopep8/issues/712 r.raise_for_status() # This will raise an HTTPError if the status is 4xx or 5xx + + def add_rows(self, rows: Iterator[BigTableRow]) -> None: + BATCH_SIZE = 100 + + batch = [] + for row in rows: + batch.append(row) + if len(batch) >= BATCH_SIZE: + self.add_rows_batch(batch) + batch = [] + + if len(batch) > 0: + self.add_rows_batch(batch) diff --git a/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableMutationsStrategy_test.py b/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableMutationsStrategy_test.py index 0c2e9821b2279..360098a64513c 100644 --- a/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableMutationsStrategy_test.py +++ b/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableMutationsStrategy_test.py @@ -57,3 +57,20 @@ def test_add_rows(self, mock_post): self.gbt.add_rows(test_data) self.assertEqual(1, mock_post.call_count) + @patch.object(requests, 'post') + def test_add_rows_batch(self, mock_post): + test_columns = [ + Column('strcol', 'string'), + Column('numcol', 'number') + ] + + mock_post.return_value.status_code = 200 + + self.gbt.prepare_table(test_columns) + + mock_post.reset_mock() + test_rows = list([{"strcol": f"{i}", "numcol": i} for i in range(1000)]) + self.gbt.add_rows(test_rows) + + self.assertEqual(10, mock_post.call_count) + From 24468cc377b4b20a47453522daa8ba65decd937f Mon Sep 17 00:00:00 2001 From: Scott Willeke Date: Thu, 20 Jun 2024 18:16:38 -0400 Subject: [PATCH 12/39] fix: update api host config to include protocol --- .../destination-glide/destination_glide/destination.py | 3 ++- .../connectors/destination-glide/destination_glide/glide.py | 4 ++-- .../connectors/destination-glide/destination_glide/spec.json | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py b/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py index 9ac45943db2d7..2b5fee2bb9620 100644 --- a/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py +++ b/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py @@ -23,6 +23,7 @@ import uuid CONFIG_GLIDE_API_VERSION_DEFAULT = "tables" +CONFIG_GLIDE_API_HOST_DEFAULT = "https://api.glideapp.io" logger = getLogger() @@ -68,7 +69,7 @@ def write( :return: Iterable of AirbyteStateMessages wrapped in AirbyteMessage structs """ # load user-specified config: - api_host = config['api_host'] + api_host = config.get('api_host', CONFIG_GLIDE_API_HOST_DEFAULT) api_path_root = config['api_path_root'] api_key = config['api_key'] table_id = config['table_id'] diff --git a/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py b/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py index e2cc10e652ecd..b5e59e56c45df 100644 --- a/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py +++ b/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py @@ -49,7 +49,7 @@ def headers(self) -> Dict[str, str]: } def url(self, path: str) -> str: - return f"https://{self.api_host}/{self.api_path_root}/{path}" + return f"{self.api_host}/{self.api_path_root}/{path}" """ An API client for interacting with a Glide Big Table. @@ -212,7 +212,7 @@ def headers(self) -> Dict[str, str]: } def url(self, path: str) -> str: - return f"https://{self.api_host}/{self.api_path_root}/{path}" + return f"{self.api_host}/{self.api_path_root}/{path}" def prepare_table(self, columns: List[Column]) -> None: logger.debug(f"prepare_table for table '{self.table_id}. Expecting columns: '{[c.id for c in columns]}'.") diff --git a/airbyte-integrations/connectors/destination-glide/destination_glide/spec.json b/airbyte-integrations/connectors/destination-glide/destination_glide/spec.json index 6d034121b5ede..ad2ad7288216e 100644 --- a/airbyte-integrations/connectors/destination-glide/destination_glide/spec.json +++ b/airbyte-integrations/connectors/destination-glide/destination_glide/spec.json @@ -11,7 +11,7 @@ "properties": { "api_host": { "type": "string", - "description": "The host name of the Glide API destination" + "description": "The protocol and host name (e.g. 'https://api.glideapp.io'). of the Glide API destination" }, "api_path_root": { "type": "string", From e24d8a64df16a6c488dcc1cc6b4845cd7cdd7119 Mon Sep 17 00:00:00 2001 From: Scott Willeke Date: Thu, 20 Jun 2024 18:53:16 -0400 Subject: [PATCH 13/39] fix: serialize columns for prepare tables with kind --- .../connectors/destination-glide/destination_glide/glide.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py b/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py index b5e59e56c45df..6b37c3343fe30 100644 --- a/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py +++ b/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py @@ -25,7 +25,7 @@ class Column(dict): def __init__(self, id: str, type: str): if type not in ALLOWED_COLUMN_TYPES: raise ValueError(f"Column type {type} not allowed. Must be one of {ALLOWED_COLUMN_TYPES}") # nopep8 - dict.__init__(self, id=id, type=type, displayName=id) + dict.__init__(self, id=id, type={"kind":type}, displayName=id) def id(self) -> str: return self['id'] From 7138915ef518fd0681eba186bba7e380af8e8367 Mon Sep 17 00:00:00 2001 From: Scott Willeke Date: Thu, 20 Jun 2024 19:09:47 -0400 Subject: [PATCH 14/39] fix: add option to ignore columns --- .../destination-glide/destination_glide/glide.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py b/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py index 6b37c3343fe30..076f9ea4b6793 100644 --- a/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py +++ b/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py @@ -31,7 +31,8 @@ def id(self) -> str: return self['id'] def type(self) -> str: - return self['type'] + # NOTE: we serialize this as {kind: ""} per the rest API's serialization + return self['type']['kind'] def __eq__(self, other): if isinstance(other, Column): @@ -156,7 +157,11 @@ def _add_row_batch(self, rows: List[BigTableRow]) -> None: self.url(f"/tables/{self.table_id}/rows"), headers=self.headers(), json={ - "rows": rows + "rows": rows, + "options": { + "unknownColumns": "ignore" + } + } ) if r.status_code != 200: From b044d2b62fb65c907bfb8edc8e21202f7300b847 Mon Sep 17 00:00:00 2001 From: Scott Willeke Date: Thu, 20 Jun 2024 20:15:42 -0400 Subject: [PATCH 15/39] chore: remove unused rows code --- .../destination_glide/glide.py | 57 ++++++------------- .../connectors/destination-glide/todo.md | 2 +- .../GlideBigTableRestStrategy_test.py | 2 +- 3 files changed, 20 insertions(+), 41 deletions(-) diff --git a/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py b/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py index 076f9ea4b6793..5f6fbde618f6f 100644 --- a/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py +++ b/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py @@ -16,16 +16,18 @@ "dateTime", "json", ] - + + class Column(dict): """ Represents a Column in the glide API. NOTE: inherits from dict to be serializable to json. """ + def __init__(self, id: str, type: str): if type not in ALLOWED_COLUMN_TYPES: raise ValueError(f"Column type {type} not allowed. Must be one of {ALLOWED_COLUMN_TYPES}") # nopep8 - dict.__init__(self, id=id, type={"kind":type}, displayName=id) + dict.__init__(self, id=id, type={"kind": type}, displayName=id) def id(self) -> str: return self['id'] @@ -42,6 +44,7 @@ def __eq__(self, other): def __repr__(self): return f"Column(id='{self.id()}', type='{self.type()}')" + class GlideBigTableBase(ABC): def headers(self) -> Dict[str, str]: return { @@ -72,13 +75,6 @@ def prepare_table(self, columns: List[Column]) -> None: """ pass - @abstractmethod - def rows(self) -> Iterator[BigTableRow]: - """ - Gets the rows as of the Glide Big Table. - """ - pass - @abstractmethod def delete_all(self) -> None: """ @@ -107,7 +103,7 @@ def create(cls, strategy: str) -> GlideBigTableBase: "mutations": GlideBigTableMutationsStrategy() } if strategy not in implementation_map: - raise ValueError(f"Strategy '{strategy}' not found. Expected one of '{implmap.keys()}'.") + raise ValueError(f"Strategy '{strategy}' not found. Expected one of '{implmap.keys()}'.") # nopep8 return implementation_map[strategy] @@ -129,24 +125,6 @@ def prepare_table(self, columns: List[Column]) -> None: if r.status_code != 200: logger.error(f"prepare table request failed with status {r.status_code}: {r.text}.") # nopep8 - def rows(self) -> Iterator[BigTableRow]: - r = requests.get( - self.url(f"/tables/{self.table_id}/rows"), - headers=self.headers(), - ) - if r.status_code != 200: - logger.error(f"get rows request failed with status {r.status_code}: {r.text}.") # nopep8 because https://github.com/hhatto/autopep8/issues/712 - r.raise_for_status() # This will raise an HTTPError if the status is 4xx or 5xx - - result = r.json() - - # TODO: Paging?? - - # the result looks like an array of results; each result has a rows member that has an array or JSON rows: - for row in result: - for r in row['rows']: - yield r - def delete_all(self) -> None: logger.warning(f"delete_all call is ignored in {type(self).__class__.__name__}") # nopep8 pass @@ -161,27 +139,27 @@ def _add_row_batch(self, rows: List[BigTableRow]) -> None: "options": { "unknownColumns": "ignore" } - } ) if r.status_code != 200: - logger.error(f"add rows batch failed with status {r.status_code}: {r.text}") + logger.error(f"add rows batch failed with status {r.status_code}: {r.text}") # nopep8 else: logger.debug(f"add rows batch succeeded") def add_rows(self, rows: Iterator[BigTableRow]) -> None: - BATCH_SIZE = 100 - + BATCH_SIZE = 500 + batch = [] for row in rows: batch.append(row) if len(batch) >= BATCH_SIZE: self._add_row_batch(batch) batch = [] - + if len(batch) > 0: self._add_row_batch(batch) + class GlideBigTableMutationsStrategy(GlideBigTableBase): def __init__(self): # TODO: hardcoded for now using old api @@ -220,7 +198,7 @@ def url(self, path: str) -> str: return f"{self.api_host}/{self.api_path_root}/{path}" def prepare_table(self, columns: List[Column]) -> None: - logger.debug(f"prepare_table for table '{self.table_id}. Expecting columns: '{[c.id for c in columns]}'.") + logger.debug(f"prepare_table for table '{self.table_id}. Expecting columns: '{[c.id for c in columns]}'.") # nopep8 self.delete_all() for col in columns: @@ -279,9 +257,10 @@ def delete_all(self) -> None: ) if r.status_code != 200: logger.error(f"delete request failed with status {r.status_code}: {r.text} trying to delete row id {row['$rowID']} with row: {row}") # nopep8 because https://github.com/hhatto/autopep8/issues/712 - r.raise_for_status() # This will raise an HTTPError if the status is 4xx or 5xx + r.raise_for_status() # This will raise an HTTPError if the status is 4xx or 5xx else: - logger.debug(f"Deleted row successfully (rowID:'{row['$rowID']}'") + logger.debug( + f"Deleted row successfully (rowID:'{row['$rowID']}'") def add_rows_batch(self, rows: Iterator[BigTableRow]) -> None: mutations = [] @@ -314,16 +293,16 @@ def add_rows_batch(self, rows: Iterator[BigTableRow]) -> None: if r.status_code != 200: logger.error(f"add rows failed with status {r.status_code}: {r.text}") # nopep8 because https://github.com/hhatto/autopep8/issues/712 r.raise_for_status() # This will raise an HTTPError if the status is 4xx or 5xx - + def add_rows(self, rows: Iterator[BigTableRow]) -> None: BATCH_SIZE = 100 - + batch = [] for row in rows: batch.append(row) if len(batch) >= BATCH_SIZE: self.add_rows_batch(batch) batch = [] - + if len(batch) > 0: self.add_rows_batch(batch) diff --git a/airbyte-integrations/connectors/destination-glide/todo.md b/airbyte-integrations/connectors/destination-glide/todo.md index dc72ca98771bf..3ec906184c382 100644 --- a/airbyte-integrations/connectors/destination-glide/todo.md +++ b/airbyte-integrations/connectors/destination-glide/todo.md @@ -1,7 +1,7 @@ - [+] feat: choose a strategy based on config - [+] chore: stop writing any record data to logs - [+] chore: cleanup logs generally -- [ ] fix: "add rows failed with status 400: {"message":"More than 500 mutations"}" in mutation api (batch them to ~100 or something) +- [+] fix: "add rows failed with status 400: {"message":"More than 500 mutations"}" in mutation api (batch them to ~100 or something) - [+] fix: batch row adds in rest api - [ ] fix: use friendly names for the properties on the config page - [ ] fix: ensure table_id is not optional (or come up with a way to consistently name tables to find it again and use that?) diff --git a/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableRestStrategy_test.py b/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableRestStrategy_test.py index 4178b0ba39ad0..f68c76681c3f4 100644 --- a/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableRestStrategy_test.py +++ b/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableRestStrategy_test.py @@ -61,7 +61,7 @@ def test_add_rows_batching(self, mock_post): mock_post.return_value.status_code = 200 mock_post.return_value.json.return_value = {'data': 'test'} - test_rows = list([{"strcol": f"{i}", "numcol": i} for i in range(1000)]) + test_rows = list([{"strcol": f"{i}", "numcol": i} for i in range(5000)]) self.gbt.add_rows(test_rows) From 8de46e977e5b00b8f5c028f38fb75189334d88d5 Mon Sep 17 00:00:00 2001 From: Scott Willeke Date: Fri, 21 Jun 2024 22:49:52 -0400 Subject: [PATCH 16/39] feat: snuggle rows in a batch --- .../destination_glide/destination.py | 122 +++++++++------- .../destination_glide/glide.py | 137 ++++++++++++------ .../destination_glide/log.py | 2 +- .../destination-glide/scripts/test-unit.sh | 2 +- .../connectors/destination-glide/todo.md | 14 +- .../GlideBigTableMutationsStrategy_test.py | 12 +- .../GlideBigTableRestStrategy_test.py | 130 ++++++++++++----- .../unit_tests/destination_test.py | 124 ++++++++++++---- 8 files changed, 369 insertions(+), 174 deletions(-) diff --git a/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py b/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py index 2b5fee2bb9620..4b0622a967b42 100644 --- a/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py +++ b/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py @@ -27,29 +27,31 @@ logger = getLogger() + def mapJsonSchemaTypeToGlideType(json_type: str) -> str: jsonSchemaTypeToGlideType = { - "string":"string", + "string": "string", "number": "number", "integer": "number", - "boolean":"boolean", + "boolean": "boolean", } if isinstance(json_type, list): - logger.debug(f"Found list type '{json_type}'. Attempting to map to a primitive type.") # nopep8 because + logger.debug(f"Found list type '{json_type}'. Attempting to map to a primitive type.") # nopep8 # find the first type that is not 'null' and supported and use that instead: for t in json_type: if t != "null" and t in jsonSchemaTypeToGlideType: - logger.debug(f"Mapped json schema list type of '{json_type}' to '{t}'.") # nopep8 because + logger.debug(f"Mapped json schema list type of '{json_type}' to '{t}'.") # nopep8 json_type = t break - + # NOTE: if json_type is still a list, it won't be Hashable and we can't use it as a key in the dict if isinstance(json_type, Hashable) and json_type in jsonSchemaTypeToGlideType: return jsonSchemaTypeToGlideType[json_type] - logger.warning(f"Unsupported JSON schema type for glide '{json_type}'. Will use string.") + logger.warning(f"Unsupported JSON schema type for glide '{json_type}'. Will use string.") # nopep8 return "string" + class DestinationGlide(Destination): def write( self, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog, input_messages: Iterable[AirbyteMessage] @@ -73,67 +75,82 @@ def write( api_path_root = config['api_path_root'] api_key = config['api_key'] table_id = config['table_id'] - glide_api_version = config.get('glide_api_version', CONFIG_GLIDE_API_VERSION_DEFAULT) + glide_api_version = config.get( + 'glide_api_version', CONFIG_GLIDE_API_VERSION_DEFAULT) - # TODO: choose a strategy based on config + # choose a strategy based on config: glide = GlideBigTableFactory.create(glide_api_version) - logger.debug(f"Using glide api strategy '{glide.__class__.__name__}' for glide_api_version '{glide_api_version}'.") + logger.debug(f"Using glide api strategy '{glide.__class__.__name__}' for glide_api_version '{glide_api_version}'.") # nopep8 glide.init(api_host, api_key, api_path_root, table_id) - # go through each stream and add it as needed: + if len(configured_catalog.streams) > 1: + # TODO: Consider how we might want to support multiple streams. Review other airbyte destinations + raise Exception(f"The Glide destination expects only a single stream to be streamed into a Glide Table but found '{len(configured_catalog.streams)}' streams. Please select only one stream from the source.") # nopep8 + + # configure the table based on the stream catalog: stream_names = {s.stream.name for s in configured_catalog.streams} for configured_stream in configured_catalog.streams: if configured_stream.destination_sync_mode != DestinationSyncMode.overwrite: - raise Exception(f'Only destination sync mode overwrite is supported, but received "{configured_stream.destination_sync_mode}".') # nopep8 because https://github.com/hhatto/autopep8/issues/712 - - # upsert the GBT with schema to prepare for dumping the data into it + raise Exception(f'Only destination sync mode overwrite is supported, but received "{configured_stream.destination_sync_mode}".') # nopep8 because https://github.com/hhatto/autopep8/issues/712 + + # upsert the GBT with schema to set_schema for dumping the data into it columns = [] properties = configured_stream.stream.json_schema["properties"] for prop_name in properties.keys(): prop = properties[prop_name] prop_type = prop["type"] prop_format = prop["format"] if "format" in prop else "" - logger.debug(f"Found column/property '{prop_name}' with type '{prop_type}' and format '{prop_format}' in stream {configured_stream.stream.name}.") - columns.append(Column(prop_name, mapJsonSchemaTypeToGlideType(prop_type))) - - glide.prepare_table(columns) - - # stream the records into the GBT: - buffer = defaultdict(list) - logger.debug("Processing messages...") - for message in input_messages: - logger.debug(f"processing message {message.type}...") - if message.type == Type.RECORD: - logger.debug("buffering record...") - data = message.record.data - stream = message.record.stream - if stream not in stream_names: - logger.warning( - f"Stream {stream} was not present in configured streams, skipping") - continue - - # TODO: Check the columns match the columns that we saw in configured_catalog per https://docs.airbyte.com/understanding-airbyte/airbyte-protocol#destination - - # add to buffer - record_id = str(uuid.uuid4()) - stream_buffer = buffer[stream] - stream_buffer.append( - (record_id, datetime.datetime.now().isoformat(), data)) - logger.debug(f"buffering record complete.") # nopep8 because https://github.com/hhatto/autopep8/issues/712 - - elif message.type == Type.STATE: - # This is a queue from the source that we should save the buffer of records from message.type == Type.RECORD messages. See https://docs.airbyte.com/understanding-airbyte/airbyte-protocol#state--the-whole-sync - for stream_name in buffer.keys(): - stream_records = buffer[stream_name] - logger.debug(f"Saving buffered records to Glide API (stream: '{stream_name}' count: '{len(stream_records)}')...") # nopep8 because https://github.com/hhatto/autopep8/issues/712 - DATA_INDEX = 2 - data_rows = [row_tuple[DATA_INDEX] for row_tuple in stream_records] + logger.debug(f"Found column/property '{prop_name}' with type '{prop_type}' and format '{prop_format}' in stream {configured_stream.stream.name}.") # nopep8 + columns.append( + Column(prop_name, mapJsonSchemaTypeToGlideType(prop_type))) + + glide.set_schema(columns) + + # stream the records into the GBT: + buffers = defaultdict(list) + logger.debug("Processing messages...") + for message in input_messages: + logger.debug(f"processing message {message.type}...") + if message.type == Type.RECORD: + logger.debug("buffering record...") + stream_name = message.record.stream + if stream_name not in stream_names: + logger.warning( + f"Stream {stream_name} was not present in configured streams, skipping") + continue + + # TODO: check the columns match the columns that we saw in configured_catalog per https://docs.airbyte.com/understanding-airbyte/airbyte-protocol#destination + # add to buffer + record_data = message.record.data + record_id = str(uuid.uuid4()) + stream_buffer = buffers[stream_name] + stream_buffer.append( + (record_id, datetime.datetime.now().isoformat(), record_data)) + logger.debug("buffering record complete.") + + elif message.type == Type.STATE: + # `Type.State` is a signal from the source that we should save the previous batch of `Type.RECORD` messages to the destination. + # It is a checkpoint that enables partial success. + # See https://docs.airbyte.com/understanding-airbyte/airbyte-protocol#state--checkpointing + logger.info(f"Writing buffered records to Glide API from {len(buffers.keys())} streams...") # nopep8 because + for stream_name in buffers.keys(): + stream_buffer = buffers[stream_name] + logger.info(f"Saving buffered records to Glide API (stream: '{stream_name}' count: '{len(stream_buffer)}')...") # nopep8 because https://github.com/hhatto/autopep8/issues/712 + DATA_INDEX = 2 + data_rows = [row_tuple[DATA_INDEX] + for row_tuple in stream_buffer] + if len(data_rows) > 0: glide.add_rows(data_rows) - logger.debug(f"Saving buffered records to Glide API complete.") # nopep8 because https://github.com/hhatto/autopep8/issues/712 + stream_buffer.clear() + logger.info(f"Saving buffered records to Glide API complete.") # nopep8 because https://github.com/hhatto/autopep8/issues/712 + + yield message + else: + logger.warn(f"Ignoring unknown Airbyte input message type: {message.type}") # nopep8 because https://github.com/hhatto/autopep8/issues/712 + + # commit the stash to the table + glide.commit() - yield message - else: - logger.warn(f"Ignoring unknown Airbyte input message type: {message.type}") pass def check(self, logger: logging.Logger, config: Mapping[str, Any]) -> AirbyteConnectionStatus: @@ -154,4 +171,3 @@ def check(self, logger: logging.Logger, config: Mapping[str, Any]) -> AirbyteCon return AirbyteConnectionStatus(status=Status.SUCCEEDED) except Exception as e: return AirbyteConnectionStatus(status=Status.FAILED, message=f"An exception occurred: {repr(e)}") - diff --git a/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py b/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py index 5f6fbde618f6f..da54af85a9487 100644 --- a/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py +++ b/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py @@ -56,10 +56,17 @@ def url(self, path: str) -> str: return f"{self.api_host}/{self.api_path_root}/{path}" """ - An API client for interacting with a Glide Big Table. + An API client for interacting with a Glide Big Table. The intention is to + create a new table or update an existing table including the table's schema + and the table's rows. + + The protocol is to call `init`, `set_schema`, `add_rows` one or more times, and `commit` in that order. """ def init(self, api_host, api_key, api_path_root, table_id): + """ + Sets the connection information for the table. + """ self.api_host = api_host self.api_key = api_key self.api_path_root = api_path_root @@ -68,23 +75,24 @@ def init(self, api_host, api_key, api_path_root, table_id): pass @abstractmethod - def prepare_table(self, columns: List[Column]) -> None: + def set_schema(self, columns: List[Column]) -> None: """ - Prepares the table with the given columns. + set_schemas the table with the given schema. Each column is a json-schema property where the key is the column name and the type is the . """ pass @abstractmethod - def delete_all(self) -> None: + def add_rows(self, rows: Iterator[BigTableRow]) -> None: """ - Deletes all rows in the table. + Adds rows to the table. """ pass - def add_rows(self, rows: Iterator[BigTableRow]) -> None: + @abstractmethod + def commit(self) -> None: """ - Adds rows to the table. + Commits the table. """ pass @@ -108,57 +116,96 @@ def create(cls, strategy: str) -> GlideBigTableBase: class GlideBigTableRestStrategy(GlideBigTableBase): - def prepare_table(self, columns: List[Column]) -> None: - logger.debug(f"prepare_table columns: {columns}") - # update the table: - r = requests.put( - self.url(f"/tables/{self.table_id}"), + def reset(self): + self.stash_id = None + self.stash_serial = 0 + + def __init__(self): + super().__init__() + self.reset() + + def set_schema(self, columns: List[Column]) -> None: + logger.debug(f"set_schema columns: {columns}") + if columns is None: + raise ValueError("columns must be provided") + if len(columns) == 0: + raise ValueError("columns must be provided") + self.reset() + self.columns = columns + # Create stash we can stash records into for later + r = requests.post( + self.url(f"/stashes"), headers=self.headers(), - json={ - "name": self.table_id, - "schema": { - "columns": columns, - }, - "rows": [] - } ) - if r.status_code != 200: - logger.error(f"prepare table request failed with status {r.status_code}: {r.text}.") # nopep8 + try: + r.raise_for_status() + except Exception as e: + raise Exception(f"failed to create stash") from e # nopep8 - def delete_all(self) -> None: - logger.warning(f"delete_all call is ignored in {type(self).__class__.__name__}") # nopep8 - pass + result = r.json() + self.stash_id = result["data"]["stashID"] + self.stash_serial = 0 + logger.info(f"Created stash for records with id '{self.stash_id}'") + + def raise_if_set_schema_not_called(self): + if self.stash_id is None: + raise ValueError( + "set_schema must be called before add_rows or commit") def _add_row_batch(self, rows: List[BigTableRow]) -> None: + # TODO: add rows to stash/serial https://web.postman.co/workspace/glideapps-Workspace~46b48d24-5fc1-44b6-89aa-8d6751db0fc5/request/9026518-c282ef52-4909-4806-88bf-08510ee80770 logger.debug(f"Adding rows batch with size {len(rows)}") r = requests.post( - self.url(f"/tables/{self.table_id}/rows"), + self.url(f"/stashes/{self.stash_id}/{self.stash_serial}"), headers=self.headers(), json={ - "rows": rows, + "data": rows, "options": { + # ignore columns in rows that are not part of schema: "unknownColumns": "ignore" } } ) - if r.status_code != 200: - logger.error(f"add rows batch failed with status {r.status_code}: {r.text}") # nopep8 - else: - logger.debug(f"add rows batch succeeded") + try: + r.raise_for_status() + except Exception as e: + raise Exception(f"failed to add rows batch for serial '{self.stash_serial}'") from e # nopep8 + + logger.info(f"Add rows batch for serial '{self.stash_serial}' succeeded.") # nopep8 + self.stash_serial += 1 def add_rows(self, rows: Iterator[BigTableRow]) -> None: + self.raise_if_set_schema_not_called() + # TODO: to optimize batch size for variable number and size of columns, we could estimate row byte size based on the first row and choose a batch size based on that. BATCH_SIZE = 500 - batch = [] - for row in rows: - batch.append(row) - if len(batch) >= BATCH_SIZE: - self._add_row_batch(batch) - batch = [] - - if len(batch) > 0: + for i in range(0, len(rows), BATCH_SIZE): + batch = rows[i:i + min(BATCH_SIZE, len(rows) - i)] self._add_row_batch(batch) + def finalize_stash(self) -> None: + # overwrite the existing table with the right schema and rows: + r = requests.put( + self.url(f"/tables/{self.table_id}"), + headers=self.headers(), + json={ + "schema": { + "columns": self.columns, + }, + "rows": { + "$stashID": self.stash_id + } + } + ) + try: + r.raise_for_status() + except Exception as e: + raise Exception(f"failed to finalize stash") from e # nopep8 + + def commit(self) -> None: + self.raise_if_set_schema_not_called() + self.finalize_stash() + class GlideBigTableMutationsStrategy(GlideBigTableBase): def __init__(self): @@ -197,8 +244,8 @@ def headers(self) -> Dict[str, str]: def url(self, path: str) -> str: return f"{self.api_host}/{self.api_path_root}/{path}" - def prepare_table(self, columns: List[Column]) -> None: - logger.debug(f"prepare_table for table '{self.table_id}. Expecting columns: '{[c.id for c in columns]}'.") # nopep8 + def set_schema(self, columns: List[Column]) -> None: + logger.debug(f"set_schema for table '{self.table_id}. Expecting columns: '{[c.id for c in columns]}'.") # nopep8 self.delete_all() for col in columns: @@ -225,7 +272,7 @@ def rows(self) -> Iterator[BigTableRow]: ) if r.status_code != 200: logger.error(f"get rows request failed with status {r.status_code}: {r.text}.") # nopep8 because https://github.com/hhatto/autopep8/issues/712 - r.raise_for_status() # This will raise an HTTPError if the status is 4xx or 5xx + r.raise_for_status() result = r.json() @@ -257,7 +304,7 @@ def delete_all(self) -> None: ) if r.status_code != 200: logger.error(f"delete request failed with status {r.status_code}: {r.text} trying to delete row id {row['$rowID']} with row: {row}") # nopep8 because https://github.com/hhatto/autopep8/issues/712 - r.raise_for_status() # This will raise an HTTPError if the status is 4xx or 5xx + r.raise_for_status() else: logger.debug( f"Deleted row successfully (rowID:'{row['$rowID']}'") @@ -292,7 +339,7 @@ def add_rows_batch(self, rows: Iterator[BigTableRow]) -> None: ) if r.status_code != 200: logger.error(f"add rows failed with status {r.status_code}: {r.text}") # nopep8 because https://github.com/hhatto/autopep8/issues/712 - r.raise_for_status() # This will raise an HTTPError if the status is 4xx or 5xx + r.raise_for_status() def add_rows(self, rows: Iterator[BigTableRow]) -> None: BATCH_SIZE = 100 @@ -306,3 +353,7 @@ def add_rows(self, rows: Iterator[BigTableRow]) -> None: if len(batch) > 0: self.add_rows_batch(batch) + + def commit(self) -> None: + logger.debug("commit table (noop).") + pass diff --git a/airbyte-integrations/connectors/destination-glide/destination_glide/log.py b/airbyte-integrations/connectors/destination-glide/destination_glide/log.py index 69ca6ac655c29..5e815d6c75abd 100644 --- a/airbyte-integrations/connectors/destination-glide/destination_glide/log.py +++ b/airbyte-integrations/connectors/destination-glide/destination_glide/log.py @@ -9,7 +9,7 @@ # Create a file handler # TODO REMOVE? handler = logging.FileHandler('destination-glide.log') -handler.setLevel(logging.DEBUG) +handler.setLevel(logging.INFO) # Create a logging format formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s') diff --git a/airbyte-integrations/connectors/destination-glide/scripts/test-unit.sh b/airbyte-integrations/connectors/destination-glide/scripts/test-unit.sh index 488421db4323b..52bcb42674cb9 100755 --- a/airbyte-integrations/connectors/destination-glide/scripts/test-unit.sh +++ b/airbyte-integrations/connectors/destination-glide/scripts/test-unit.sh @@ -2,4 +2,4 @@ this_dir=$(cd $(dirname "$0"); pwd) # this script's directory this_script=$(basename $0) -poetry run pytest unit_tests +poetry run pytest unit_tests "$@" diff --git a/airbyte-integrations/connectors/destination-glide/todo.md b/airbyte-integrations/connectors/destination-glide/todo.md index 3ec906184c382..1a8353f411193 100644 --- a/airbyte-integrations/connectors/destination-glide/todo.md +++ b/airbyte-integrations/connectors/destination-glide/todo.md @@ -3,10 +3,16 @@ - [+] chore: cleanup logs generally - [+] fix: "add rows failed with status 400: {"message":"More than 500 mutations"}" in mutation api (batch them to ~100 or something) - [+] fix: batch row adds in rest api -- [ ] fix: use friendly names for the properties on the config page -- [ ] fix: ensure table_id is not optional (or come up with a way to consistently name tables to find it again and use that?) - [+] chore: unify test framework across unit/integration tests (remove pytest?) -- [ ] chore: clean up todos +- [+] feat: snuggle rows in batch +- [ ] feat: support multiple streams from source by using a naming convention and discovering existing tables with GET /tables +- [ ] fix: replace "hostname" and "path" configs with "baseUrl" +- [ ] feat: verify actual host/api/auth connection in check: https://docs.airbyte.com/understanding-airbyte/airbyte-protocol#check +- [ ] feat: add a default timeout everywhere per https://requests.readthedocs.io/en/latest/user/quickstart/#timeouts +- [ ] fix: use friendly names for the properties on the config page + - [ ] chore: update readme +- [ ] chore: unit test for Column to ensure it serializes to json as expected +- [ ] chore: remove GlideBigTableMutationsStrategy? +- [ ] chore: clean up todos - [ ] chore: figure out why the dev-write script stopped working (freezes) -- [ ] feat: make the check verify api connection and config settings diff --git a/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableMutationsStrategy_test.py b/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableMutationsStrategy_test.py index 360098a64513c..22c552c2d09c1 100644 --- a/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableMutationsStrategy_test.py +++ b/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableMutationsStrategy_test.py @@ -15,22 +15,22 @@ def setUp(self): self.api_path_root, self.table_id) @patch.object(requests, 'post') - def test_prepare_table_valid(self, mock_post): + def test_set_schema_valid(self, mock_post): mock_post.return_value.status_code = 200 test_columns = [ Column('id', 'string'), Column('name', 'string') ] - self.gbt.prepare_table(test_columns) + self.gbt.set_schema(test_columns) @patch.object(requests, 'post') - def test_prepare_table_invalid_column(self, mock_post): + def test_set_schema_invalid_column(self, mock_post): mock_post.return_value.status_code = 200 test_columns = [ Column('id', 'string'), Column('this column wont be found', 'string') ] - self.gbt.prepare_table(test_columns) + self.gbt.set_schema(test_columns) @patch.object(requests, 'post') def test_add_rows(self, mock_post): @@ -41,7 +41,7 @@ def test_add_rows(self, mock_post): mock_post.return_value.status_code = 200 - self.gbt.prepare_table(test_columns) + self.gbt.set_schema(test_columns) mock_post.reset_mock() test_data = [ @@ -66,7 +66,7 @@ def test_add_rows_batch(self, mock_post): mock_post.return_value.status_code = 200 - self.gbt.prepare_table(test_columns) + self.gbt.set_schema(test_columns) mock_post.reset_mock() test_rows = list([{"strcol": f"{i}", "numcol": i} for i in range(1000)]) diff --git a/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableRestStrategy_test.py b/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableRestStrategy_test.py index f68c76681c3f4..fae940a51c421 100644 --- a/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableRestStrategy_test.py +++ b/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableRestStrategy_test.py @@ -1,71 +1,131 @@ +from destination_glide.glide import GlideBigTableRestStrategy, Column +import requests # for mocking it import unittest +from unittest import skip from unittest.mock import patch -import requests # for mocking it -from destination_glide.glide import GlideBigTableRestStrategy, Column +import uuid class TestGlideBigTableRestStrategy(unittest.TestCase): + api_host = "https://test-api-host.com" + api_key = "test-api-key" + api_path_root = "/test/api/path/root" + table_id = "" + stash_id = "" - api_host = 'https://test-api-host.com' - api_key = 'test-api-key' - api_path_root = '/test/api/path/root' - table_id = 'test-table-id' + test_columns = [ + Column("test-str", "string"), + Column("test-num", "number") + ] def setUp(self): + self.table_id = f"test-table-id-{str(uuid.uuid4())}" + self.stash_id = f"stash-id-{str(uuid.uuid4())}" self.gbt = GlideBigTableRestStrategy() self.gbt.init(self.api_host, self.api_key, self.api_path_root, self.table_id) - @patch.object(requests, 'put') - def test_prepare_table_valid(self, mock_put): - mock_put.return_value.status_code = 200 - mock_put.return_value.json.return_value = {'data': 'test'} + def mock_post_for_set_schema(self, mock_post): + mock_post.return_value.status_code = 200 + mock_post.return_value.json.return_value = { + "data": { + "stashID": self.stash_id + } + } + + @patch.object(requests, "post") + def test_set_schema_valid(self, mock_post): + self.mock_post_for_set_schema(mock_post) test_columns = [ - Column('test-str', 'string'), - Column('test-num', 'number') + Column("test-str", "string"), + Column("test-num", "number") ] - self.gbt.prepare_table(test_columns) + self.gbt.set_schema(test_columns) - mock_put.assert_called_once() - self.assertListEqual( - mock_put.call_args[1]['json']['schema']['columns'], test_columns) + mock_post.assert_called_once() - @patch.object(requests, 'put') - def test_prepare_table_invalid_col_type(self, mock_put): - mock_put.return_value.status_code = 200 - mock_put.return_value.json.return_value = {'data': 'test'} + @patch.object(requests, "post") + def test_set_schema_invalid_col_type(self, mock_post): + self.mock_post_for_set_schema(mock_post) with self.assertRaises(ValueError): - self.gbt.prepare_table([ - Column('test-str', 'string'), - Column('test-num', 'invalid-type') + self.gbt.set_schema([ + Column("test-str", "string"), + Column("test-num", "invalid-type") ]) - @patch.object(requests, 'post') + @patch.object(requests, "post") def test_add_rows(self, mock_post): - mock_post.return_value.status_code = 200 - mock_post.return_value.json.return_value = {'data': 'test'} + self.mock_post_for_set_schema(mock_post) + self.gbt.set_schema(self.test_columns) + mock_post.reset_mock() test_rows = [ - {"strcol": "one", "numcol": 1}, - {"strcol": "two", "numcol": 2} + {"test-str": "one", "test-num": 1}, + {"test-str": "two", "test-num": 2} ] self.gbt.add_rows(test_rows) mock_post.assert_called_once() - assert mock_post.call_args[1]['json']['rows'] == test_rows + self.assertEqual( + mock_post.call_args.kwargs["json"]["data"], test_rows) - @patch.object(requests, 'post') + @patch.object(requests, "post") def test_add_rows_batching(self, mock_post): - mock_post.return_value.status_code = 200 - mock_post.return_value.json.return_value = {'data': 'test'} + self.mock_post_for_set_schema(mock_post) + + self.gbt.set_schema(self.test_columns) + + mock_post.reset_mock() + TEST_ROW_COUNT = 2001 + test_rows = list([ + {"test-str": f"one {i}", "test-num": i} + for i in range(TEST_ROW_COUNT) + ]) - test_rows = list([{"strcol": f"{i}", "numcol": i} for i in range(5000)]) - self.gbt.add_rows(test_rows) - self.assertEqual(10, mock_post.call_count) + self.assertEqual(5, mock_post.call_count) + # validate that the last row is what we expect: + self.assertEqual(mock_post.call_args.kwargs["json"]["data"], + [ + {"test-str": f"one {TEST_ROW_COUNT-1}", "test-num": TEST_ROW_COUNT-1} + ]) + + @skip("future version that supports multiple streams") + def test_add_rows_with_multiple_streams(): + # when multiple streams are coming into destination, ensure adds rows for all + pass + + def test_commit_with_pre_existing_table(self): + with patch.object(requests, "post") as mock_post: + self.mock_post_for_set_schema(mock_post) + self.gbt.set_schema(self.test_columns) + test_rows = [ + {"test-str": "one", "test-num": 1}, + {"test-str": "two", "test-num": 2} + ] + mock_post.reset_mock() + self.gbt.add_rows(test_rows) + + with patch.object(requests, "put") as mock_put: + self.gbt.commit() + # it should have called post to create a new table + mock_post.assert_called_once() + self.assertEqual( + mock_put.call_args.kwargs["json"]["rows"]["$stashID"], self.stash_id) + + @skip("future version that supports multiple streams") + def test_commit_with_non_existing_table(self): + # TODO: in a future version, we want to search for the table and if not found, create it. if found, update it (put). + pass + + @skip("future version that supports multiple streams") + def test_commit_with_multiple_streams(self): + # when multiple streams are coming into destination, ensure stash is committed for each. + pass + if __name__ == '__main__': unittest.main() diff --git a/airbyte-integrations/connectors/destination-glide/unit_tests/destination_test.py b/airbyte-integrations/connectors/destination-glide/unit_tests/destination_test.py index 4bb3dee1aa465..3b9fed7bd601a 100644 --- a/airbyte-integrations/connectors/destination-glide/unit_tests/destination_test.py +++ b/airbyte-integrations/connectors/destination-glide/unit_tests/destination_test.py @@ -47,7 +47,7 @@ def table_schema() -> str: def AirbyteLogger() -> logging.Logger: - return logging.getLogger('airbyte') + logger = logging.getLogger('airbyte') def configured_catalog(test_table_name: str, table_schema: str) -> ConfiguredAirbyteCatalog: @@ -61,20 +61,11 @@ def configured_catalog(test_table_name: str, table_schema: str) -> ConfiguredAir return ConfiguredAirbyteCatalog(streams=[overwrite_stream]) -def airbyte_message_record1(test_table_name: str): +def create_airbyte_message_record(test_table_name: str, record_index: int): return AirbyteMessage( type=Type.RECORD, record=AirbyteRecordMessage( - stream=test_table_name, data={"key_str": "value1", "key_int": 3}, emitted_at=int(datetime.now().timestamp()) * 1000 - ), - ) - - -def airbyte_message_record2(test_table_name: str): - return AirbyteMessage( - type=Type.RECORD, - record=AirbyteRecordMessage( - stream=test_table_name, data={"key_str": "value2", "key_int": 2}, emitted_at=int(datetime.now().timestamp()) * 1000 + stream=test_table_name, data={"key_str": f"value{record_index}", "key_int": record_index}, emitted_at=int(datetime.now().timestamp()) * 1000 ), ) @@ -83,7 +74,7 @@ def airbyte_message_state(test_table_name: str): return AirbyteMessage( type=Type.STATE, state=AirbyteStateMessage( - data={"opaque": "to destination"} + data={"data is": " opaque to destination"} ) ) @@ -106,24 +97,85 @@ def test_check_with_valid_config(config: Mapping): assert outcome.status == Status.SUCCEEDED @patch.object(GlideBigTableFactory, "create") - def test_write(self, mock_factory: Callable): + def test_write_simple(self, mock_factory: Callable): mock_bigtable = CreateMockGlideBigTable() mock_factory.return_value = mock_bigtable destination = DestinationGlide() generator = destination.write( - config=create_default_config(), configured_catalog=configured_catalog(self.test_table_name, table_schema=table_schema()), input_messages=[ - airbyte_message_record1(self.test_table_name), airbyte_message_record2(self.test_table_name), airbyte_message_state(self.test_table_name)] + config=create_default_config(), + configured_catalog=configured_catalog(self.test_table_name, + table_schema=table_schema()), + input_messages=[ + create_airbyte_message_record(self.test_table_name, 1), + create_airbyte_message_record(self.test_table_name, 2), + airbyte_message_state(self.test_table_name) + ] ) - # expecting only to return the state message: + # invoke the generator to get the results: result = list(generator) - assert len(result) == 1 + self.assertEqual(1, len(result)) + + # ensure it called set_schema, multiple add_rows, followed by commit: + self.assertEqual(4, len(mock_bigtable.mock_calls)) + # NOTE: the call objects in Mock.mock_calls, are three-tuples of (name, positional args, keyword args). + CALL_METHOD_NAME_INDEX = 0 + self.assertEqual( + "init", mock_bigtable.mock_calls[0][CALL_METHOD_NAME_INDEX]) + self.assertEqual( + "set_schema", mock_bigtable.mock_calls[1][CALL_METHOD_NAME_INDEX]) + self.assertEqual( + "add_rows", mock_bigtable.mock_calls[2][CALL_METHOD_NAME_INDEX]) + self.assertEqual( + "commit", mock_bigtable.mock_calls[3][CALL_METHOD_NAME_INDEX]) - # todo: validate args on these calls - mock_bigtable.prepare_table.assert_called_once() - mock_bigtable.add_rows.assert_called_once() + @patch.object(GlideBigTableFactory, "create") + def test_write_with_checkpoints(self, mock_factory: Callable): + """ + Ensures that the destination writes rows to the table as they are streamed from the source: + """ + mock_bigtable = CreateMockGlideBigTable() + mock_factory.return_value = mock_bigtable + + destination = DestinationGlide() + + # create enough records to cause a batch: + RECORD_COUNT = 10 + input_messages = [] + input_messages.extend([create_airbyte_message_record( + self.test_table_name, i) for i in range(RECORD_COUNT)]) + # create first checkpoint record: + input_messages.append(airbyte_message_state(self.test_table_name)) + input_messages.extend([create_airbyte_message_record( + self.test_table_name, i) for i in range(RECORD_COUNT)]) + # create second checkpoint record: + input_messages.append(airbyte_message_state(self.test_table_name)) + + generator = destination.write( + config=create_default_config(), + configured_catalog=configured_catalog(self.test_table_name, + table_schema=table_schema()), + input_messages=input_messages + ) + + # invoke the generator to get the results: + result = list(generator) + # we had two state records so we expect them to be yielded: + self.assertEqual(2, len(result)) + + # ensure it called add_rows multiple times: + self.assertGreaterEqual(mock_bigtable.add_rows.call_count, 2) + # NOTE: the call objects in Mock.mock_calls, are three-tuples of (name, positional args, keyword args). + CALL_METHOD_NAME_INDEX = 0 + self.assertEqual( + "init", mock_bigtable.mock_calls[0][CALL_METHOD_NAME_INDEX]) + self.assertEqual( + "set_schema", mock_bigtable.mock_calls[1][CALL_METHOD_NAME_INDEX]) + mock_bigtable.add_rows.assert_called() + self.assertEqual( + "commit", mock_bigtable.mock_calls[mock_bigtable.call_count - 1][CALL_METHOD_NAME_INDEX]) @patch.object(GlideBigTableFactory, "create") def test_with_invalid_column_types(self, mock_factory: Callable): @@ -143,19 +195,25 @@ def test_with_invalid_column_types(self, mock_factory: Callable): } generator = destination.write( - config=create_default_config(), configured_catalog=configured_catalog(self.test_table_name, table_schema=test_schema), input_messages=[ - airbyte_message_record1(self.test_table_name), airbyte_message_record2(self.test_table_name), airbyte_message_state(self.test_table_name)] + config=create_default_config(), + configured_catalog=configured_catalog(self.test_table_name, + table_schema=test_schema), + input_messages=[ + create_airbyte_message_record(self.test_table_name, 1), + create_airbyte_message_record(self.test_table_name, 2), + airbyte_message_state(self.test_table_name) + ] ) # expecting only to return the state message: result = list(generator) assert len(result) == 1 - mock_bigtable.prepare_table.assert_called_once() + mock_bigtable.set_schema.assert_called_once() mock_bigtable.add_rows.assert_called_once() # get the columns we passed into teh API and verify the type defaulted to string: - prepared_cols = mock_bigtable.prepare_table.call_args[0][0] - null_column = [col for col in prepared_cols if col.id() + schema_calls = mock_bigtable.set_schema.call_args[0][0] + null_column = [col for col in schema_calls if col.id() == "obj_null_col"][0] self.assertEqual(null_column.type(), "string") @@ -164,8 +222,6 @@ def test_api_version_passes_correct_strategy(self, mock_factory: Mock): mock_bigtable = CreateMockGlideBigTable() mock_factory.return_value = mock_bigtable - - config = { "api_host": "foo", "api_path_root": "bar", @@ -173,11 +229,17 @@ def test_api_version_passes_correct_strategy(self, mock_factory: Mock): "table_id": "buz", "glide_api_version": "mutations" } - + destination = DestinationGlide() generator = destination.write( - config=config, configured_catalog=configured_catalog(self.test_table_name, table_schema=table_schema()), input_messages=[ - airbyte_message_record1(self.test_table_name), airbyte_message_record2(self.test_table_name), airbyte_message_state(self.test_table_name)] + config=config, + configured_catalog=configured_catalog( + self.test_table_name, table_schema=table_schema()), + input_messages=[ + create_airbyte_message_record(self.test_table_name, 1), + create_airbyte_message_record(self.test_table_name, 2), + airbyte_message_state(self.test_table_name) + ] ) # expecting only to return the state message: result = list(generator) From 8a332011a3c69385254f4ce06096f90a24c61dd1 Mon Sep 17 00:00:00 2001 From: Scott Willeke Date: Fri, 21 Jun 2024 23:16:58 -0400 Subject: [PATCH 17/39] chore: cleanup logging --- .../destination_glide/destination.py | 5 +++-- .../destination_glide/glide.py | 8 +++++--- .../destination_glide/log.py | 19 +------------------ .../connectors/destination-glide/todo.md | 2 +- .../unit_tests/destination_test.py | 2 ++ 5 files changed, 12 insertions(+), 24 deletions(-) diff --git a/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py b/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py index 4b0622a967b42..d7d6f47b7776c 100644 --- a/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py +++ b/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py @@ -16,7 +16,7 @@ from .glide import Column, GlideBigTableFactory import json -from .log import getLogger +from .log import LOG_LEVEL_DEFAULT import logging import requests from typing import Any, Iterable, Mapping @@ -25,7 +25,8 @@ CONFIG_GLIDE_API_VERSION_DEFAULT = "tables" CONFIG_GLIDE_API_HOST_DEFAULT = "https://api.glideapp.io" -logger = getLogger() +logger = logging.getLogger(__name__) +logger.setLevel(LOG_LEVEL_DEFAULT) def mapJsonSchemaTypeToGlideType(json_type: str) -> str: diff --git a/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py b/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py index da54af85a9487..3d9119b637a24 100644 --- a/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py +++ b/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py @@ -1,10 +1,11 @@ from abc import ABC, abstractmethod +from .log import LOG_LEVEL_DEFAULT +import logging import requests from typing import Dict, Any, Iterator, List -from .log import getLogger - -logger = getLogger() +logger = logging.getLogger(__name__) +logger.setLevel(LOG_LEVEL_DEFAULT) BigTableRow = Dict[str, Any] @@ -201,6 +202,7 @@ def finalize_stash(self) -> None: r.raise_for_status() except Exception as e: raise Exception(f"failed to finalize stash") from e # nopep8 + logger.info(f"Successfully finalized record stash for table '{self.table_id}' (stash ID '{self.stash_id}')") def commit(self) -> None: self.raise_if_set_schema_not_called() diff --git a/airbyte-integrations/connectors/destination-glide/destination_glide/log.py b/airbyte-integrations/connectors/destination-glide/destination_glide/log.py index 5e815d6c75abd..0423943b4ee5b 100644 --- a/airbyte-integrations/connectors/destination-glide/destination_glide/log.py +++ b/airbyte-integrations/connectors/destination-glide/destination_glide/log.py @@ -1,21 +1,4 @@ import logging -# Create a logger - -logger = logging.getLogger("destination-glide") -logger.setLevel(logging.DEBUG) - -# Create a file handler -# TODO REMOVE? -handler = logging.FileHandler('destination-glide.log') -handler.setLevel(logging.INFO) -# Create a logging format -formatter = logging.Formatter( - '%(asctime)s - %(name)s - %(levelname)s - %(message)s') -handler.setFormatter(formatter) -# Add the handlers to the logger -logger.addHandler(handler) - -def getLogger() -> logging.Logger: - return logger \ No newline at end of file +LOG_LEVEL_DEFAULT = logging.INFO diff --git a/airbyte-integrations/connectors/destination-glide/todo.md b/airbyte-integrations/connectors/destination-glide/todo.md index 1a8353f411193..6d55ea91b35fe 100644 --- a/airbyte-integrations/connectors/destination-glide/todo.md +++ b/airbyte-integrations/connectors/destination-glide/todo.md @@ -4,7 +4,7 @@ - [+] fix: "add rows failed with status 400: {"message":"More than 500 mutations"}" in mutation api (batch them to ~100 or something) - [+] fix: batch row adds in rest api - [+] chore: unify test framework across unit/integration tests (remove pytest?) -- [+] feat: snuggle rows in batch +- [+] feat: snuggle rows in a batch - [ ] feat: support multiple streams from source by using a naming convention and discovering existing tables with GET /tables - [ ] fix: replace "hostname" and "path" configs with "baseUrl" - [ ] feat: verify actual host/api/auth connection in check: https://docs.airbyte.com/understanding-airbyte/airbyte-protocol#check diff --git a/airbyte-integrations/connectors/destination-glide/unit_tests/destination_test.py b/airbyte-integrations/connectors/destination-glide/unit_tests/destination_test.py index 3b9fed7bd601a..bfeacad539e43 100644 --- a/airbyte-integrations/connectors/destination-glide/unit_tests/destination_test.py +++ b/airbyte-integrations/connectors/destination-glide/unit_tests/destination_test.py @@ -13,6 +13,7 @@ from datetime import datetime from destination_glide import DestinationGlide from destination_glide.glide import GlideBigTableBase, GlideBigTableFactory +from destination_glide.log import LOG_LEVEL_DEFAULT import json import logging import random @@ -48,6 +49,7 @@ def table_schema() -> str: def AirbyteLogger() -> logging.Logger: logger = logging.getLogger('airbyte') + logger.setLevel(LOG_LEVEL_DEFAULT) def configured_catalog(test_table_name: str, table_schema: str) -> ConfiguredAirbyteCatalog: From 17239a70d2b5995f5c001a951737c2397ddf7e00 Mon Sep 17 00:00:00 2001 From: Scott Willeke Date: Sat, 22 Jun 2024 02:22:06 -0400 Subject: [PATCH 18/39] feat: support multiple streams from source --- .../destination_glide/destination.py | 45 ++-- .../destination_glide/glide.py | 220 +++++------------- .../destination_glide/spec.json | 10 - .../connectors/destination-glide/todo.md | 2 +- .../GlideBigTableMutationsStrategy_test.py | 76 ------ .../GlideBigTableRestStrategy_test.py | 68 ++++-- .../unit_tests/destination_test.py | 103 ++++---- 7 files changed, 192 insertions(+), 332 deletions(-) delete mode 100644 airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableMutationsStrategy_test.py diff --git a/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py b/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py index d7d6f47b7776c..0f7dcabe4a0ec 100644 --- a/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py +++ b/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py @@ -29,7 +29,7 @@ logger.setLevel(LOG_LEVEL_DEFAULT) -def mapJsonSchemaTypeToGlideType(json_type: str) -> str: +def airbyteTypeToGlideType(json_type: str) -> str: jsonSchemaTypeToGlideType = { "string": "string", "number": "number", @@ -75,37 +75,39 @@ def write( api_host = config.get('api_host', CONFIG_GLIDE_API_HOST_DEFAULT) api_path_root = config['api_path_root'] api_key = config['api_key'] - table_id = config['table_id'] glide_api_version = config.get( 'glide_api_version', CONFIG_GLIDE_API_VERSION_DEFAULT) + # configure the table based on the stream catalog: # choose a strategy based on config: - glide = GlideBigTableFactory.create(glide_api_version) - logger.debug(f"Using glide api strategy '{glide.__class__.__name__}' for glide_api_version '{glide_api_version}'.") # nopep8 - glide.init(api_host, api_key, api_path_root, table_id) - if len(configured_catalog.streams) > 1: - # TODO: Consider how we might want to support multiple streams. Review other airbyte destinations - raise Exception(f"The Glide destination expects only a single stream to be streamed into a Glide Table but found '{len(configured_catalog.streams)}' streams. Please select only one stream from the source.") # nopep8 + def create_table_client_for_stream(stream_name): + # TODO: sanitize stream_name chars and length for GBT name + glide = GlideBigTableFactory.create(glide_api_version) + logger.debug(f"Using glide api strategy '{glide.__class__.__name__}' for glide_api_version '{glide_api_version}'.") # nopep8 + glide.init(api_host, api_key, api_path_root, stream_name) + return glide - # configure the table based on the stream catalog: + table_clients = {} stream_names = {s.stream.name for s in configured_catalog.streams} for configured_stream in configured_catalog.streams: if configured_stream.destination_sync_mode != DestinationSyncMode.overwrite: raise Exception(f'Only destination sync mode overwrite is supported, but received "{configured_stream.destination_sync_mode}".') # nopep8 because https://github.com/hhatto/autopep8/issues/712 + glide = create_table_client_for_stream( + configured_stream.stream.name) # upsert the GBT with schema to set_schema for dumping the data into it columns = [] properties = configured_stream.stream.json_schema["properties"] - for prop_name in properties.keys(): - prop = properties[prop_name] + for (prop_name, prop) in properties.items(): prop_type = prop["type"] - prop_format = prop["format"] if "format" in prop else "" - logger.debug(f"Found column/property '{prop_name}' with type '{prop_type}' and format '{prop_format}' in stream {configured_stream.stream.name}.") # nopep8 + logger.debug(f"Found column/property '{prop_name}' with type '{prop_type}' in stream {configured_stream.stream.name}.") # nopep8 columns.append( - Column(prop_name, mapJsonSchemaTypeToGlideType(prop_type))) + Column(prop_name, airbyteTypeToGlideType(prop_type)) + ) glide.set_schema(columns) + table_clients[configured_stream.stream.name] = glide # stream the records into the GBT: buffers = defaultdict(list) @@ -120,7 +122,6 @@ def write( f"Stream {stream_name} was not present in configured streams, skipping") continue - # TODO: check the columns match the columns that we saw in configured_catalog per https://docs.airbyte.com/understanding-airbyte/airbyte-protocol#destination # add to buffer record_data = message.record.data record_id = str(uuid.uuid4()) @@ -133,24 +134,32 @@ def write( # `Type.State` is a signal from the source that we should save the previous batch of `Type.RECORD` messages to the destination. # It is a checkpoint that enables partial success. # See https://docs.airbyte.com/understanding-airbyte/airbyte-protocol#state--checkpointing - logger.info(f"Writing buffered records to Glide API from {len(buffers.keys())} streams...") # nopep8 because + logger.info(f"Writing buffered records to Glide API from {len(buffers.keys())} streams...") # nopep8 for stream_name in buffers.keys(): stream_buffer = buffers[stream_name] - logger.info(f"Saving buffered records to Glide API (stream: '{stream_name}' count: '{len(stream_buffer)}')...") # nopep8 because https://github.com/hhatto/autopep8/issues/712 + logger.info(f"Saving buffered records to Glide API (stream: '{stream_name}', record count: '{len(stream_buffer)}')...") # nopep8 DATA_INDEX = 2 data_rows = [row_tuple[DATA_INDEX] for row_tuple in stream_buffer] if len(data_rows) > 0: + if stream_name not in table_clients: + raise Exception( + f"Stream '{stream_name}' not found in table_clients") + glide = table_clients[stream_name] glide.add_rows(data_rows) stream_buffer.clear() logger.info(f"Saving buffered records to Glide API complete.") # nopep8 because https://github.com/hhatto/autopep8/issues/712 + # dump all buffers now as we just wrote them to the table: + buffers = defaultdict(list) yield message else: logger.warn(f"Ignoring unknown Airbyte input message type: {message.type}") # nopep8 because https://github.com/hhatto/autopep8/issues/712 # commit the stash to the table - glide.commit() + for stream_name, glide in table_clients.items(): + glide.commit() + logger.info(f"Committed stream '{stream_name}' to Glide.") pass diff --git a/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py b/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py index 3d9119b637a24..8c0bb89ce391d 100644 --- a/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py +++ b/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py @@ -64,16 +64,15 @@ def url(self, path: str) -> str: The protocol is to call `init`, `set_schema`, `add_rows` one or more times, and `commit` in that order. """ - def init(self, api_host, api_key, api_path_root, table_id): + def init(self, api_host, api_key, api_path_root, table_name): """ Sets the connection information for the table. """ self.api_host = api_host self.api_key = api_key self.api_path_root = api_path_root - self.table_id = table_id + self.table_name = table_name # todo: validate args - pass @abstractmethod def set_schema(self, columns: List[Column]) -> None: @@ -108,8 +107,7 @@ def create(cls, strategy: str) -> GlideBigTableBase: Creates a new instance of the default implementation for the GlideBigTable API client. """ implementation_map = { - "tables": GlideBigTableRestStrategy(), - "mutations": GlideBigTableMutationsStrategy() + "tables": GlideBigTableRestStrategy() } if strategy not in implementation_map: raise ValueError(f"Strategy '{strategy}' not found. Expected one of '{implmap.keys()}'.") # nopep8 @@ -135,13 +133,13 @@ def set_schema(self, columns: List[Column]) -> None: self.columns = columns # Create stash we can stash records into for later r = requests.post( - self.url(f"/stashes"), + self.url(f"stashes"), headers=self.headers(), ) try: r.raise_for_status() except Exception as e: - raise Exception(f"failed to create stash") from e # nopep8 + raise Exception(f"failed to create stash. Response was '{r.text}'") from e # nopep8 result = r.json() self.stash_id = result["data"]["stashID"] @@ -157,7 +155,7 @@ def _add_row_batch(self, rows: List[BigTableRow]) -> None: # TODO: add rows to stash/serial https://web.postman.co/workspace/glideapps-Workspace~46b48d24-5fc1-44b6-89aa-8d6751db0fc5/request/9026518-c282ef52-4909-4806-88bf-08510ee80770 logger.debug(f"Adding rows batch with size {len(rows)}") r = requests.post( - self.url(f"/stashes/{self.stash_id}/{self.stash_serial}"), + self.url(f"stashes/{self.stash_id}/{self.stash_serial}"), headers=self.headers(), json={ "data": rows, @@ -170,9 +168,9 @@ def _add_row_batch(self, rows: List[BigTableRow]) -> None: try: r.raise_for_status() except Exception as e: - raise Exception(f"failed to add rows batch for serial '{self.stash_serial}'") from e # nopep8 + raise Exception(f"failed to add rows batch for serial '{self.stash_serial}'. Response was '{r.text}'") from e # nopep8 - logger.info(f"Add rows batch for serial '{self.stash_serial}' succeeded.") # nopep8 + logger.info(f"Added {len(rows)} rows as batch for serial '{self.stash_serial}' successfully.") # nopep8 self.stash_serial += 1 def add_rows(self, rows: Iterator[BigTableRow]) -> None: @@ -184,14 +182,15 @@ def add_rows(self, rows: Iterator[BigTableRow]) -> None: batch = rows[i:i + min(BATCH_SIZE, len(rows) - i)] self._add_row_batch(batch) - def finalize_stash(self) -> None: - # overwrite the existing table with the right schema and rows: - r = requests.put( - self.url(f"/tables/{self.table_id}"), + def create_table_from_stash(self) -> None: + logger.info(f"Creating new table for table name '{self.table_name}'...") # nopep8 + r = requests.post( + self.url(f"tables"), headers=self.headers(), json={ + "name": self.table_name, "schema": { - "columns": self.columns, + "columns": self.columns }, "rows": { "$stashID": self.stash_id @@ -201,161 +200,56 @@ def finalize_stash(self) -> None: try: r.raise_for_status() except Exception as e: - raise Exception(f"failed to finalize stash") from e # nopep8 - logger.info(f"Successfully finalized record stash for table '{self.table_id}' (stash ID '{self.stash_id}')") - - def commit(self) -> None: - self.raise_if_set_schema_not_called() - self.finalize_stash() + raise Exception(f"failed to create table '{self.table_name}'. Response was '{r.text}'.") from e # nopep8 + logger.info(f"Creating table '{self.table_name}' succeeded.") -class GlideBigTableMutationsStrategy(GlideBigTableBase): - def __init__(self): - # TODO: hardcoded for now using old api - self.hardcoded_app_id = "Ix9CEuP6DiFugfjhSG5t" - self.hardcoded_column_lookup = { - '_airtable_id': {'type': "string", 'name': "Name"}, - '_airtable_created_time': {'type': "date-time", 'name': "AwkFL"}, - '_airtable_table_name': {'type': "string", 'name': "QF0zI"}, - 'id': {'type': "string", 'name': "tLPjZ"}, - 'name': {'type': "string", 'name': "1ZqF1"}, - 'host_id': {'type': "string", 'name': "B7fYe"}, - 'host_name': {'type': "string", 'name': "oyVzO"}, - 'neighbourhood_group': {'type': "string", 'name': "15J8U"}, - 'neighbourhood': {'type': "string", 'name': "Fy28U"}, - 'latitude': {'type': "number", 'name': "TLpMC"}, - 'longitude': {'type': "number", 'name': "oazQO"}, - 'room_type': {'type': "string", 'name': "TPJDZ"}, - 'price': {'type': "number", 'name': "7xzlG"}, - 'minimum_nights': {'type': "number", 'name': "usoY5"}, - 'number_of_reviews': {'type': "number", 'name': "XFXmR"}, - 'last_review': {'type': "date-time", 'name': "oseZl"}, - 'reviews_per_month': {'type': "number", 'name': "alw2R"}, - 'calculated_host_listings_count': {'type': "number", 'name': "hKws0"}, - 'availability_365': {'type': "number", 'name': "qZsgl"}, - 'number_of_reviews_ltm': {'type': "number", 'name': "rWisS"}, - 'license': {'type': "string", 'name': "7PVig"} - } - - def headers(self) -> Dict[str, str]: - return { - "Content-Type": "application/json", - f"Authorization": f"Bearer {self.api_key}" - } - - def url(self, path: str) -> str: - return f"{self.api_host}/{self.api_path_root}/{path}" - - def set_schema(self, columns: List[Column]) -> None: - logger.debug(f"set_schema for table '{self.table_id}. Expecting columns: '{[c.id for c in columns]}'.") # nopep8 - self.delete_all() - - for col in columns: - if col.id not in self.hardcoded_column_lookup: - logger.warning( - f"Column '{col.id}' not found in hardcoded column lookup. Will be ignored.") - - def rows(self) -> Iterator[BigTableRow]: - """ - Gets the rows as of the Glide Big Table. - """ - r = requests.post( - self.url("function/queryTables"), + def overwrite_table_from_stash(self, table_id) -> None: + # overwrite the specified table's schema and rows with the stash: + r = requests.put( + self.url(f"tables/{table_id}"), headers=self.headers(), json={ - "appID": self.hardcoded_app_id, - "queries": [ - { - "tableName": self.table_id, - "utc": True - } - ] - } - ) - if r.status_code != 200: - logger.error(f"get rows request failed with status {r.status_code}: {r.text}.") # nopep8 because https://github.com/hhatto/autopep8/issues/712 - r.raise_for_status() - - result = r.json() - - # the result looks like an array of results; each result has a rows member that has an array or JSON rows: - for row in result: - for r in row['rows']: - yield r - - def delete_all(self) -> None: - # TODO: perf: don't put these in a list - rows = list(self.rows()) - logger.debug(f"Iterating over {len(rows)} rows to delete") - - for row in rows: - # TODO: lame. batch these into one request with multiple mutations - r = requests.post( - self.url("function/mutateTables"), - headers=self.headers(), - json={ - "appID": self.hardcoded_app_id, - "mutations": [ - { - "kind": "delete-row", - "tableName": self.table_id, - "rowID": row['$rowID'] - } - ] - } - ) - if r.status_code != 200: - logger.error(f"delete request failed with status {r.status_code}: {r.text} trying to delete row id {row['$rowID']} with row: {row}") # nopep8 because https://github.com/hhatto/autopep8/issues/712 - r.raise_for_status() - else: - logger.debug( - f"Deleted row successfully (rowID:'{row['$rowID']}'") - - def add_rows_batch(self, rows: Iterator[BigTableRow]) -> None: - mutations = [] - for row in rows: - # row is columnLabel -> value, but glide's mutate uses a column "name". We hard-code the lookup for our table here: - mutated_row = dict() - for k, v in row.items(): - if k in self.hardcoded_column_lookup: - col_info = self.hardcoded_column_lookup[k] - mutated_row[col_info["name"]] = v - else: - logger.error( - f"Column {k} not found in column lookup. Ignoring column") - - mutations.append( - { - "kind": "add-row-to-table", - "tableName": self.table_id, - "columnValues": mutated_row + "schema": { + "columns": self.columns, + }, + "rows": { + "$stashID": self.stash_id } - ) - r = requests.post( - self.url("function/mutateTables"), - headers=self.headers(), - json={ - "appID": self.hardcoded_app_id, - "mutations": mutations } ) - if r.status_code != 200: - logger.error(f"add rows failed with status {r.status_code}: {r.text}") # nopep8 because https://github.com/hhatto/autopep8/issues/712 + try: r.raise_for_status() - - def add_rows(self, rows: Iterator[BigTableRow]) -> None: - BATCH_SIZE = 100 - - batch = [] - for row in rows: - batch.append(row) - if len(batch) >= BATCH_SIZE: - self.add_rows_batch(batch) - batch = [] - - if len(batch) > 0: - self.add_rows_batch(batch) + except Exception as e: + raise Exception(f"failed to overwrite table '{table_id}'. Response was '{r.text}'") from e # nopep8 def commit(self) -> None: - logger.debug("commit table (noop).") - pass + self.raise_if_set_schema_not_called() + # first see if the table already exists + r = requests.get( + self.url(f"tables"), + headers=self.headers() + ) + try: + r.raise_for_status() + except Exception as e: + raise Exception(f"Failed to get table list. Response was '{r.text}'.") from e # nopep8 + + found_table_id = None + # confirm if table exists: + body = r.json() + if "data" not in body: + raise Exception(f"get tables response did not include data in body. Status was: {r.status_code}: {r.text}.") # nopep8 + + for table in body["data"]: + if table["name"] == self.table_name: + found_table_id = table["id"] + logger.info(f"Found existing table to reuse for table name '{self.table_name}' with ID '{found_table_id}'.") # nopep8 + break + + if found_table_id != None: + self.overwrite_table_from_stash(found_table_id) + else: + self.create_table_from_stash() + + logger.info(f"Successfully committed record stash for table '{self.table_name}' (stash ID '{self.stash_id}')") # nopep8 diff --git a/airbyte-integrations/connectors/destination-glide/destination_glide/spec.json b/airbyte-integrations/connectors/destination-glide/destination_glide/spec.json index ad2ad7288216e..f32a6b07840a4 100644 --- a/airbyte-integrations/connectors/destination-glide/destination_glide/spec.json +++ b/airbyte-integrations/connectors/destination-glide/destination_glide/spec.json @@ -21,16 +21,6 @@ "type": "string", "description": "The user's key for the Glide API destination. Needs permission to write to the specified GBT.", "airbyte_secret": true - }, - "table_id": { - "type": "string", - "description": "The ID of the table that the Glide destination should use. If it does not exist it will be created.", - "airbyte_secret": true - }, - "glide_api_version": { - "type": "string", - "description": "The Glide API version to use. Options are 'tables' or 'mutations'. Default is 'tables'.", - "enum": ["tables", "mutations"] } } } diff --git a/airbyte-integrations/connectors/destination-glide/todo.md b/airbyte-integrations/connectors/destination-glide/todo.md index 6d55ea91b35fe..3fbd6440a4310 100644 --- a/airbyte-integrations/connectors/destination-glide/todo.md +++ b/airbyte-integrations/connectors/destination-glide/todo.md @@ -5,7 +5,7 @@ - [+] fix: batch row adds in rest api - [+] chore: unify test framework across unit/integration tests (remove pytest?) - [+] feat: snuggle rows in a batch -- [ ] feat: support multiple streams from source by using a naming convention and discovering existing tables with GET /tables +- [+] feat: support multiple streams from source by using a naming convention and discovering existing tables with GET /tables - [ ] fix: replace "hostname" and "path" configs with "baseUrl" - [ ] feat: verify actual host/api/auth connection in check: https://docs.airbyte.com/understanding-airbyte/airbyte-protocol#check - [ ] feat: add a default timeout everywhere per https://requests.readthedocs.io/en/latest/user/quickstart/#timeouts diff --git a/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableMutationsStrategy_test.py b/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableMutationsStrategy_test.py deleted file mode 100644 index 22c552c2d09c1..0000000000000 --- a/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableMutationsStrategy_test.py +++ /dev/null @@ -1,76 +0,0 @@ -from destination_glide.glide import GlideBigTableMutationsStrategy, Column -import requests -import unittest -from unittest.mock import patch - -class TestGlideBigTableMutationsStrategy(unittest.TestCase): - api_host = 'https://test-api-host.com' - api_key = 'test-api-key' - api_path_root = '/test/api/path/root' - table_id = 'test-table-id' - - def setUp(self): - self.gbt = GlideBigTableMutationsStrategy() - self.gbt.init(self.api_host, self.api_key, - self.api_path_root, self.table_id) - - @patch.object(requests, 'post') - def test_set_schema_valid(self, mock_post): - mock_post.return_value.status_code = 200 - test_columns = [ - Column('id', 'string'), - Column('name', 'string') - ] - self.gbt.set_schema(test_columns) - - @patch.object(requests, 'post') - def test_set_schema_invalid_column(self, mock_post): - mock_post.return_value.status_code = 200 - test_columns = [ - Column('id', 'string'), - Column('this column wont be found', 'string') - ] - self.gbt.set_schema(test_columns) - - @patch.object(requests, 'post') - def test_add_rows(self, mock_post): - test_columns = [ - Column('id', 'string'), - Column('this column wont be found', 'string') - ] - - mock_post.return_value.status_code = 200 - - self.gbt.set_schema(test_columns) - - mock_post.reset_mock() - test_data = [ - { - 'id': '1', - 'col2': 'test name' - }, - { - 'id': '2', - 'col2': 'test name2' - } - ] - self.gbt.add_rows(test_data) - self.assertEqual(1, mock_post.call_count) - - @patch.object(requests, 'post') - def test_add_rows_batch(self, mock_post): - test_columns = [ - Column('strcol', 'string'), - Column('numcol', 'number') - ] - - mock_post.return_value.status_code = 200 - - self.gbt.set_schema(test_columns) - - mock_post.reset_mock() - test_rows = list([{"strcol": f"{i}", "numcol": i} for i in range(1000)]) - self.gbt.add_rows(test_rows) - - self.assertEqual(10, mock_post.call_count) - diff --git a/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableRestStrategy_test.py b/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableRestStrategy_test.py index fae940a51c421..ade04df3cf817 100644 --- a/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableRestStrategy_test.py +++ b/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableRestStrategy_test.py @@ -9,8 +9,9 @@ class TestGlideBigTableRestStrategy(unittest.TestCase): api_host = "https://test-api-host.com" api_key = "test-api-key" - api_path_root = "/test/api/path/root" + api_path_root = "test/api/path/root" table_id = "" + table_name = "" stash_id = "" test_columns = [ @@ -20,10 +21,11 @@ class TestGlideBigTableRestStrategy(unittest.TestCase): def setUp(self): self.table_id = f"test-table-id-{str(uuid.uuid4())}" + self.table_name = f"test-table-name-{str(uuid.uuid4())}" self.stash_id = f"stash-id-{str(uuid.uuid4())}" self.gbt = GlideBigTableRestStrategy() self.gbt.init(self.api_host, self.api_key, - self.api_path_root, self.table_id) + self.api_path_root, self.table_name) def mock_post_for_set_schema(self, mock_post): mock_post.return_value.status_code = 200 @@ -93,11 +95,6 @@ def test_add_rows_batching(self, mock_post): {"test-str": f"one {TEST_ROW_COUNT-1}", "test-num": TEST_ROW_COUNT-1} ]) - @skip("future version that supports multiple streams") - def test_add_rows_with_multiple_streams(): - # when multiple streams are coming into destination, ensure adds rows for all - pass - def test_commit_with_pre_existing_table(self): with patch.object(requests, "post") as mock_post: self.mock_post_for_set_schema(mock_post) @@ -109,22 +106,55 @@ def test_commit_with_pre_existing_table(self): mock_post.reset_mock() self.gbt.add_rows(test_rows) - with patch.object(requests, "put") as mock_put: - self.gbt.commit() - # it should have called post to create a new table - mock_post.assert_called_once() - self.assertEqual( - mock_put.call_args.kwargs["json"]["rows"]["$stashID"], self.stash_id) + with patch.object(requests, "get") as mock_get: + # mock the `GET /tables` response to include the table: + mock_get.return_value.status_code = 200 + mock_get.return_value.json.return_value = { + "data": [ + { + "name": self.table_name, + "id": self.table_id + } + ] + } + mock_post.reset_mock() + with patch.object(requests, "put") as mock_put: + self.gbt.commit() + # it should have called put to overwrite a table and NOT called post + mock_put.assert_called_once() + self.assertEqual( + mock_put.call_args.kwargs["json"]["rows"]["$stashID"], self.stash_id) + # it should have NOT created a new table via post: + mock_post.assert_not_called() - @skip("future version that supports multiple streams") def test_commit_with_non_existing_table(self): # TODO: in a future version, we want to search for the table and if not found, create it. if found, update it (put). - pass + with patch.object(requests, "post") as mock_post: + self.mock_post_for_set_schema(mock_post) + self.gbt.set_schema(self.test_columns) + test_rows = [ + {"test-str": "one", "test-num": 1}, + {"test-str": "two", "test-num": 2} + ] + mock_post.reset_mock() + self.gbt.add_rows(test_rows) + + with patch.object(requests, "get") as mock_get: + # mock the `GET /tables` response to include the table: + mock_get.return_value.status_code = 200 + mock_get.return_value.json.return_value = { + "data": [] + } + mock_post.reset_mock() + with patch.object(requests, "put") as mock_put: + self.gbt.commit() + # it should not have tried to overwrite a table with put + mock_put.assert_not_called() + self.assertEqual( + mock_post.call_args.kwargs["json"]["rows"]["$stashID"], self.stash_id) + # it should have created a new table with post: + mock_put.assert_not_called() - @skip("future version that supports multiple streams") - def test_commit_with_multiple_streams(self): - # when multiple streams are coming into destination, ensure stash is committed for each. - pass if __name__ == '__main__': diff --git a/airbyte-integrations/connectors/destination-glide/unit_tests/destination_test.py b/airbyte-integrations/connectors/destination-glide/unit_tests/destination_test.py index bfeacad539e43..67021d1f2d66e 100644 --- a/airbyte-integrations/connectors/destination-glide/unit_tests/destination_test.py +++ b/airbyte-integrations/connectors/destination-glide/unit_tests/destination_test.py @@ -52,9 +52,8 @@ def AirbyteLogger() -> logging.Logger: logger.setLevel(LOG_LEVEL_DEFAULT) -def configured_catalog(test_table_name: str, table_schema: str) -> ConfiguredAirbyteCatalog: +def create_configured_catalog_default(test_table_name: str, table_schema: str) -> ConfiguredAirbyteCatalog: overwrite_stream = ConfiguredAirbyteStream( - # TODO: I'm not sure if we should expect incoming streams SyncMode.incremental and only the destination to be full_refresh or they should stream=AirbyteStream(name=test_table_name, json_schema=table_schema, supported_sync_modes=[SyncMode.incremental]), sync_mode=SyncMode.incremental, @@ -63,16 +62,16 @@ def configured_catalog(test_table_name: str, table_schema: str) -> ConfiguredAir return ConfiguredAirbyteCatalog(streams=[overwrite_stream]) -def create_airbyte_message_record(test_table_name: str, record_index: int): +def create_airbyte_message_record(stream_name: str, record_index: int): return AirbyteMessage( type=Type.RECORD, record=AirbyteRecordMessage( - stream=test_table_name, data={"key_str": f"value{record_index}", "key_int": record_index}, emitted_at=int(datetime.now().timestamp()) * 1000 + stream=stream_name, data={"key_str": f"value{record_index}", "key_int": record_index}, emitted_at=int(datetime.now().timestamp()) * 1000 ), ) -def airbyte_message_state(test_table_name: str): +def create_airbyte_message_state(stream_name: str): return AirbyteMessage( type=Type.STATE, state=AirbyteStateMessage( @@ -107,12 +106,12 @@ def test_write_simple(self, mock_factory: Callable): generator = destination.write( config=create_default_config(), - configured_catalog=configured_catalog(self.test_table_name, - table_schema=table_schema()), + configured_catalog=create_configured_catalog_default(self.test_table_name, + table_schema=table_schema()), input_messages=[ create_airbyte_message_record(self.test_table_name, 1), create_airbyte_message_record(self.test_table_name, 2), - airbyte_message_state(self.test_table_name) + create_airbyte_message_state(self.test_table_name) ] ) @@ -149,16 +148,16 @@ def test_write_with_checkpoints(self, mock_factory: Callable): input_messages.extend([create_airbyte_message_record( self.test_table_name, i) for i in range(RECORD_COUNT)]) # create first checkpoint record: - input_messages.append(airbyte_message_state(self.test_table_name)) + input_messages.append(create_airbyte_message_state(self.test_table_name)) input_messages.extend([create_airbyte_message_record( self.test_table_name, i) for i in range(RECORD_COUNT)]) # create second checkpoint record: - input_messages.append(airbyte_message_state(self.test_table_name)) + input_messages.append(create_airbyte_message_state(self.test_table_name)) generator = destination.write( config=create_default_config(), - configured_catalog=configured_catalog(self.test_table_name, - table_schema=table_schema()), + configured_catalog=create_configured_catalog_default(self.test_table_name, + table_schema=table_schema()), input_messages=input_messages ) @@ -179,6 +178,50 @@ def test_write_with_checkpoints(self, mock_factory: Callable): self.assertEqual( "commit", mock_bigtable.mock_calls[mock_bigtable.call_count - 1][CALL_METHOD_NAME_INDEX]) + @patch.object(GlideBigTableFactory, "create", wraps=CreateMockGlideBigTable()) + def test_write_with_multiple_streams(self, mock_factory: Callable): + """ + multiple streams should cause multiple tables to be created, and multiple stashes to be committed + """ + # mock_bigtable = CreateMockGlideBigTable() + # mock_factory.return_value = mock_bigtable + + destination = DestinationGlide() + + # create a catalog with multiple streams: + streamA = ConfiguredAirbyteStream( + stream=AirbyteStream(name="stream-a", json_schema=table_schema(), + supported_sync_modes=[SyncMode.incremental]), + sync_mode=SyncMode.incremental, + destination_sync_mode=DestinationSyncMode.overwrite, + ) + streamB = ConfiguredAirbyteStream( + stream=AirbyteStream(name="stream-b", json_schema=table_schema(), + supported_sync_modes=[SyncMode.incremental]), + sync_mode=SyncMode.incremental, + destination_sync_mode=DestinationSyncMode.overwrite, + ) + + configured_catalog = ConfiguredAirbyteCatalog( + streams=[streamA, streamB]) + + generator = destination.write( + config=create_default_config(), + configured_catalog=configured_catalog, + input_messages=[ + create_airbyte_message_record("stream-a", 0), + create_airbyte_message_record("stream-b", 0), + create_airbyte_message_state("stream-a"), + create_airbyte_message_state("stream-b") + ] + ) + + # invoke the generator to get the results: + result = list(generator) + + # multiple tables should have been created, and multiple stashes been committed + self.assertEqual(mock_factory.call_count, 2) + @patch.object(GlideBigTableFactory, "create") def test_with_invalid_column_types(self, mock_factory: Callable): mock_bigtable = CreateMockGlideBigTable() @@ -198,12 +241,12 @@ def test_with_invalid_column_types(self, mock_factory: Callable): generator = destination.write( config=create_default_config(), - configured_catalog=configured_catalog(self.test_table_name, - table_schema=test_schema), + configured_catalog=create_configured_catalog_default(self.test_table_name, + table_schema=test_schema), input_messages=[ create_airbyte_message_record(self.test_table_name, 1), create_airbyte_message_record(self.test_table_name, 2), - airbyte_message_state(self.test_table_name) + create_airbyte_message_state(self.test_table_name) ] ) @@ -218,33 +261,3 @@ def test_with_invalid_column_types(self, mock_factory: Callable): null_column = [col for col in schema_calls if col.id() == "obj_null_col"][0] self.assertEqual(null_column.type(), "string") - - @patch.object(GlideBigTableFactory, "create") - def test_api_version_passes_correct_strategy(self, mock_factory: Mock): - mock_bigtable = CreateMockGlideBigTable() - mock_factory.return_value = mock_bigtable - - config = { - "api_host": "foo", - "api_path_root": "bar", - "api_key": "baz", - "table_id": "buz", - "glide_api_version": "mutations" - } - - destination = DestinationGlide() - generator = destination.write( - config=config, - configured_catalog=configured_catalog( - self.test_table_name, table_schema=table_schema()), - input_messages=[ - create_airbyte_message_record(self.test_table_name, 1), - create_airbyte_message_record(self.test_table_name, 2), - airbyte_message_state(self.test_table_name) - ] - ) - # expecting only to return the state message: - result = list(generator) - - passed_strategy = mock_factory.call_args[0][0] - self.assertEqual("mutations", passed_strategy) From 77bc93a3dec8c22bdf0ee0e76e7c1952a1858628 Mon Sep 17 00:00:00 2001 From: Scott Willeke Date: Thu, 27 Jun 2024 07:58:12 -0700 Subject: [PATCH 19/39] docs: Readme update to help others dev-test locally --- .../connectors/destination-glide/README.md | 86 +++++++++++++++++-- .../connectors/destination-glide/todo.md | 22 ++--- 2 files changed, 92 insertions(+), 16 deletions(-) diff --git a/airbyte-integrations/connectors/destination-glide/README.md b/airbyte-integrations/connectors/destination-glide/README.md index 44afb91a9e47e..9115f521d6e7c 100644 --- a/airbyte-integrations/connectors/destination-glide/README.md +++ b/airbyte-integrations/connectors/destination-glide/README.md @@ -3,23 +3,93 @@ This is the repository for the Glide destination connector, written in Python. For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/destinations/glide). -## Local development +## Development + +The active todo list is at [./todo.md]. +The gist of the Glide-specific code is in `/destination_glide/destination.py` and `/destination_glide/glide.py`. ### Prerequisites -* Python (`^3.9`) -* Poetry (`^1.7`) - installation instructions [here](https://python-poetry.org/docs/#installation) +- Python (`^3.9`, tested recently with `3.12.3`) +- Poetry (`^1.7`, tested recently with `1.8.3_1`) + +I used homebrew for installing these prerequisites on macOS. + +### Unit Tests + +The unit tests for that code are in `/destination-glide/unit_tests`. To run them run: + +```sh +./scripts/test-unit.sh +``` + +### Integration Tests + +The destination has a configuration in `/secrets/config.json`. That file must confirm to the configuration specification in `/destination_glide/spec.json`. It should be something like: + +```json +{ + "api_host": "http://localhost:5005", + "api_path_root": "api", + "api_key": "decafbad-1234-1234-1234-decafbad" +} +``` + +The spec also specifies the configuration UI within the Airbyte product itself for configuring the destination. + +There are a set of simple integration tests that Airbyte provides that can be triggered with the following scripts: + +```sh +./scripts/dev-check.sh +./scripts/dev-spec.sh +./scripts/dev-write.sh +``` + +These simply call commands that Airbyte provides in their connector template. The dev-write one appears to be the most comprehensive, but I've struggled to get that one to consistently run (see TODO). + +### Build & Deployment + +The Airbyte destination is packed as Docker image. This script uses Airbyte-provided tooling named `airbyte-ci` that leverages the same tooling they use in their CI pipeline to build the container. + +To install the tooling see [`airbyte-ci` README in this repo](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) This worked for me other than to make it work on macOS with homebrew-provided python, I don't have `python` on my path only `python3` so I had to change that in a couple places in the `Makefile` in the root of this repo. +```sh +./scripts/build-docker-image.sh +``` + +We are currently deploying this to a public repository for ease of access from an Airbyte OSS instance. To deploy it to a docker container registry use the script at: + +```sh +./scripts/push-docker-image.sh +``` + +### Running in Airbyte OSS Locally + +To install Airbyte follow the guide at https://docs.airbyte.com/deploying-airbyte/quickstart. On macOS this uses homebrew to install k8s kind locally and get an airbyte cluster running. It took a while but worked smoothly for me. I am currently using `airbytehq/tap/abctl (0.5.0)`. + +Once install it should be available at http://localhost:8000/. You should have been prompted for username/pw during install. + +### Installing Glide Destination in Airbyte OSS + +To install the destination into Airbyte OSS follow these steps: +1. Click on **Settings** on the far left then select **Destinations** in the sub-panel. You should see a list of **Available destination connectors**. +2. At the top click the **+ New Connector** button fill in the fields. The **Docker repository name** and **Docker image tag** are the important bits. + +Once installed, you can upgrade it to a new version by visiting the same settings page and changing the tag in the **Change to** box and clicking the **Change** button. + +--- + +## Old (from Airbyte's Template) ### Installing the connector From this connector directory, run: + ```bash poetry install --with dev ``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/destinations/glide) @@ -31,6 +101,7 @@ See `integration_tests/sample_config.json` for a sample config file. and place them into `secrets/config.json`. ### Locally running the connector + ``` poetry run destination-glide spec poetry run destination-glide check --config secrets/config.json @@ -49,6 +120,7 @@ poetry run pytest tests 1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) 2. Run the following command to build the docker image: + ```bash airbyte-ci connectors --name=destination-glide build ``` @@ -58,6 +130,7 @@ An image will be available on your host with the tag `airbyte/destination-glide: ### Running as a docker container Then run any of the connector commands as follows: + ``` docker run --rm airbyte/destination-glide:dev spec docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-glide:dev check --config /secrets/config.json @@ -91,10 +164,11 @@ Please commit the changes to `pyproject.toml` and `poetry.lock` files. ## Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? + 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-glide test` 2. Bump the connector version (please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors)): - - bump the `dockerImageTag` value in in `metadata.yaml` - - bump the `version` value in `pyproject.toml` + - bump the `dockerImageTag` value in in `metadata.yaml` + - bump the `version` value in `pyproject.toml` 3. Make sure the `metadata.yaml` content is up to date. 4. Make sure the connector documentation and its changelog is up to date (`docs/integrations/destinations/glide.md`). 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). diff --git a/airbyte-integrations/connectors/destination-glide/todo.md b/airbyte-integrations/connectors/destination-glide/todo.md index 3fbd6440a4310..50a863aa3b2bf 100644 --- a/airbyte-integrations/connectors/destination-glide/todo.md +++ b/airbyte-integrations/connectors/destination-glide/todo.md @@ -1,18 +1,20 @@ -- [+] feat: choose a strategy based on config -- [+] chore: stop writing any record data to logs -- [+] chore: cleanup logs generally -- [+] fix: "add rows failed with status 400: {"message":"More than 500 mutations"}" in mutation api (batch them to ~100 or something) -- [+] fix: batch row adds in rest api -- [+] chore: unify test framework across unit/integration tests (remove pytest?) -- [+] feat: snuggle rows in a batch -- [+] feat: support multiple streams from source by using a naming convention and discovering existing tables with GET /tables +- [x] feat: choose a strategy based on config +- [x] chore: stop writing any record data to logs +- [x] chore: cleanup logs generally +- [x] fix: "add rows failed with status 400: {"message":"More than 500 mutations"}" in mutation api (batch them to ~100 or something) +- [x] fix: batch row adds in rest api +- [x] chore: unify test framework across unit/integration tests (remove pytest?) +- [x] feat: snuggle rows in a batch +- [x] feat: support multiple streams from source by using a naming convention and discovering existing tables with GET /tables +- [ ] feat: add airbyte\_ prefix to all tables to prevent user from inadvertently overwriting existing table. - [ ] fix: replace "hostname" and "path" configs with "baseUrl" +- [ ] fix: use friendly names for the properties on the config page - [ ] feat: verify actual host/api/auth connection in check: https://docs.airbyte.com/understanding-airbyte/airbyte-protocol#check - [ ] feat: add a default timeout everywhere per https://requests.readthedocs.io/en/latest/user/quickstart/#timeouts -- [ ] fix: use friendly names for the properties on the config page - [ ] chore: update readme +- [ ] feat: incremental update support for airbyte (explore what APIs we need to do this) - [ ] chore: unit test for Column to ensure it serializes to json as expected -- [ ] chore: remove GlideBigTableMutationsStrategy? +- [x] chore: remove GlideBigTableMutationsStrategy? - [ ] chore: clean up todos - [ ] chore: figure out why the dev-write script stopped working (freezes) From 7c2e2ce73e90acd7f62571330fa7364fed61c4fa Mon Sep 17 00:00:00 2001 From: Scott Willeke Date: Mon, 22 Jul 2024 16:21:46 -0700 Subject: [PATCH 20/39] fix: updates the 'add rows' API request to support the new payload shape + tests --- .../destination_glide/glide.py | 14 +-- .../GlideBigTableRestStrategy_int_test.py | 119 ++++++++++++++++++ .../scripts/test-integration.sh | 13 ++ 3 files changed, 136 insertions(+), 10 deletions(-) create mode 100644 airbyte-integrations/connectors/destination-glide/integration_tests/GlideBigTableRestStrategy_int_test.py create mode 100755 airbyte-integrations/connectors/destination-glide/scripts/test-integration.sh diff --git a/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py b/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py index 8c0bb89ce391d..a60849edc9953 100644 --- a/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py +++ b/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py @@ -107,11 +107,11 @@ def create(cls, strategy: str) -> GlideBigTableBase: Creates a new instance of the default implementation for the GlideBigTable API client. """ implementation_map = { - "tables": GlideBigTableRestStrategy() + "tables": lambda: GlideBigTableRestStrategy() } if strategy not in implementation_map: raise ValueError(f"Strategy '{strategy}' not found. Expected one of '{implmap.keys()}'.") # nopep8 - return implementation_map[strategy] + return implementation_map[strategy]() class GlideBigTableRestStrategy(GlideBigTableBase): @@ -152,18 +152,12 @@ def raise_if_set_schema_not_called(self): "set_schema must be called before add_rows or commit") def _add_row_batch(self, rows: List[BigTableRow]) -> None: - # TODO: add rows to stash/serial https://web.postman.co/workspace/glideapps-Workspace~46b48d24-5fc1-44b6-89aa-8d6751db0fc5/request/9026518-c282ef52-4909-4806-88bf-08510ee80770 logger.debug(f"Adding rows batch with size {len(rows)}") r = requests.post( self.url(f"stashes/{self.stash_id}/{self.stash_serial}"), headers=self.headers(), - json={ - "data": rows, - "options": { - # ignore columns in rows that are not part of schema: - "unknownColumns": "ignore" - } - } + json=rows + ) try: r.raise_for_status() diff --git a/airbyte-integrations/connectors/destination-glide/integration_tests/GlideBigTableRestStrategy_int_test.py b/airbyte-integrations/connectors/destination-glide/integration_tests/GlideBigTableRestStrategy_int_test.py new file mode 100644 index 0000000000000..f5ffd39298abe --- /dev/null +++ b/airbyte-integrations/connectors/destination-glide/integration_tests/GlideBigTableRestStrategy_int_test.py @@ -0,0 +1,119 @@ +from datetime import datetime +from destination_glide.glide import GlideBigTableRestStrategy, Column, GlideBigTableFactory +import os +import unittest +from unittest import skip +from unittest.mock import patch +import uuid +import logging +import random + +log = logging.getLogger("test") +log.setLevel(logging.DEBUG) + +class TestGlideBigTableRestStrategy(unittest.TestCase): + ''' + Tests against a working Glide /tables API endpoint rather than being mocked like the one in unit tests. + ''' + + api_host = "https://functions.prod.internal.glideapps.com" + api_key = None + api_path_root = "api" + + + def setUp(self): + self.api_key = os.getenv("GLIDE_API_KEY") + if self.api_key is None: + raise Exception("GLIDE_API_KEY environment variable is not set.") + + # The protocol is to call `init`, `set_schema`, `add_rows` one or more times, and `commit` in that order. + + def test_new_table(self): + + # init + gbt = GlideBigTableFactory().create("tables") + + table_name = f"test-table-{str(uuid.uuid4())}" + gbt.init(self.api_host, self.api_key, self.api_path_root, table_name) + + # set_schema + test_columns = [ + Column("test-str", "string"), + Column("test-num", "number") + ] + gbt.set_schema(test_columns) + + # add_rows + for batch in range(3): + now = datetime.now() + test_rows = range(3) + test_rows = [ + { + "test-str": f"test-str-{now.isoformat()}-{batch}-{i}", + "test-num": (batch * 1000) + i, + } + for i in test_rows + ] + + # this creates the stashes: + gbt.add_rows(test_rows) + + ## this commits the stages by upserting the table: + # wraps= allows us to spy on the gbt's method here and confirm it created the table rather than overwrote it: + with patch.object(gbt, "overwrite_table_from_stash", wraps=gbt.overwrite_table_from_stash) as mock_overwrite_table_from_stash: + with patch.object(gbt, "create_table_from_stash", wraps=gbt.create_table_from_stash) as mock_create_table_from_stash: + gbt.commit() + mock_overwrite_table_from_stash.assert_not_called() + mock_create_table_from_stash.assert_called_once() + + + def test_updating_table(self): + # init + + table_name = f"test-table-{str(uuid.uuid4())}" + gbt = GlideBigTableFactory().create("tables") + gbt.init(self.api_host, self.api_key, self.api_path_root, table_name) + + # set_schema + test_columns = [ + Column("test-str", "string"), + Column("test-num", "number") + ] + gbt.set_schema(test_columns) + + # add_rows + test_rows = [ + { + "test-str": f"test-str-{datetime.now().isoformat()}", + "test-num": random.randint(0, 100000), + } + ] + gbt.add_rows(test_rows) + + ## this commits the stages by upserting the table: + gbt.commit() + + ##### NOW update the existing table we just created: + + # now do the update the second table now: + gbt = GlideBigTableFactory().create("tables") + gbt.init(self.api_host, self.api_key, self.api_path_root, table_name) + gbt.set_schema(test_columns) + + now = datetime.now() + test_rows = [ + { + "test-str": f"test-str-{datetime.now().isoformat()}", + "test-num": random.randint(0, 100000), + } + ] + gbt.add_rows(test_rows) + + # wraps= allows us to spy on the gbt's method here and confirm it overwrote the table rather than created it: + with patch.object(gbt, "overwrite_table_from_stash", wraps=gbt.overwrite_table_from_stash) as mock_overwrite_table_from_stash: + with patch.object(gbt, "create_table_from_stash", wraps=gbt.create_table_from_stash) as mock_create_table_from_stash: + gbt.commit() + mock_overwrite_table_from_stash.assert_called_once() + mock_create_table_from_stash.assert_not_called() + + diff --git a/airbyte-integrations/connectors/destination-glide/scripts/test-integration.sh b/airbyte-integrations/connectors/destination-glide/scripts/test-integration.sh new file mode 100755 index 0000000000000..14a20ed2ef35c --- /dev/null +++ b/airbyte-integrations/connectors/destination-glide/scripts/test-integration.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +this_dir=$(cd $(dirname "$0"); pwd) # this script's directory +this_script=$(basename $0) + +# if not GLIDE_API_KEY then print error and exit +if [ -z "$GLIDE_API_KEY" ]; then + echo "**************************************************" + echo "GLIDE_API_KEY is not set." + echo "You probably want to run this like \`GLIDE_API_KEY=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxx ./${this_script}\`\n" + exit 1 +fi + +poetry run pytest integration_tests "$@" From fff1f05a0e4fcbf3f744483dedd68e2913c04acc Mon Sep 17 00:00:00 2001 From: Scott Willeke Date: Wed, 24 Jul 2024 18:22:22 -0700 Subject: [PATCH 21/39] fix: update to new api domain, remove API 'version' config --- .../destination_glide/destination.py | 15 +++++------ .../destination_glide/glide.py | 15 +++-------- .../GlideBigTableRestStrategy_int_test.py | 27 +++++++++++++------ .../destination-glide/scripts/test-unit.sh | 1 + .../GlideBigTableRestStrategy_test.py | 7 +++-- 5 files changed, 33 insertions(+), 32 deletions(-) diff --git a/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py b/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py index 0f7dcabe4a0ec..34f798d044e72 100644 --- a/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py +++ b/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py @@ -22,8 +22,8 @@ from typing import Any, Iterable, Mapping import uuid -CONFIG_GLIDE_API_VERSION_DEFAULT = "tables" -CONFIG_GLIDE_API_HOST_DEFAULT = "https://api.glideapp.io" +CONFIG_GLIDE_API_HOST_DEFAULT = "https://api.glideapps.com" +CONFIG_GLIDE_API_PATH_ROOT_DEFAULT = "" logger = logging.getLogger(__name__) logger.setLevel(LOG_LEVEL_DEFAULT) @@ -73,19 +73,16 @@ def write( """ # load user-specified config: api_host = config.get('api_host', CONFIG_GLIDE_API_HOST_DEFAULT) - api_path_root = config['api_path_root'] - api_key = config['api_key'] - glide_api_version = config.get( - 'glide_api_version', CONFIG_GLIDE_API_VERSION_DEFAULT) + api_path_root = config.get('api_path_root', CONFIG_GLIDE_API_PATH_ROOT_DEFAULT) + api_key = config.get('api_key') # configure the table based on the stream catalog: # choose a strategy based on config: def create_table_client_for_stream(stream_name): # TODO: sanitize stream_name chars and length for GBT name - glide = GlideBigTableFactory.create(glide_api_version) - logger.debug(f"Using glide api strategy '{glide.__class__.__name__}' for glide_api_version '{glide_api_version}'.") # nopep8 - glide.init(api_host, api_key, api_path_root, stream_name) + glide = GlideBigTableFactory.create() + glide.init(api_key, stream_name, api_host, api_path_root) return glide table_clients = {} diff --git a/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py b/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py index a60849edc9953..d4dee296c4b7a 100644 --- a/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py +++ b/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py @@ -18,7 +18,6 @@ "json", ] - class Column(dict): """ Represents a Column in the glide API. @@ -54,7 +53,7 @@ def headers(self) -> Dict[str, str]: } def url(self, path: str) -> str: - return f"{self.api_host}/{self.api_path_root}/{path}" + return f"{self.api_host}/{self.api_path_root + '/' if self.api_path_root != '' else ''}{path}" """ An API client for interacting with a Glide Big Table. The intention is to @@ -64,7 +63,7 @@ def url(self, path: str) -> str: The protocol is to call `init`, `set_schema`, `add_rows` one or more times, and `commit` in that order. """ - def init(self, api_host, api_key, api_path_root, table_name): + def init(self, api_key, table_name, api_host="https://api.glideapps.com", api_path_root=""): """ Sets the connection information for the table. """ @@ -102,17 +101,11 @@ class GlideBigTableFactory: Factory for creating a GlideBigTableBase API client. """ @classmethod - def create(cls, strategy: str) -> GlideBigTableBase: + def create(cls) -> GlideBigTableBase: """ Creates a new instance of the default implementation for the GlideBigTable API client. """ - implementation_map = { - "tables": lambda: GlideBigTableRestStrategy() - } - if strategy not in implementation_map: - raise ValueError(f"Strategy '{strategy}' not found. Expected one of '{implmap.keys()}'.") # nopep8 - return implementation_map[strategy]() - + return GlideBigTableRestStrategy() class GlideBigTableRestStrategy(GlideBigTableBase): def reset(self): diff --git a/airbyte-integrations/connectors/destination-glide/integration_tests/GlideBigTableRestStrategy_int_test.py b/airbyte-integrations/connectors/destination-glide/integration_tests/GlideBigTableRestStrategy_int_test.py index f5ffd39298abe..9b0942d0fd19d 100644 --- a/airbyte-integrations/connectors/destination-glide/integration_tests/GlideBigTableRestStrategy_int_test.py +++ b/airbyte-integrations/connectors/destination-glide/integration_tests/GlideBigTableRestStrategy_int_test.py @@ -16,9 +16,20 @@ class TestGlideBigTableRestStrategy(unittest.TestCase): Tests against a working Glide /tables API endpoint rather than being mocked like the one in unit tests. ''' - api_host = "https://functions.prod.internal.glideapps.com" api_key = None - api_path_root = "api" + + env = "prod" + if (env == "poc"): + api_host = "https://functions.prod.internal.glideapps.com" + api_path_root = "api" + elif env=="staging": + api_host = "https://api.staging.glideapps.com" + api_path_root = "" + elif env=="prod": + api_host = "https://api.glideapps.com" + api_path_root = "" + else: + raise Exception(f"Unknown env: {env}") def setUp(self): @@ -31,10 +42,10 @@ def setUp(self): def test_new_table(self): # init - gbt = GlideBigTableFactory().create("tables") + gbt = GlideBigTableFactory().create() table_name = f"test-table-{str(uuid.uuid4())}" - gbt.init(self.api_host, self.api_key, self.api_path_root, table_name) + gbt.init(self.api_key, table_name, self.api_host, self.api_path_root) # set_schema test_columns = [ @@ -71,8 +82,8 @@ def test_updating_table(self): # init table_name = f"test-table-{str(uuid.uuid4())}" - gbt = GlideBigTableFactory().create("tables") - gbt.init(self.api_host, self.api_key, self.api_path_root, table_name) + gbt = GlideBigTableFactory().create() + gbt.init(self.api_key, table_name, self.api_host, self.api_path_root) # set_schema test_columns = [ @@ -96,8 +107,8 @@ def test_updating_table(self): ##### NOW update the existing table we just created: # now do the update the second table now: - gbt = GlideBigTableFactory().create("tables") - gbt.init(self.api_host, self.api_key, self.api_path_root, table_name) + gbt = GlideBigTableFactory().create() + gbt.init(self.api_key, table_name, self.api_host, self.api_path_root) gbt.set_schema(test_columns) now = datetime.now() diff --git a/airbyte-integrations/connectors/destination-glide/scripts/test-unit.sh b/airbyte-integrations/connectors/destination-glide/scripts/test-unit.sh index 52bcb42674cb9..8f55f0e395b75 100755 --- a/airbyte-integrations/connectors/destination-glide/scripts/test-unit.sh +++ b/airbyte-integrations/connectors/destination-glide/scripts/test-unit.sh @@ -2,4 +2,5 @@ this_dir=$(cd $(dirname "$0"); pwd) # this script's directory this_script=$(basename $0) +# NOTE: -k EXPRESSION Only run tests which match the given substring expression. An expression is a Python evaluable expression where all names are substring-matched against test names and their parent classes. Example: -k 'test_method or test_other' matches all test functions and classes whose name contains 'test_method' or 'test_other'... poetry run pytest unit_tests "$@" diff --git a/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableRestStrategy_test.py b/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableRestStrategy_test.py index ade04df3cf817..d0e4721c2cba2 100644 --- a/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableRestStrategy_test.py +++ b/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableRestStrategy_test.py @@ -24,8 +24,7 @@ def setUp(self): self.table_name = f"test-table-name-{str(uuid.uuid4())}" self.stash_id = f"stash-id-{str(uuid.uuid4())}" self.gbt = GlideBigTableRestStrategy() - self.gbt.init(self.api_host, self.api_key, - self.api_path_root, self.table_name) + self.gbt.init(self.api_key, self.table_name, self.api_host, self.api_path_root) def mock_post_for_set_schema(self, mock_post): mock_post.return_value.status_code = 200 @@ -71,7 +70,7 @@ def test_add_rows(self, mock_post): mock_post.assert_called_once() self.assertEqual( - mock_post.call_args.kwargs["json"]["data"], test_rows) + mock_post.call_args.kwargs["json"], test_rows) @patch.object(requests, "post") def test_add_rows_batching(self, mock_post): @@ -90,7 +89,7 @@ def test_add_rows_batching(self, mock_post): self.assertEqual(5, mock_post.call_count) # validate that the last row is what we expect: - self.assertEqual(mock_post.call_args.kwargs["json"]["data"], + self.assertEqual(mock_post.call_args.kwargs["json"], [ {"test-str": f"one {TEST_ROW_COUNT-1}", "test-num": TEST_ROW_COUNT-1} ]) From 824f67df612a4bd80f4cfb9db1c5202a8c536952 Mon Sep 17 00:00:00 2001 From: Scott Willeke Date: Thu, 25 Jul 2024 16:43:03 -0700 Subject: [PATCH 22/39] feat: pushes airbyte to Glide GCP Artifact registry The readme contains instructions for how to use this with Airbyte and how to push new versions. There were also some fixes to our airbyte config to make the API path and host optional and they default to prod. This is probably getting close to closing glideapps/glide#27833 once it is reviewed --- .../connectors/destination-glide/README.md | 60 ++++++++++++++++++- .../destination_glide/spec.json | 6 +- .../scripts/build-docker-image.sh | 3 +- .../scripts/push-docker-image.sh | 27 +++++---- 4 files changed, 79 insertions(+), 17 deletions(-) diff --git a/airbyte-integrations/connectors/destination-glide/README.md b/airbyte-integrations/connectors/destination-glide/README.md index 9115f521d6e7c..5a0c9343eca9c 100644 --- a/airbyte-integrations/connectors/destination-glide/README.md +++ b/airbyte-integrations/connectors/destination-glide/README.md @@ -47,6 +47,12 @@ There are a set of simple integration tests that Airbyte provides that can be tr These simply call commands that Airbyte provides in their connector template. The dev-write one appears to be the most comprehensive, but I've struggled to get that one to consistently run (see TODO). +There are additional tests you can run against the live API in `integration_tests/` that you can run with: + +```sh +./scripts/test-integration.sh +``` + ### Build & Deployment The Airbyte destination is packed as Docker image. This script uses Airbyte-provided tooling named `airbyte-ci` that leverages the same tooling they use in their CI pipeline to build the container. @@ -57,7 +63,22 @@ To install the tooling see [`airbyte-ci` README in this repo](https://github.com ./scripts/build-docker-image.sh ``` -We are currently deploying this to a public repository for ease of access from an Airbyte OSS instance. To deploy it to a docker container registry use the script at: +We are deploying this to Glide's docker repository in Google Artifact Registry (GAR) for ease of access from an Airbyte OSS instance. To deploy it to a docker container registry you need to authenticate docker to GAR and push it. The steps are: + +#### Glide Google Artifact Registry Docker Repo: + +Our repo is at https://console.cloud.google.com/artifacts/docker/glide-connectors/us-central1/airbyte-glide-destination?project=glide-connectors + +#### Pushing a Docker Image to Google Artifact Registry + +Read access is available for all, but to push you have to authenticate. Authenticate docker by running the following command which adds credential helper to `~/.docker/config.json`: + +``` +gcloud auth configure-docker \ + us-central1-docker.pkg.dev +``` + +Then you can use gcloud and docker commands as normal. To push a new image version run: ```sh ./scripts/push-docker-image.sh @@ -69,13 +90,48 @@ To install Airbyte follow the guide at https://docs.airbyte.com/deploying-airbyt Once install it should be available at http://localhost:8000/. You should have been prompted for username/pw during install. -### Installing Glide Destination in Airbyte OSS +### Installing Glide Destination in Airbyte OSS (on Kubernetes) + +Install kind (macOS): + +NOTE: abctl installs kind, but it doesn't install the kind CLI, so to work with the airbyte kubernetes cluster follow these steps: + +```sh +brew install kind +``` + +Then you can list the clusters and you should see one named `airbyte-abctl`: + +```sh +$ kind get clusters +airbyte-abctl +``` + +To use it with kubectl (or k9s) use: + +```sh +# this updates ~/.kube/config to add the cluster +kind export kubeconfig --name airbyte-abctl + +# set the context to the cluster context that kind added: +kubectl config set-context kind-airbyte-abctl + +# now kubectl works! +kubectl get namespaces +kubectl get -n airbyte-abctl pods +``` + +Now we follow the course guidance at https://docs.airbyte.com/operator-guides/using-custom-connectors/#for-kubernetes-airbyte-deployments ... + +We made our docker registry public so you don't have to authenticate airbyte's kubernetes deployment. To install the destination into Airbyte OSS follow these steps: 1. Click on **Settings** on the far left then select **Destinations** in the sub-panel. You should see a list of **Available destination connectors**. 2. At the top click the **+ New Connector** button fill in the fields. The **Docker repository name** and **Docker image tag** are the important bits. +For repository name use `us-central1-docker.pkg.dev/glide-connectors/airbyte-glide-destination/destination-glide` and for tag, run `gcloud artifacts docker tags list us-central1-docker.pkg.dev/glide-connectors/airbyte-glide-destination/destination-glide` to get the available tags and choose the latest. + Once installed, you can upgrade it to a new version by visiting the same settings page and changing the tag in the **Change to** box and clicking the **Change** button. --- diff --git a/airbyte-integrations/connectors/destination-glide/destination_glide/spec.json b/airbyte-integrations/connectors/destination-glide/destination_glide/spec.json index f32a6b07840a4..8f8f8dbe4ee2b 100644 --- a/airbyte-integrations/connectors/destination-glide/destination_glide/spec.json +++ b/airbyte-integrations/connectors/destination-glide/destination_glide/spec.json @@ -6,16 +6,16 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "Destination Glide", "type": "object", - "required": ["api_host", "api_path_root", "api_key"], + "required": ["api_key"], "additionalProperties": false, "properties": { "api_host": { "type": "string", - "description": "The protocol and host name (e.g. 'https://api.glideapp.io'). of the Glide API destination" + "description": "The protocol and host name of the Glide API destination. Defaults to 'https://api.glideapps.com'." }, "api_path_root": { "type": "string", - "description": "The path root of the Glide API destination" + "description": "The path root of the Glide API destination. Defaults to empty." }, "api_key": { "type": "string", diff --git a/airbyte-integrations/connectors/destination-glide/scripts/build-docker-image.sh b/airbyte-integrations/connectors/destination-glide/scripts/build-docker-image.sh index bdebc21ea7412..fe64db0fb3339 100755 --- a/airbyte-integrations/connectors/destination-glide/scripts/build-docker-image.sh +++ b/airbyte-integrations/connectors/destination-glide/scripts/build-docker-image.sh @@ -2,5 +2,6 @@ this_dir=$(cd $(dirname "$0"); pwd) # this script's directory this_script=$(basename $0) -airbyte-ci connectors --name=destination-glide build +# segment sometimes fails, so AIRBYTE_CI_DISABLE_TELEMETRY=true to disable it per airbyte-ci/connectors/pipelines/README.md +AIRBYTE_CI_DISABLE_TELEMETRY=true airbyte-ci connectors --name=destination-glide build diff --git a/airbyte-integrations/connectors/destination-glide/scripts/push-docker-image.sh b/airbyte-integrations/connectors/destination-glide/scripts/push-docker-image.sh index 37054bc1500c0..53fb62033fe6c 100755 --- a/airbyte-integrations/connectors/destination-glide/scripts/push-docker-image.sh +++ b/airbyte-integrations/connectors/destination-glide/scripts/push-docker-image.sh @@ -3,15 +3,22 @@ this_dir=$(cd $(dirname "$0"); pwd) # this script's directory this_script=$(basename $0) #!/usr/bin/env bash +LOCAL_BUILT_IMAGE_NAME=airbyte/destination-glide:dev +GLIDE_DOCKER_IMAGE_NAME=us-central1-docker.pkg.dev/glide-connectors/airbyte-glide-destination/destination-glide + +echo "Pushing to remote image repository: $GLIDE_DOCKER_IMAGE_NAME" +docker inspect --format='Pushing the local image "{{index .RepoTags 0}}" created at "{{.Created}}"' $LOCAL_BUILT_IMAGE_NAME + # Fetch the list of tags from Docker Hub -tags=$(wget -q -O - "https://hub.docker.com/v2/namespaces/activescott/repositories/destination-glide/tags?page_size=10" | grep -o '"name": *"[^"]*' | grep -o '[^"]*$' | grep -E '([0-9]+\.)+[0-9]+' ) -`` -echo "Found tags: $tags" +#tags=$(wget -q -O - "https://hub.docker.com/v2/namespaces/activescott/repositories/destination-glide/tags?page_size=10" | grep -o '"name": *"[^"]*' | grep -o '[^"]*$' | grep -E '([0-9]+\.)+[0-9]+' ) +# Get the list of tags using gcloud with the appropriate format +tags=$(gcloud artifacts docker tags list $GLIDE_DOCKER_IMAGE_NAME --format="get(tag)") + # Sort the tags and get the highest one -highest_tag=$(echo "$tags" | sort -V | tail -n 1) +highest_tag=$(echo "$tags" | awk -F'/' '{print $NF}' | sort -V | tail -n 1) -echo "found highest tag: $highest_tag" +echo "found highest tag on remote: $highest_tag" # Increment the version IFS='.' read -ra ADDR <<< "$highest_tag" @@ -28,12 +35,10 @@ else exit 1 fi - # Tag the local image with the new version -# TODO: airbyte/destination-glide is the local docker image that airbyte's CI builds it and names it as locally. Can't we change this ?? -docker image tag airbyte/destination-glide:dev activescott/destination-glide:$new_version - -# Push the image with the new tag -docker push activescott/destination-glide:$new_version +# NOTE: airbyte/destination-glide is the local docker image that airbyte's CI builds and names locally. +docker image tag $LOCAL_BUILT_IMAGE_NAME $GLIDE_DOCKER_IMAGE_NAME:$new_version +# Push the image with the new tag +docker push $GLIDE_DOCKER_IMAGE_NAME:$new_version From 2edeb2a393f903645b7d7e2c6d116454c4157407 Mon Sep 17 00:00:00 2001 From: Scott Willeke Date: Thu, 25 Jul 2024 17:13:28 -0700 Subject: [PATCH 23/39] fix: updated the docker push script to optionally tag the pushed image as latest --- .../destination-glide/scripts/push-docker-image.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/airbyte-integrations/connectors/destination-glide/scripts/push-docker-image.sh b/airbyte-integrations/connectors/destination-glide/scripts/push-docker-image.sh index 53fb62033fe6c..5ed039611e1e8 100755 --- a/airbyte-integrations/connectors/destination-glide/scripts/push-docker-image.sh +++ b/airbyte-integrations/connectors/destination-glide/scripts/push-docker-image.sh @@ -42,3 +42,15 @@ docker image tag $LOCAL_BUILT_IMAGE_NAME $GLIDE_DOCKER_IMAGE_NAME:$new_version # Push the image with the new tag docker push $GLIDE_DOCKER_IMAGE_NAME:$new_version + +echo "Do you want to make this new version the 'latest' tag in Docker too (so that new pulls get this version by default)? (y/n)" +read answer + +if [ "$answer" != "${answer#[Yy]}" ] ; then + echo "Taging latest..." + docker image tag $LOCAL_BUILT_IMAGE_NAME $GLIDE_DOCKER_IMAGE_NAME:latest + docker push $GLIDE_DOCKER_IMAGE_NAME:latest +else + echo "Exiting..." + exit 0 +fi From 1441dca32ab7b838beb9d3d9a66dd951321b1e73 Mon Sep 17 00:00:00 2001 From: Scott Willeke Date: Tue, 6 Aug 2024 08:02:22 -0700 Subject: [PATCH 24/39] chore: temporary script that is pushing to airbyte cloud. You'll need to modify the values --- .../scripts/push-to-airbyte-cloud.sh | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100755 airbyte-integrations/connectors/destination-glide/scripts/push-to-airbyte-cloud.sh diff --git a/airbyte-integrations/connectors/destination-glide/scripts/push-to-airbyte-cloud.sh b/airbyte-integrations/connectors/destination-glide/scripts/push-to-airbyte-cloud.sh new file mode 100755 index 0000000000000..daa74a0e6aedf --- /dev/null +++ b/airbyte-integrations/connectors/destination-glide/scripts/push-to-airbyte-cloud.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +this_dir=$(cd $(dirname "$0"); pwd) # this script's directory +this_script=$(basename $0) + +#!/usr/bin/env bash + +GLIDE_DOCKER_IMAGE_NAME=us-central1-docker.pkg.dev/glide-connectors/airbyte-glide-destination/destination-glide +#us-central1-docker.pkg.dev/airbyte-custom-connectors//:tag-a + +GLIDE_AT_AIRBYTE_COMPANY_NAME=glide # ERROR: (gcloud.artifacts.docker.tags.list) PERMISSION_DENIED: Permission 'artifactregistry.repositories.get' denied on resource '//artifactregistry.googleapis.com/projects/airbyte-custom-connectors/locations/us-central1/repositories/glide' (or it may not exist). This command is authenticated as scott.willeke@heyglide.com which is the active account specified by the [core/account] property. +GLIDE_AT_AIRBYTE_COMPANY_NAME=heyglide +AIRBYTE_DOCKER_IMAGE_NAME=us-central1-docker.pkg.dev/airbyte-custom-connectors/$GLIDE_AT_AIRBYTE_COMPANY_NAME/destination-glide + +#echo "Pushing to remote image repository: $GLIDE_DOCKER_IMAGE_NAME" +#docker inspect --format='Pushing the local image "{{index .RepoTags 0}}" created at "{{.Created}}"' $LOCAL_BUILT_IMAGE_NAME + +echo "Getting the list of tags in the airbyte cloud docker image repository:" +gcloud artifacts docker tags list $AIRBYTE_DOCKER_IMAGE_NAME + + +new_version=0.0.29 + +echo "" +echo "The new version will be $new_version and will be pushed to Airbyte's docker repo for cloud at "$AIRBYTE_DOCKER_IMAGE_NAME". Do you want to continue? (y/n)" +read answer + +if [ "$answer" != "${answer#[Yy]}" ] ; then + echo "Continuing..." +else + echo "Exiting..." + exit 1 +fi + +docker image tag $GLIDE_DOCKER_IMAGE_NAME:$new_version $AIRBYTE_DOCKER_IMAGE_NAME:$new_version +docker push $AIRBYTE_DOCKER_IMAGE_NAME:$new_version + From dc76b148ca1328adb670c3182a7adc8597e8d39d Mon Sep 17 00:00:00 2001 From: Alex Corrado Date: Fri, 2 Aug 2024 16:48:29 +0100 Subject: [PATCH 25/39] Update README to clarify setup steps --- .../connectors/destination-glide/README.md | 41 ++++++++++++------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/airbyte-integrations/connectors/destination-glide/README.md b/airbyte-integrations/connectors/destination-glide/README.md index 5a0c9343eca9c..11f9b56d6f877 100644 --- a/airbyte-integrations/connectors/destination-glide/README.md +++ b/airbyte-integrations/connectors/destination-glide/README.md @@ -5,37 +5,48 @@ For information about how to use this connector within Airbyte, see [the documen ## Development -The active todo list is at [./todo.md]. -The gist of the Glide-specific code is in `/destination_glide/destination.py` and `/destination_glide/glide.py`. +The active todo list is at `./todo.md`. -### Prerequisites +The gist of the Glide-specific code is in `destination_glide/destination.py` and `destination_glide/glide.py`. -- Python (`^3.9`, tested recently with `3.12.3`) -- Poetry (`^1.7`, tested recently with `1.8.3_1`) +### Setup -I used homebrew for installing these prerequisites on macOS. +1. Ensure you have the following prerequisites installed: -### Unit Tests + - Python (`^3.9`, tested recently with `3.12.3`) + - Poetry (`^1.7`, tested recently with `1.8.3_1`) -The unit tests for that code are in `/destination-glide/unit_tests`. To run them run: + You can use homebrew to install these on macOS. + +2. Once you have the above, run: -```sh -./scripts/test-unit.sh ``` +poetry install +``` + +### Running the Tests -### Integration Tests +Create the file `secrets/config.json`. It must confirm to the configuration specification in `destination_glide/spec.json`, which also specifies the configuration UI within the Airbyte product itself for configuring the destination. -The destination has a configuration in `/secrets/config.json`. That file must confirm to the configuration specification in `/destination_glide/spec.json`. It should be something like: +It should be something like: ```json { - "api_host": "http://localhost:5005", - "api_path_root": "api", + "api_host": "https://api.staging.glideapps.com", + "api_path_root": "", "api_key": "decafbad-1234-1234-1234-decafbad" } ``` -The spec also specifies the configuration UI within the Airbyte product itself for configuring the destination. +#### Unit Tests + +The unit tests for that code are in `destination-glide/unit_tests`. To run them run: + +```sh +./scripts/test-unit.sh +``` + +#### Integration Tests There are a set of simple integration tests that Airbyte provides that can be triggered with the following scripts: From a46808d4e3ce951bb02e1363023d4208fe661ec8 Mon Sep 17 00:00:00 2001 From: Alex Corrado Date: Fri, 2 Aug 2024 16:49:10 +0100 Subject: [PATCH 26/39] Fix doc comment --- .../destination-glide/destination_glide/glide.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py b/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py index d4dee296c4b7a..3cf5cded4fe2c 100644 --- a/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py +++ b/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py @@ -46,6 +46,14 @@ def __repr__(self): class GlideBigTableBase(ABC): + """ + An API client for interacting with a Glide Big Table. The intention is to + create a new table or update an existing table including the table's schema + and the table's rows. + + The protocol is to call `init`, `set_schema`, `add_rows` one or more times, and `commit` in that order. + """ + def headers(self) -> Dict[str, str]: return { "Content-Type": "application/json", @@ -55,14 +63,6 @@ def headers(self) -> Dict[str, str]: def url(self, path: str) -> str: return f"{self.api_host}/{self.api_path_root + '/' if self.api_path_root != '' else ''}{path}" - """ - An API client for interacting with a Glide Big Table. The intention is to - create a new table or update an existing table including the table's schema - and the table's rows. - - The protocol is to call `init`, `set_schema`, `add_rows` one or more times, and `commit` in that order. - """ - def init(self, api_key, table_name, api_host="https://api.glideapps.com", api_path_root=""): """ Sets the connection information for the table. From 19551c24b125161bf0e4cf982bbec933e5113511 Mon Sep 17 00:00:00 2001 From: Alex Corrado Date: Fri, 2 Aug 2024 17:07:33 +0100 Subject: [PATCH 27/39] Tidy a bit; no need to call create stash endpoint anymore --- .../destination_glide/destination.py | 3 +-- .../destination_glide/glide.py | 25 ++++++------------- 2 files changed, 8 insertions(+), 20 deletions(-) diff --git a/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py b/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py index 34f798d044e72..560c813504b66 100644 --- a/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py +++ b/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py @@ -91,8 +91,7 @@ def create_table_client_for_stream(stream_name): if configured_stream.destination_sync_mode != DestinationSyncMode.overwrite: raise Exception(f'Only destination sync mode overwrite is supported, but received "{configured_stream.destination_sync_mode}".') # nopep8 because https://github.com/hhatto/autopep8/issues/712 - glide = create_table_client_for_stream( - configured_stream.stream.name) + glide = create_table_client_for_stream(configured_stream.stream.name) # upsert the GBT with schema to set_schema for dumping the data into it columns = [] properties = configured_stream.stream.json_schema["properties"] diff --git a/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py b/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py index 3cf5cded4fe2c..1d48be44d74bd 100644 --- a/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py +++ b/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py @@ -1,4 +1,5 @@ from abc import ABC, abstractmethod +import uuid from .log import LOG_LEVEL_DEFAULT import logging import requests @@ -18,6 +19,9 @@ "json", ] +# TODO: to optimize batch size for variable number and size of columns, we could estimate row byte size based on the first row and choose a batch size based on that. +BATCH_SIZE = 500 + class Column(dict): """ Represents a Column in the glide API. @@ -109,7 +113,8 @@ def create(cls) -> GlideBigTableBase: class GlideBigTableRestStrategy(GlideBigTableBase): def reset(self): - self.stash_id = None + self.columns = None + self.stash_id = str(uuid.uuid4()) self.stash_serial = 0 def __init__(self): @@ -124,23 +129,9 @@ def set_schema(self, columns: List[Column]) -> None: raise ValueError("columns must be provided") self.reset() self.columns = columns - # Create stash we can stash records into for later - r = requests.post( - self.url(f"stashes"), - headers=self.headers(), - ) - try: - r.raise_for_status() - except Exception as e: - raise Exception(f"failed to create stash. Response was '{r.text}'") from e # nopep8 - - result = r.json() - self.stash_id = result["data"]["stashID"] - self.stash_serial = 0 - logger.info(f"Created stash for records with id '{self.stash_id}'") def raise_if_set_schema_not_called(self): - if self.stash_id is None: + if self.columns is None: raise ValueError( "set_schema must be called before add_rows or commit") @@ -162,8 +153,6 @@ def _add_row_batch(self, rows: List[BigTableRow]) -> None: def add_rows(self, rows: Iterator[BigTableRow]) -> None: self.raise_if_set_schema_not_called() - # TODO: to optimize batch size for variable number and size of columns, we could estimate row byte size based on the first row and choose a batch size based on that. - BATCH_SIZE = 500 batch = [] for i in range(0, len(rows), BATCH_SIZE): batch = rows[i:i + min(BATCH_SIZE, len(rows) - i)] From 00a7e9551746790061b451ba8edffb662e4f2659 Mon Sep 17 00:00:00 2001 From: Alex Corrado Date: Mon, 5 Aug 2024 13:30:29 +0100 Subject: [PATCH 28/39] Ignore columns that aren't in the schema --- .../connectors/destination-glide/destination_glide/glide.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py b/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py index 1d48be44d74bd..af66c729dc6b5 100644 --- a/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py +++ b/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py @@ -161,7 +161,7 @@ def add_rows(self, rows: Iterator[BigTableRow]) -> None: def create_table_from_stash(self) -> None: logger.info(f"Creating new table for table name '{self.table_name}'...") # nopep8 r = requests.post( - self.url(f"tables"), + self.url(f"tables?onSchemaError=dropColumns"), headers=self.headers(), json={ "name": self.table_name, @@ -183,7 +183,7 @@ def create_table_from_stash(self) -> None: def overwrite_table_from_stash(self, table_id) -> None: # overwrite the specified table's schema and rows with the stash: r = requests.put( - self.url(f"tables/{table_id}"), + self.url(f"tables/{table_id}?onSchemaError=dropColumns"), headers=self.headers(), json={ "schema": { From 5a89e513821ded39e313a7c095db3823ad8aaaff Mon Sep 17 00:00:00 2001 From: Alex Corrado Date: Mon, 5 Aug 2024 13:34:11 +0100 Subject: [PATCH 29/39] Simplify: encapsulate buffering into Glide client, don't bother with state --- .../destination_glide/destination.py | 36 ++------ .../destination_glide/glide.py | 87 +++++++++---------- .../destination_glide/spec.json | 2 +- 3 files changed, 50 insertions(+), 75 deletions(-) diff --git a/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py b/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py index 560c813504b66..835dac3cc57cc 100644 --- a/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py +++ b/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py @@ -79,10 +79,10 @@ def write( # configure the table based on the stream catalog: # choose a strategy based on config: - def create_table_client_for_stream(stream_name): + def create_table_client_for_stream(stream_name, columns): # TODO: sanitize stream_name chars and length for GBT name glide = GlideBigTableFactory.create() - glide.init(api_key, stream_name, api_host, api_path_root) + glide.init(api_key, stream_name, columns, api_host, api_path_root) return glide table_clients = {} @@ -91,7 +91,6 @@ def create_table_client_for_stream(stream_name): if configured_stream.destination_sync_mode != DestinationSyncMode.overwrite: raise Exception(f'Only destination sync mode overwrite is supported, but received "{configured_stream.destination_sync_mode}".') # nopep8 because https://github.com/hhatto/autopep8/issues/712 - glide = create_table_client_for_stream(configured_stream.stream.name) # upsert the GBT with schema to set_schema for dumping the data into it columns = [] properties = configured_stream.stream.json_schema["properties"] @@ -102,11 +101,10 @@ def create_table_client_for_stream(stream_name): Column(prop_name, airbyteTypeToGlideType(prop_type)) ) - glide.set_schema(columns) + glide = create_table_client_for_stream(configured_stream.stream.name, columns) table_clients[configured_stream.stream.name] = glide # stream the records into the GBT: - buffers = defaultdict(list) logger.debug("Processing messages...") for message in input_messages: logger.debug(f"processing message {message.type}...") @@ -119,36 +117,16 @@ def create_table_client_for_stream(stream_name): continue # add to buffer - record_data = message.record.data - record_id = str(uuid.uuid4()) - stream_buffer = buffers[stream_name] - stream_buffer.append( - (record_id, datetime.datetime.now().isoformat(), record_data)) + client = table_clients[stream_name] + client.add_row(message.record.data) logger.debug("buffering record complete.") elif message.type == Type.STATE: # `Type.State` is a signal from the source that we should save the previous batch of `Type.RECORD` messages to the destination. # It is a checkpoint that enables partial success. # See https://docs.airbyte.com/understanding-airbyte/airbyte-protocol#state--checkpointing - logger.info(f"Writing buffered records to Glide API from {len(buffers.keys())} streams...") # nopep8 - for stream_name in buffers.keys(): - stream_buffer = buffers[stream_name] - logger.info(f"Saving buffered records to Glide API (stream: '{stream_name}', record count: '{len(stream_buffer)}')...") # nopep8 - DATA_INDEX = 2 - data_rows = [row_tuple[DATA_INDEX] - for row_tuple in stream_buffer] - if len(data_rows) > 0: - if stream_name not in table_clients: - raise Exception( - f"Stream '{stream_name}' not found in table_clients") - glide = table_clients[stream_name] - glide.add_rows(data_rows) - stream_buffer.clear() - logger.info(f"Saving buffered records to Glide API complete.") # nopep8 because https://github.com/hhatto/autopep8/issues/712 - - # dump all buffers now as we just wrote them to the table: - buffers = defaultdict(list) - yield message + # FIXME: I don't think partial success applies to us since we only support overwrite mode anyway? + logger.info(f"Ignoring state message: {message.state}") else: logger.warn(f"Ignoring unknown Airbyte input message type: {message.type}") # nopep8 because https://github.com/hhatto/autopep8/issues/712 diff --git a/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py b/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py index af66c729dc6b5..bca381bd24e35 100644 --- a/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py +++ b/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py @@ -19,8 +19,7 @@ "json", ] -# TODO: to optimize batch size for variable number and size of columns, we could estimate row byte size based on the first row and choose a batch size based on that. -BATCH_SIZE = 500 +DEFAULT_BATCH_SIZE = 1500 class Column(dict): """ @@ -55,7 +54,7 @@ class GlideBigTableBase(ABC): create a new table or update an existing table including the table's schema and the table's rows. - The protocol is to call `init`, `set_schema`, `add_rows` one or more times, and `commit` in that order. + The protocol is to call `init`, then `add_row` or `add_rows` one or more times, and finally, `commit`, in that order. """ def headers(self) -> Dict[str, str]: @@ -67,21 +66,25 @@ def headers(self) -> Dict[str, str]: def url(self, path: str) -> str: return f"{self.api_host}/{self.api_path_root + '/' if self.api_path_root != '' else ''}{path}" - def init(self, api_key, table_name, api_host="https://api.glideapps.com", api_path_root=""): + def init(self, api_key, table_name, columns, api_host="https://api.glideapps.com", api_path_root="", batch_size = DEFAULT_BATCH_SIZE): """ Sets the connection information for the table. """ - self.api_host = api_host + # todo: validate args self.api_key = api_key + self.api_host = api_host self.api_path_root = api_path_root + self.table_name = table_name - # todo: validate args + self.columns = columns + + # TODO: to optimize batch size for variable number and size of columns, we could estimate row byte size based on the first row and choose a batch size based on that. + self.batch_size = batch_size @abstractmethod - def set_schema(self, columns: List[Column]) -> None: + def add_row(self, row: BigTableRow) -> None: """ - set_schemas the table with the given schema. - Each column is a json-schema property where the key is the column name and the type is the . + Adds a row to the table. """ pass @@ -112,54 +115,45 @@ def create(cls) -> GlideBigTableBase: return GlideBigTableRestStrategy() class GlideBigTableRestStrategy(GlideBigTableBase): - def reset(self): - self.columns = None - self.stash_id = str(uuid.uuid4()) - self.stash_serial = 0 - def __init__(self): super().__init__() - self.reset() - - def set_schema(self, columns: List[Column]) -> None: - logger.debug(f"set_schema columns: {columns}") - if columns is None: - raise ValueError("columns must be provided") - if len(columns) == 0: - raise ValueError("columns must be provided") - self.reset() - self.columns = columns + self.stash_id = str(uuid.uuid4()) + self.stash_serial = 0 + self.buffer = [] - def raise_if_set_schema_not_called(self): - if self.columns is None: - raise ValueError( - "set_schema must be called before add_rows or commit") + def _flush_buffer(self): + rows = self.buffer + if len(rows) == 0: + return + self.buffer.clear() - def _add_row_batch(self, rows: List[BigTableRow]) -> None: - logger.debug(f"Adding rows batch with size {len(rows)}") + path = f"stashes/{self.stash_id}/{self.stash_serial}" + logger.debug(f"Flushing {len(rows)} rows to {path} ...") r = requests.post( - self.url(f"stashes/{self.stash_id}/{self.stash_serial}"), + self.url(path), headers=self.headers(), json=rows - ) try: r.raise_for_status() except Exception as e: - raise Exception(f"failed to add rows batch for serial '{self.stash_serial}'. Response was '{r.text}'") from e # nopep8 + raise Exception(f"Failed to post rows batch to {path} : {r.text}") from e # nopep8 - logger.info(f"Added {len(rows)} rows as batch for serial '{self.stash_serial}' successfully.") # nopep8 + logger.info(f"Successfully posted {len(rows)} rows to {path}") # nopep8 self.stash_serial += 1 + def add_row(self, row: BigTableRow) -> None: + self.buffer.append(row) + if len(self.buffer) >= self.batch_size: + self._flush_buffer() + def add_rows(self, rows: Iterator[BigTableRow]) -> None: - self.raise_if_set_schema_not_called() - batch = [] - for i in range(0, len(rows), BATCH_SIZE): - batch = rows[i:i + min(BATCH_SIZE, len(rows) - i)] - self._add_row_batch(batch) + self.buffer.extend(rows) + if len(self.buffer) >= self.batch_size: + self._flush_buffer() def create_table_from_stash(self) -> None: - logger.info(f"Creating new table for table name '{self.table_name}'...") # nopep8 + logger.info(f"Creating new table '{self.table_name}' ...") # nopep8 r = requests.post( self.url(f"tables?onSchemaError=dropColumns"), headers=self.headers(), @@ -176,9 +170,9 @@ def create_table_from_stash(self) -> None: try: r.raise_for_status() except Exception as e: - raise Exception(f"failed to create table '{self.table_name}'. Response was '{r.text}'.") from e # nopep8 + raise Exception(f"Failed to create table '{self.table_name}' : {r.text}") from e # nopep8 - logger.info(f"Creating table '{self.table_name}' succeeded.") + logger.info(f"Successfully created table '{self.table_name}'") def overwrite_table_from_stash(self, table_id) -> None: # overwrite the specified table's schema and rows with the stash: @@ -197,10 +191,9 @@ def overwrite_table_from_stash(self, table_id) -> None: try: r.raise_for_status() except Exception as e: - raise Exception(f"failed to overwrite table '{table_id}'. Response was '{r.text}'") from e # nopep8 + raise Exception(f"Failed to overwrite table '{table_id}' : {r.text}") from e # nopep8 def commit(self) -> None: - self.raise_if_set_schema_not_called() # first see if the table already exists r = requests.get( self.url(f"tables"), @@ -209,7 +202,7 @@ def commit(self) -> None: try: r.raise_for_status() except Exception as e: - raise Exception(f"Failed to get table list. Response was '{r.text}'.") from e # nopep8 + raise Exception(f"Failed to get table list: {r.text}") from e # nopep8 found_table_id = None # confirm if table exists: @@ -223,6 +216,10 @@ def commit(self) -> None: logger.info(f"Found existing table to reuse for table name '{self.table_name}' with ID '{found_table_id}'.") # nopep8 break + # flush any remaining buffer to the stash + self._flush_buffer() + + # commit the stash to the table if found_table_id != None: self.overwrite_table_from_stash(found_table_id) else: diff --git a/airbyte-integrations/connectors/destination-glide/destination_glide/spec.json b/airbyte-integrations/connectors/destination-glide/destination_glide/spec.json index 8f8f8dbe4ee2b..bf3c77220d323 100644 --- a/airbyte-integrations/connectors/destination-glide/destination_glide/spec.json +++ b/airbyte-integrations/connectors/destination-glide/destination_glide/spec.json @@ -1,7 +1,7 @@ { "documentationUrl": "https://docs.airbyte.com/integrations/destinations/glide", "supported_destination_sync_modes": ["overwrite"], - "supportsIncremental": true, + "supportsIncremental": false, "connectionSpecification": { "$schema": "http://json-schema.org/draft-07/schema#", "title": "Destination Glide", From 7c21f2c9530f8b5bf7bff7cd625c69ca549cbc33 Mon Sep 17 00:00:00 2001 From: Alex Corrado Date: Mon, 5 Aug 2024 16:08:44 +0100 Subject: [PATCH 30/39] Fix up tests --- .../destination_glide/destination.py | 3 +- .../destination_glide/glide.py | 2 +- .../GlideBigTableRestStrategy_int_test.py | 19 ++--- .../GlideBigTableRestStrategy_test.py | 79 ++++--------------- .../unit_tests/destination_test.py | 27 +++---- 5 files changed, 34 insertions(+), 96 deletions(-) diff --git a/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py b/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py index 835dac3cc57cc..b0fd4896699c3 100644 --- a/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py +++ b/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py @@ -135,7 +135,8 @@ def create_table_client_for_stream(stream_name, columns): glide.commit() logger.info(f"Committed stream '{stream_name}' to Glide.") - pass + # see https://stackoverflow.com/a/36863998 + yield from () def check(self, logger: logging.Logger, config: Mapping[str, Any]) -> AirbyteConnectionStatus: """ diff --git a/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py b/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py index bca381bd24e35..a23bd7b2f9304 100644 --- a/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py +++ b/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py @@ -125,7 +125,7 @@ def _flush_buffer(self): rows = self.buffer if len(rows) == 0: return - self.buffer.clear() + self.buffer = [] path = f"stashes/{self.stash_id}/{self.stash_serial}" logger.debug(f"Flushing {len(rows)} rows to {path} ...") diff --git a/airbyte-integrations/connectors/destination-glide/integration_tests/GlideBigTableRestStrategy_int_test.py b/airbyte-integrations/connectors/destination-glide/integration_tests/GlideBigTableRestStrategy_int_test.py index 9b0942d0fd19d..9018de730d577 100644 --- a/airbyte-integrations/connectors/destination-glide/integration_tests/GlideBigTableRestStrategy_int_test.py +++ b/airbyte-integrations/connectors/destination-glide/integration_tests/GlideBigTableRestStrategy_int_test.py @@ -18,7 +18,7 @@ class TestGlideBigTableRestStrategy(unittest.TestCase): api_key = None - env = "prod" + env = "staging" if (env == "poc"): api_host = "https://functions.prod.internal.glideapps.com" api_path_root = "api" @@ -37,7 +37,7 @@ def setUp(self): if self.api_key is None: raise Exception("GLIDE_API_KEY environment variable is not set.") - # The protocol is to call `init`, `set_schema`, `add_rows` one or more times, and `commit` in that order. + # The protocol is to call `init`, `add_row` or `add_rows` one or more times, and `commit` in that order. def test_new_table(self): @@ -45,14 +45,11 @@ def test_new_table(self): gbt = GlideBigTableFactory().create() table_name = f"test-table-{str(uuid.uuid4())}" - gbt.init(self.api_key, table_name, self.api_host, self.api_path_root) - - # set_schema test_columns = [ Column("test-str", "string"), Column("test-num", "number") ] - gbt.set_schema(test_columns) + gbt.init(self.api_key, table_name, test_columns, self.api_host, self.api_path_root) # add_rows for batch in range(3): @@ -82,15 +79,12 @@ def test_updating_table(self): # init table_name = f"test-table-{str(uuid.uuid4())}" - gbt = GlideBigTableFactory().create() - gbt.init(self.api_key, table_name, self.api_host, self.api_path_root) - - # set_schema test_columns = [ Column("test-str", "string"), Column("test-num", "number") ] - gbt.set_schema(test_columns) + gbt = GlideBigTableFactory().create() + gbt.init(self.api_key, table_name, test_columns, self.api_host, self.api_path_root) # add_rows test_rows = [ @@ -108,8 +102,7 @@ def test_updating_table(self): # now do the update the second table now: gbt = GlideBigTableFactory().create() - gbt.init(self.api_key, table_name, self.api_host, self.api_path_root) - gbt.set_schema(test_columns) + gbt.init(self.api_key, table_name, test_columns, self.api_host, self.api_path_root) now = datetime.now() test_rows = [ diff --git a/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableRestStrategy_test.py b/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableRestStrategy_test.py index d0e4721c2cba2..712e1e2f5a6f6 100644 --- a/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableRestStrategy_test.py +++ b/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableRestStrategy_test.py @@ -12,7 +12,7 @@ class TestGlideBigTableRestStrategy(unittest.TestCase): api_path_root = "test/api/path/root" table_id = "" table_name = "" - stash_id = "" + batch_size = 100 test_columns = [ Column("test-str", "string"), @@ -22,64 +22,26 @@ class TestGlideBigTableRestStrategy(unittest.TestCase): def setUp(self): self.table_id = f"test-table-id-{str(uuid.uuid4())}" self.table_name = f"test-table-name-{str(uuid.uuid4())}" - self.stash_id = f"stash-id-{str(uuid.uuid4())}" self.gbt = GlideBigTableRestStrategy() - self.gbt.init(self.api_key, self.table_name, self.api_host, self.api_path_root) - - def mock_post_for_set_schema(self, mock_post): - mock_post.return_value.status_code = 200 - mock_post.return_value.json.return_value = { - "data": { - "stashID": self.stash_id - } - } - - @patch.object(requests, "post") - def test_set_schema_valid(self, mock_post): - self.mock_post_for_set_schema(mock_post) - - test_columns = [ - Column("test-str", "string"), - Column("test-num", "number") - ] - self.gbt.set_schema(test_columns) - - mock_post.assert_called_once() - - @patch.object(requests, "post") - def test_set_schema_invalid_col_type(self, mock_post): - self.mock_post_for_set_schema(mock_post) + self.gbt.init(self.api_key, self.table_name, self.test_columns, self.api_host, self.api_path_root, self.batch_size) + def test_invalid_col_type(self): with self.assertRaises(ValueError): - self.gbt.set_schema([ - Column("test-str", "string"), - Column("test-num", "invalid-type") - ]) + Column("test-num", "invalid-type") @patch.object(requests, "post") def test_add_rows(self, mock_post): - self.mock_post_for_set_schema(mock_post) - - self.gbt.set_schema(self.test_columns) - mock_post.reset_mock() test_rows = [ {"test-str": "one", "test-num": 1}, {"test-str": "two", "test-num": 2} ] self.gbt.add_rows(test_rows) - - mock_post.assert_called_once() - self.assertEqual( - mock_post.call_args.kwargs["json"], test_rows) + mock_post.assert_not_called() @patch.object(requests, "post") def test_add_rows_batching(self, mock_post): - self.mock_post_for_set_schema(mock_post) - - self.gbt.set_schema(self.test_columns) - - mock_post.reset_mock() - TEST_ROW_COUNT = 2001 + # the batch size isn't currently strict, so the extra row will be sent in the same batch + TEST_ROW_COUNT = self.batch_size + 1 test_rows = list([ {"test-str": f"one {i}", "test-num": i} for i in range(TEST_ROW_COUNT) @@ -87,22 +49,16 @@ def test_add_rows_batching(self, mock_post): self.gbt.add_rows(test_rows) - self.assertEqual(5, mock_post.call_count) - # validate that the last row is what we expect: - self.assertEqual(mock_post.call_args.kwargs["json"], - [ - {"test-str": f"one {TEST_ROW_COUNT-1}", "test-num": TEST_ROW_COUNT-1} - ]) + self.assertEqual(1, mock_post.call_count) + self.assertEqual(mock_post.call_args.kwargs["json"], test_rows) def test_commit_with_pre_existing_table(self): with patch.object(requests, "post") as mock_post: - self.mock_post_for_set_schema(mock_post) - self.gbt.set_schema(self.test_columns) - test_rows = [ - {"test-str": "one", "test-num": 1}, - {"test-str": "two", "test-num": 2} - ] - mock_post.reset_mock() + TEST_ROW_COUNT = self.batch_size + test_rows = list([ + {"test-str": f"one {i}", "test-num": i} + for i in range(TEST_ROW_COUNT) + ]) self.gbt.add_rows(test_rows) with patch.object(requests, "get") as mock_get: @@ -121,21 +77,16 @@ def test_commit_with_pre_existing_table(self): self.gbt.commit() # it should have called put to overwrite a table and NOT called post mock_put.assert_called_once() - self.assertEqual( - mock_put.call_args.kwargs["json"]["rows"]["$stashID"], self.stash_id) # it should have NOT created a new table via post: mock_post.assert_not_called() def test_commit_with_non_existing_table(self): # TODO: in a future version, we want to search for the table and if not found, create it. if found, update it (put). with patch.object(requests, "post") as mock_post: - self.mock_post_for_set_schema(mock_post) - self.gbt.set_schema(self.test_columns) test_rows = [ {"test-str": "one", "test-num": 1}, {"test-str": "two", "test-num": 2} ] - mock_post.reset_mock() self.gbt.add_rows(test_rows) with patch.object(requests, "get") as mock_get: @@ -149,8 +100,6 @@ def test_commit_with_non_existing_table(self): self.gbt.commit() # it should not have tried to overwrite a table with put mock_put.assert_not_called() - self.assertEqual( - mock_post.call_args.kwargs["json"]["rows"]["$stashID"], self.stash_id) # it should have created a new table with post: mock_put.assert_not_called() diff --git a/airbyte-integrations/connectors/destination-glide/unit_tests/destination_test.py b/airbyte-integrations/connectors/destination-glide/unit_tests/destination_test.py index 67021d1f2d66e..9a9439f379e92 100644 --- a/airbyte-integrations/connectors/destination-glide/unit_tests/destination_test.py +++ b/airbyte-integrations/connectors/destination-glide/unit_tests/destination_test.py @@ -117,18 +117,16 @@ def test_write_simple(self, mock_factory: Callable): # invoke the generator to get the results: result = list(generator) - self.assertEqual(1, len(result)) + self.assertEqual(0, len(result)) - # ensure it called set_schema, multiple add_rows, followed by commit: + # ensure it called init, multiple add_rows, followed by commit: self.assertEqual(4, len(mock_bigtable.mock_calls)) # NOTE: the call objects in Mock.mock_calls, are three-tuples of (name, positional args, keyword args). CALL_METHOD_NAME_INDEX = 0 self.assertEqual( "init", mock_bigtable.mock_calls[0][CALL_METHOD_NAME_INDEX]) self.assertEqual( - "set_schema", mock_bigtable.mock_calls[1][CALL_METHOD_NAME_INDEX]) - self.assertEqual( - "add_rows", mock_bigtable.mock_calls[2][CALL_METHOD_NAME_INDEX]) + "add_row", mock_bigtable.mock_calls[2][CALL_METHOD_NAME_INDEX]) self.assertEqual( "commit", mock_bigtable.mock_calls[3][CALL_METHOD_NAME_INDEX]) @@ -163,18 +161,16 @@ def test_write_with_checkpoints(self, mock_factory: Callable): # invoke the generator to get the results: result = list(generator) - # we had two state records so we expect them to be yielded: - self.assertEqual(2, len(result)) + # we had two state records but we don't want to yield them + self.assertEqual(0, len(result)) # ensure it called add_rows multiple times: - self.assertGreaterEqual(mock_bigtable.add_rows.call_count, 2) + self.assertGreaterEqual(mock_bigtable.add_row.call_count, 2) # NOTE: the call objects in Mock.mock_calls, are three-tuples of (name, positional args, keyword args). CALL_METHOD_NAME_INDEX = 0 self.assertEqual( "init", mock_bigtable.mock_calls[0][CALL_METHOD_NAME_INDEX]) - self.assertEqual( - "set_schema", mock_bigtable.mock_calls[1][CALL_METHOD_NAME_INDEX]) - mock_bigtable.add_rows.assert_called() + mock_bigtable.add_row.assert_called() self.assertEqual( "commit", mock_bigtable.mock_calls[mock_bigtable.call_count - 1][CALL_METHOD_NAME_INDEX]) @@ -250,14 +246,13 @@ def test_with_invalid_column_types(self, mock_factory: Callable): ] ) - # expecting only to return the state message: + # expecting not to return the state message: result = list(generator) - assert len(result) == 1 + assert len(result) == 0 - mock_bigtable.set_schema.assert_called_once() - mock_bigtable.add_rows.assert_called_once() + mock_bigtable.add_row.assert_called() # get the columns we passed into teh API and verify the type defaulted to string: - schema_calls = mock_bigtable.set_schema.call_args[0][0] + schema_calls = mock_bigtable.init.call_args[0][2] null_column = [col for col in schema_calls if col.id() == "obj_null_col"][0] self.assertEqual(null_column.type(), "string") From 2e506c778d05cdf6da26dee19b5ee84e6ed90df2 Mon Sep 17 00:00:00 2001 From: Alex Corrado Date: Tue, 6 Aug 2024 13:39:39 +0100 Subject: [PATCH 31/39] push-docker-image: ignore "latest" tag --- .../connectors/destination-glide/scripts/push-docker-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/destination-glide/scripts/push-docker-image.sh b/airbyte-integrations/connectors/destination-glide/scripts/push-docker-image.sh index 5ed039611e1e8..7fa257eaf1648 100755 --- a/airbyte-integrations/connectors/destination-glide/scripts/push-docker-image.sh +++ b/airbyte-integrations/connectors/destination-glide/scripts/push-docker-image.sh @@ -16,7 +16,7 @@ docker inspect --format='Pushing the local image "{{index .RepoTags 0}}" created tags=$(gcloud artifacts docker tags list $GLIDE_DOCKER_IMAGE_NAME --format="get(tag)") # Sort the tags and get the highest one -highest_tag=$(echo "$tags" | awk -F'/' '{print $NF}' | sort -V | tail -n 1) +highest_tag=$(echo "$tags" | awk -F'/' '{print $NF}' | sed 's/latest//g' | sort -V | tail -n 1) echo "found highest tag on remote: $highest_tag" From 40a730c759d4751e15cad4a29784db6381735e02 Mon Sep 17 00:00:00 2001 From: Scott Willeke Date: Wed, 7 Aug 2024 07:22:09 -0700 Subject: [PATCH 32/39] fix: this builds the AMD64 architecture image explicitly and pushes to Airbyte's repo for suing it in Airbyte Cloud (#13) --- .../scripts/build-docker-image.sh | 5 ++- .../scripts/push-docker-image.sh | 43 ++++++++++++++----- 2 files changed, 36 insertions(+), 12 deletions(-) diff --git a/airbyte-integrations/connectors/destination-glide/scripts/build-docker-image.sh b/airbyte-integrations/connectors/destination-glide/scripts/build-docker-image.sh index fe64db0fb3339..d92539bd6b346 100755 --- a/airbyte-integrations/connectors/destination-glide/scripts/build-docker-image.sh +++ b/airbyte-integrations/connectors/destination-glide/scripts/build-docker-image.sh @@ -3,5 +3,6 @@ this_dir=$(cd $(dirname "$0"); pwd) # this script's directory this_script=$(basename $0) # segment sometimes fails, so AIRBYTE_CI_DISABLE_TELEMETRY=true to disable it per airbyte-ci/connectors/pipelines/README.md -AIRBYTE_CI_DISABLE_TELEMETRY=true airbyte-ci connectors --name=destination-glide build - +# see --architecture flag and available values in airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/commands.py +AIRBYTE_CI_DISABLE_TELEMETRY=true airbyte-ci connectors --name=destination-glide build \ + --architecture linux/amd64 --architecture linux/arm64 diff --git a/airbyte-integrations/connectors/destination-glide/scripts/push-docker-image.sh b/airbyte-integrations/connectors/destination-glide/scripts/push-docker-image.sh index 7fa257eaf1648..1683ebbe21e13 100755 --- a/airbyte-integrations/connectors/destination-glide/scripts/push-docker-image.sh +++ b/airbyte-integrations/connectors/destination-glide/scripts/push-docker-image.sh @@ -2,18 +2,33 @@ this_dir=$(cd $(dirname "$0"); pwd) # this script's directory this_script=$(basename $0) -#!/usr/bin/env bash -LOCAL_BUILT_IMAGE_NAME=airbyte/destination-glide:dev -GLIDE_DOCKER_IMAGE_NAME=us-central1-docker.pkg.dev/glide-connectors/airbyte-glide-destination/destination-glide +# So... it looks like when the airbyte-ci builds the multi-architecture images, it puts each one on a different tag locally. +# We should probably push amd64 only or at least make it the "default" and make arm tagged sepraately. +# if we built and pushed all at once we could have a single tag for both architectures. However, we're using the airbyte-ci to build and that makes pushing a separate step. +# It looks like there is experimental support for using docker manfiest to combine tags into a single multi-arch image at https://docs.docker.com/reference/cli/docker/manifest/ +LOCAL_BUILT_IMAGE_TAG_AMD64=dev-linux-amd64 +LOCAL_BUILT_IMAGE_TAG_ARM64=dev-linux-arm64 + +LOCAL_BUILT_IMAGE_NAME=airbyte/destination-glide +# GLIDE_DOCKER_IMAGE_NAME is really only useful if we want to use it entirely in Airbyte OSS. To use in Airbyte cloud we *must* push to the airbyte-managed repo. +#GLIDE_DOCKER_IMAGE_NAME=us-central1-docker.pkg.dev/glide-connectors/airbyte-glide-destination/destination-glide + +# AIRBYTE_DOCKER_IMAGE_NAME is the repo we use for Airbyte's Cloud instance. +GLIDE_AT_AIRBYTE_COMPANY_NAME=heyglide +AIRBYTE_DOCKER_IMAGE_NAME=us-central1-docker.pkg.dev/airbyte-custom-connectors/$GLIDE_AT_AIRBYTE_COMPANY_NAME/destination-glide -echo "Pushing to remote image repository: $GLIDE_DOCKER_IMAGE_NAME" -docker inspect --format='Pushing the local image "{{index .RepoTags 0}}" created at "{{.Created}}"' $LOCAL_BUILT_IMAGE_NAME +echo "Will push to remote image repository: $AIRBYTE_DOCKER_IMAGE_NAME" +docker inspect --format='Pushing the local image "{{index .RepoTags 0}}" created at "{{.Created}}"' $LOCAL_BUILT_IMAGE_NAME:$LOCAL_BUILT_IMAGE_TAG_AMD64 +if [ $? -ne 0 ]; then + echo "Error: The local image $LOCAL_BUILT_IMAGE_NAME does not exist. Please build the image first." + exit 1 +fi # Fetch the list of tags from Docker Hub #tags=$(wget -q -O - "https://hub.docker.com/v2/namespaces/activescott/repositories/destination-glide/tags?page_size=10" | grep -o '"name": *"[^"]*' | grep -o '[^"]*$' | grep -E '([0-9]+\.)+[0-9]+' ) # Get the list of tags using gcloud with the appropriate format -tags=$(gcloud artifacts docker tags list $GLIDE_DOCKER_IMAGE_NAME --format="get(tag)") +tags=$(gcloud artifacts docker tags list $AIRBYTE_DOCKER_IMAGE_NAME --format="get(tag)") # Sort the tags and get the highest one highest_tag=$(echo "$tags" | awk -F'/' '{print $NF}' | sed 's/latest//g' | sort -V | tail -n 1) @@ -38,18 +53,26 @@ fi # Tag the local image with the new version # NOTE: airbyte/destination-glide is the local docker image that airbyte's CI builds and names locally. -docker image tag $LOCAL_BUILT_IMAGE_NAME $GLIDE_DOCKER_IMAGE_NAME:$new_version +docker image tag $LOCAL_BUILT_IMAGE_NAME:$LOCAL_BUILT_IMAGE_TAG_AMD64 $AIRBYTE_DOCKER_IMAGE_NAME:$new_version +if [ $? -ne 0 ]; then + echo "Error: The local image $LOCAL_BUILT_IMAGE_NAME does not exist. Please build the image first." + exit 1 +fi # Push the image with the new tag -docker push $GLIDE_DOCKER_IMAGE_NAME:$new_version +docker push $AIRBYTE_DOCKER_IMAGE_NAME:$new_version +if [ $? -ne 0 ]; then + echo "Error: The local image $LOCAL_BUILT_IMAGE_NAME does not exist. Please build the image first." + exit 1 +fi echo "Do you want to make this new version the 'latest' tag in Docker too (so that new pulls get this version by default)? (y/n)" read answer if [ "$answer" != "${answer#[Yy]}" ] ; then echo "Taging latest..." - docker image tag $LOCAL_BUILT_IMAGE_NAME $GLIDE_DOCKER_IMAGE_NAME:latest - docker push $GLIDE_DOCKER_IMAGE_NAME:latest + docker image tag $AIRBYTE_DOCKER_IMAGE_NAME:$new_version $AIRBYTE_DOCKER_IMAGE_NAME:latest + docker push $AIRBYTE_DOCKER_IMAGE_NAME:latest else echo "Exiting..." exit 0 From a965978898fcff28f844bddc474a54abebbe5d65 Mon Sep 17 00:00:00 2001 From: Scott Willeke Date: Wed, 25 Sep 2024 17:21:03 -0700 Subject: [PATCH 33/39] fix: destination would fail when a source reported columns without any type (#14) fixes glideapps/glide#29913 --- .../destination_glide/destination.py | 2 +- .../unit_tests/destination_test.py | 62 +++++++++++++++++++ 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py b/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py index b0fd4896699c3..f9b6e44197c2d 100644 --- a/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py +++ b/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py @@ -95,7 +95,7 @@ def create_table_client_for_stream(stream_name, columns): columns = [] properties = configured_stream.stream.json_schema["properties"] for (prop_name, prop) in properties.items(): - prop_type = prop["type"] + prop_type = prop["type"] if "type" in prop else "" logger.debug(f"Found column/property '{prop_name}' with type '{prop_type}' in stream {configured_stream.stream.name}.") # nopep8 columns.append( Column(prop_name, airbyteTypeToGlideType(prop_type)) diff --git a/airbyte-integrations/connectors/destination-glide/unit_tests/destination_test.py b/airbyte-integrations/connectors/destination-glide/unit_tests/destination_test.py index 9a9439f379e92..a45910ee9f550 100644 --- a/airbyte-integrations/connectors/destination-glide/unit_tests/destination_test.py +++ b/airbyte-integrations/connectors/destination-glide/unit_tests/destination_test.py @@ -130,6 +130,68 @@ def test_write_simple(self, mock_factory: Callable): self.assertEqual( "commit", mock_bigtable.mock_calls[3][CALL_METHOD_NAME_INDEX]) + + @patch.object(GlideBigTableFactory, "create") + def test_write_source_schema_without_type(self, mock_factory: Callable): + """ + This tests a case where the source schema has a property without a type. + This happened with Github's "issues" stream for a property. + """ + mock_bigtable = CreateMockGlideBigTable() + mock_factory.return_value = mock_bigtable + + destination = DestinationGlide() + + # create a schema with a property that has no type: + my_schema = { + "type": "object", + "properties": { + "key_str": {"type": "string"}, + "no_type_col": {} + }, + } + + generator = destination.write( + config=create_default_config(), + configured_catalog=create_configured_catalog_default(self.test_table_name, + table_schema=my_schema), + input_messages=[ + AirbyteMessage( + type=Type.RECORD, + record=AirbyteRecordMessage( + stream=self.test_table_name, data={"key_str": f"row {0}", "no_type_col": f"row {0}"}, emitted_at=int(datetime.now().timestamp()) * 1000 + ), + ), + ] + ) + + # invoke the generator to get the results: + result = list(generator) + self.assertEqual(0, len(result)) + + # ensure it called init, multiple add_rows, followed by commit: + self.assertEqual(3, len(mock_bigtable.mock_calls)) + # NOTE: the call objects in Mock.mock_calls, are three-tuples of (name, positional args, keyword args). + CALL_METHOD_NAME_INDEX = 0 + EXPECTED_CALLS = ["init", "add_row", "commit"] + # now loop through each expected call and make sure it was found in the list of actual calls made by the destination: + for expected_call in EXPECTED_CALLS: + found = False + for actual_call in mock_bigtable.mock_calls: + if actual_call[CALL_METHOD_NAME_INDEX] == expected_call: + found = True + break + self.assertTrue(found, f"Expected call {expected_call} not found in actual calls") + + # get the columns we passed into the big table during init and verify the type defaulted to string: + ARGS_INDEX = 1 + init_call_args = mock_bigtable.mock_calls[0][ARGS_INDEX] + ARGS_INDEX_COLUMNS = 2 + columns = init_call_args[ARGS_INDEX_COLUMNS] + col_no_type = [col for col in columns if col.id() == "no_type_col"] + self.assertEqual(col_no_type[0].type(), "string") + + @patch.object(GlideBigTableFactory, "create") def test_write_with_checkpoints(self, mock_factory: Callable): """ From 48251d9221ffc42e7219b08e5975d5b412eb3575 Mon Sep 17 00:00:00 2001 From: Mark Probst Date: Mon, 17 Feb 2025 12:45:23 -0500 Subject: [PATCH 34/39] Test for current 413 behavior --- .../destination_glide/glide.py | 1 + .../GlideBigTableRestStrategy_test.py | 17 +++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py b/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py index a23bd7b2f9304..21c97857c0510 100644 --- a/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py +++ b/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py @@ -137,6 +137,7 @@ def _flush_buffer(self): try: r.raise_for_status() except Exception as e: + # FIXME: if this is a 413, make the batch size smaller and retry raise Exception(f"Failed to post rows batch to {path} : {r.text}") from e # nopep8 logger.info(f"Successfully posted {len(rows)} rows to {path}") # nopep8 diff --git a/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableRestStrategy_test.py b/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableRestStrategy_test.py index 712e1e2f5a6f6..c7207be4131ec 100644 --- a/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableRestStrategy_test.py +++ b/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableRestStrategy_test.py @@ -4,6 +4,7 @@ from unittest import skip from unittest.mock import patch import uuid +from requests.exceptions import HTTPError class TestGlideBigTableRestStrategy(unittest.TestCase): @@ -52,6 +53,22 @@ def test_add_rows_batching(self, mock_post): self.assertEqual(1, mock_post.call_count) self.assertEqual(mock_post.call_args.kwargs["json"], test_rows) + @patch.object(requests, "post") + def test_add_rows_413(self, mock_post): + self.gbt.batch_size = 1 + mock_post.return_value.status_code = 413 + mock_post.return_value.text = "Payload Too Large" + mock_post.return_value.raise_for_status.side_effect = HTTPError("413 Client Error: Payload Too Large") + + with self.assertRaises(Exception) as context: + self.gbt.add_rows([ + {"test-str": "one", "test-num": 1}, + {"test-str": "two", "test-num": 2}, + {"test-str": "three", "test-num": 3}, + {"test-str": "four", "test-num": 4}]) + + self.assertIn("Failed to post rows batch", str(context.exception)) + def test_commit_with_pre_existing_table(self): with patch.object(requests, "post") as mock_post: TEST_ROW_COUNT = self.batch_size From a355d8712cb77247d2957d62aa1d29b2bd9be4d2 Mon Sep 17 00:00:00 2001 From: Mark Probst Date: Mon, 17 Feb 2025 13:34:37 -0500 Subject: [PATCH 35/39] Split when 413 happens --- .../destination_glide/glide.py | 46 ++++++++++++------- .../GlideBigTableRestStrategy_test.py | 35 +++++++++++++- 2 files changed, 64 insertions(+), 17 deletions(-) diff --git a/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py b/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py index 21c97857c0510..337f0b0fe53ea 100644 --- a/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py +++ b/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py @@ -123,25 +123,39 @@ def __init__(self): def _flush_buffer(self): rows = self.buffer - if len(rows) == 0: + if not rows: return self.buffer = [] - path = f"stashes/{self.stash_id}/{self.stash_serial}" - logger.debug(f"Flushing {len(rows)} rows to {path} ...") - r = requests.post( - self.url(path), - headers=self.headers(), - json=rows - ) - try: - r.raise_for_status() - except Exception as e: - # FIXME: if this is a 413, make the batch size smaller and retry - raise Exception(f"Failed to post rows batch to {path} : {r.text}") from e # nopep8 - - logger.info(f"Successfully posted {len(rows)} rows to {path}") # nopep8 - self.stash_serial += 1 + start_idx = 0 + chunk_size = len(rows) + stash_serial = self.stash_serial + while start_idx < len(rows): + chunk = rows[start_idx : start_idx + chunk_size] + path = f"stashes/{self.stash_id}/{stash_serial}" + logger.debug(f"Flushing {len(chunk)} rows to {path} ...") + + r = requests.post( + self.url(path), + headers=self.headers(), + json=chunk + ) + try: + r.raise_for_status() + except requests.HTTPError as e: + if r.status_code == 413 and chunk_size > 1: + chunk_size = max(1, chunk_size // 2) + logger.info(f"413 Payload Too Large. Reducing chunk size to {chunk_size} and retrying.") + continue + raise Exception(f"Failed to post rows batch to {path} : {r.text}") from e + + logger.info(f"Successfully posted {len(chunk)} rows to {path}") + stash_serial += 1 + start_idx += chunk_size + + # We only set the stash serial if the flush of all rows was + # successful, otherwise we could end up with duplicate rows. + self.stash_serial = stash_serial def add_row(self, row: BigTableRow) -> None: self.buffer.append(row) diff --git a/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableRestStrategy_test.py b/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableRestStrategy_test.py index c7207be4131ec..61febbd7f9de7 100644 --- a/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableRestStrategy_test.py +++ b/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableRestStrategy_test.py @@ -5,7 +5,7 @@ from unittest.mock import patch import uuid from requests.exceptions import HTTPError - +import json class TestGlideBigTableRestStrategy(unittest.TestCase): api_host = "https://test-api-host.com" @@ -69,6 +69,39 @@ def test_add_rows_413(self, mock_post): self.assertIn("Failed to post rows batch", str(context.exception)) + @patch.object(requests, "post") + def test_split_batches_on_413(self, mock_post): + threshold_bytes = 200 + did_fail = False + + def side_effect(*args, **kwargs): + payload = kwargs.get("json", []) + payload_size = len(json.dumps(payload).encode("utf-8")) + mock_response = requests.Response() + if payload_size > threshold_bytes: + mock_response.status_code = 413 + mock_response._content = b"Payload Too Large" + nonlocal did_fail + did_fail = True + else: + mock_response.status_code = 200 + mock_response._content = b"OK" + return mock_response + + mock_post.side_effect = side_effect + + # Force small flushes to test smaller batches + self.gbt.batch_size = 2 + + # Attempt to add rows that will exceed threshold if posted all at once + large_number_of_rows = [ + {"test-str": "foo " * 30, "test-num": i} # repeated strings for size + for i in range(10) + ] + self.gbt.add_rows(large_number_of_rows) + + self.assertTrue(did_fail) + def test_commit_with_pre_existing_table(self): with patch.object(requests, "post") as mock_post: TEST_ROW_COUNT = self.batch_size From 7914fe89fd754836c1fbf4d70c2597fc339647b8 Mon Sep 17 00:00:00 2001 From: Mark Probst Date: Mon, 17 Feb 2025 13:59:39 -0500 Subject: [PATCH 36/39] Test that the correct rows are posted --- .../unit_tests/GlideBigTableRestStrategy_test.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableRestStrategy_test.py b/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableRestStrategy_test.py index 61febbd7f9de7..5e8beaa8c151d 100644 --- a/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableRestStrategy_test.py +++ b/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableRestStrategy_test.py @@ -73,6 +73,7 @@ def test_add_rows_413(self, mock_post): def test_split_batches_on_413(self, mock_post): threshold_bytes = 200 did_fail = False + sent_rows = [] def side_effect(*args, **kwargs): payload = kwargs.get("json", []) @@ -86,6 +87,8 @@ def side_effect(*args, **kwargs): else: mock_response.status_code = 200 mock_response._content = b"OK" + nonlocal sent_rows + sent_rows.extend(payload) return mock_response mock_post.side_effect = side_effect @@ -101,6 +104,7 @@ def side_effect(*args, **kwargs): self.gbt.add_rows(large_number_of_rows) self.assertTrue(did_fail) + self.assertEqual(large_number_of_rows, sent_rows) def test_commit_with_pre_existing_table(self): with patch.object(requests, "post") as mock_post: From d6c76723a5f9eb05baef958ce3bce24c25d041b2 Mon Sep 17 00:00:00 2001 From: Mark Probst Date: Mon, 17 Feb 2025 14:03:04 -0500 Subject: [PATCH 37/39] Test with different payload size thresholds --- .../GlideBigTableRestStrategy_test.py | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableRestStrategy_test.py b/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableRestStrategy_test.py index 5e8beaa8c151d..226c17efd0095 100644 --- a/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableRestStrategy_test.py +++ b/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableRestStrategy_test.py @@ -106,6 +106,42 @@ def side_effect(*args, **kwargs): self.assertTrue(did_fail) self.assertEqual(large_number_of_rows, sent_rows) + @patch.object(requests, "post") + def test_split_batches_on_413_varied_thresholds(self, mock_post): + thresholds = [200, 512, 1024, 2048, 8192] + for threshold_bytes in thresholds: + with self.subTest(threshold_bytes=threshold_bytes): + did_fail = False + sent_rows = [] + + def side_effect(*args, **kwargs): + payload = kwargs.get("json", []) + payload_size = len(json.dumps(payload).encode("utf-8")) + mock_response = requests.Response() + if payload_size > threshold_bytes: + mock_response.status_code = 413 + mock_response._content = b"Payload Too Large" + nonlocal did_fail + did_fail = True + else: + mock_response.status_code = 200 + mock_response._content = b"OK" + sent_rows.extend(payload) + return mock_response + + mock_post.reset_mock() + mock_post.side_effect = side_effect + + self.gbt.batch_size = 2 + large_number_of_rows = [ + {"test-str": "foo " * 30, "test-num": i} + for i in range(10) + ] + self.gbt.add_rows(large_number_of_rows) + + self.assertTrue(did_fail or len(large_number_of_rows) == len(sent_rows)) + self.assertEqual(large_number_of_rows, sent_rows) + def test_commit_with_pre_existing_table(self): with patch.object(requests, "post") as mock_post: TEST_ROW_COUNT = self.batch_size From 77e2c17b9bbb47e2e1a8ada37c0bcedf71075d5c Mon Sep 17 00:00:00 2001 From: Mark Probst Date: Wed, 19 Feb 2025 11:04:34 -0500 Subject: [PATCH 38/39] Use PUT for the stashes, not POST --- .../connectors/destination-glide/destination_glide/glide.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py b/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py index 337f0b0fe53ea..8a02353a7922d 100644 --- a/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py +++ b/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py @@ -135,7 +135,7 @@ def _flush_buffer(self): path = f"stashes/{self.stash_id}/{stash_serial}" logger.debug(f"Flushing {len(chunk)} rows to {path} ...") - r = requests.post( + r = requests.put( self.url(path), headers=self.headers(), json=chunk @@ -147,9 +147,9 @@ def _flush_buffer(self): chunk_size = max(1, chunk_size // 2) logger.info(f"413 Payload Too Large. Reducing chunk size to {chunk_size} and retrying.") continue - raise Exception(f"Failed to post rows batch to {path} : {r.text}") from e + raise Exception(f"Failed to put rows batch to {path} : {r.text}") from e - logger.info(f"Successfully posted {len(chunk)} rows to {path}") + logger.info(f"Successfully put {len(chunk)} rows to {path}") stash_serial += 1 start_idx += chunk_size From 7ee5e6936634a0a8953f88e9bc04586dcaca397d Mon Sep 17 00:00:00 2001 From: Mark Probst Date: Wed, 19 Feb 2025 11:04:53 -0500 Subject: [PATCH 39/39] Push to our docker registry --- .../destination-glide/scripts/push-docker-image.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/destination-glide/scripts/push-docker-image.sh b/airbyte-integrations/connectors/destination-glide/scripts/push-docker-image.sh index 1683ebbe21e13..103116e888f6b 100755 --- a/airbyte-integrations/connectors/destination-glide/scripts/push-docker-image.sh +++ b/airbyte-integrations/connectors/destination-glide/scripts/push-docker-image.sh @@ -11,11 +11,12 @@ LOCAL_BUILT_IMAGE_TAG_ARM64=dev-linux-arm64 LOCAL_BUILT_IMAGE_NAME=airbyte/destination-glide # GLIDE_DOCKER_IMAGE_NAME is really only useful if we want to use it entirely in Airbyte OSS. To use in Airbyte cloud we *must* push to the airbyte-managed repo. -#GLIDE_DOCKER_IMAGE_NAME=us-central1-docker.pkg.dev/glide-connectors/airbyte-glide-destination/destination-glide +GLIDE_DOCKER_IMAGE_NAME=us-central1-docker.pkg.dev/glide-connectors/airbyte-glide-destination/destination-glide # AIRBYTE_DOCKER_IMAGE_NAME is the repo we use for Airbyte's Cloud instance. GLIDE_AT_AIRBYTE_COMPANY_NAME=heyglide -AIRBYTE_DOCKER_IMAGE_NAME=us-central1-docker.pkg.dev/airbyte-custom-connectors/$GLIDE_AT_AIRBYTE_COMPANY_NAME/destination-glide +#AIRBYTE_DOCKER_IMAGE_NAME=us-central1-docker.pkg.dev/airbyte-custom-connectors/$GLIDE_AT_AIRBYTE_COMPANY_NAME/destination-glide +AIRBYTE_DOCKER_IMAGE_NAME="$GLIDE_DOCKER_IMAGE_NAME" echo "Will push to remote image repository: $AIRBYTE_DOCKER_IMAGE_NAME"