Initial E2E Implementation of DeltaCAT Native Catalog APIs

.github/workflows/ci.yml

@@ -28,7 +28,7 @@ jobs:
         strategy:
             matrix:
                 python-version: ["3.9", "3.10"]
-        timeout-minutes: 30
+        timeout-minutes: 45
         steps:
             - name: "checkout repository"
               uses: actions/checkout@v4

.gitignore

@@ -30,6 +30,7 @@ package-lock.json
 *.db
 pyvenv.cfg
 **/.deltacat
+.deltacat_memory/
 
 # Generated Files
 **/.riv-meta-*

Makefile

@@ -67,5 +67,14 @@ benchmark-aws: install
 benchmark: install
 	pytest -m benchmark deltacat/benchmarking
 
+type-mappings: install
+	@echo "Generating type mappings..."
+	venv/bin/python deltacat/docs/autogen/schema/inference/generate_type_mappings.py
+	@echo "Parsing type mappings to markdown..."
+	venv/bin/python deltacat/docs/autogen/schema/inference/parse_json_type_mappings.py generate_type_mappings_results.json
+	@echo "Generating Python compatibility mapping..."
+	venv/bin/python deltacat/docs/autogen/schema/inference/parse_json_type_mappings.py generate_type_mappings_results.json --python
+	@echo "Type mappings generation complete!"
+
 publish: test test-integration rebuild
 	twine upload dist/*

README-development.md

@@ -109,6 +109,15 @@ make benchmark-aws
 ```
 Run AWS benchmarks.
 
+#### type-mappings
+```shell
+make type-mappings
+```
+Regenerates type mapping documentation and corresponding Python module. Specifically this:
+1. Regenerates the markdown documentation at `docs/schema/README.md`
+2. Regenerates the writer/reader compatibility mapping file at `utils/reader_compatibility_mapping.py`
+This should be run after any changes to PyArrow, Polars, Pandas, Ray, or Daft dependency versions.
+
 ## Cloud Integration Testing
 ### AWS
 You can deploy and test your local DeltaCAT changes on any AWS environment that can run Ray applications (e.g. EC2, Glue