Merge pull request #1 from planet-a-ventures/joscha/database-support

feat: database support

Author: joscha

.gitignore

@@ -178,4 +178,6 @@ secrets.toml
 
 .DS_Store
 
-.python-version
+.python-version
+
+/*.json

.yamlfmt

@@ -2,5 +2,4 @@ gitignore_excludes: true
 eof_newline: true
 exclude:
   - .yamllint
-  - .pre-commit-config.yaml
-  - dlt_source_notion/model/spec
+  - .pre-commit-config.yaml

README.md

@@ -15,6 +15,13 @@ Currently loads the following data:
 | -- | -- |
 | `persons` | Items of the `user` model of type `person` |
 | `bots` | Items of the `user` model of type `bot` |
+| `database_<name>_<shorthash>` | The database content (pages) of a given database. Columns are all global page columns + selected columns of properties |
+| `databases` | All metadata of each loaded database. Has the title and a reference to the database table as well. |
+| `databases__properties` | Column \<-> Label mapping for each unfiltered database property |
+| `options_<name>_<shorthash>` | All options of a `select` or `multi_select` in one of the loaded databases. |
+
+`<shorthash>` refers to the hashed `id` of an entity.
+The hash is a 4-bit BLAKE2b hash.
 
 ## Why are you not using the `dlt-hub/verified-sources` notion source / Differences
 
@@ -26,6 +33,8 @@ has a few drawbacks:
   Once you make changes to it, it effectively becomes a fork,
   making it hard to update after the fact.
 - This makes use of a preexisting client implementation
+- And Pydantic types for Notion entities,
+  which makes this implementation a lot more stable
 
 ## Usage
 

devenv.lock

@@ -3,10 +3,10 @@
     "devenv": {
       "locked": {
         "dir": "src/modules",
-        "lastModified": 1741068816,
+        "lastModified": 1742320965,
         "owner": "cachix",
         "repo": "devenv",
-        "rev": "9f6da63c162ad86b6fb84edcbd8c447fdc411c3d",
+        "rev": "6bde92766ddd3ee1630029a03d36baddd51934e2",
         "type": "github"
       },
       "original": {
@@ -55,10 +55,10 @@
         ]
       },
       "locked": {
-        "lastModified": 1740915799,
+        "lastModified": 1742300892,
         "owner": "cachix",
         "repo": "git-hooks.nix",
-        "rev": "42b1ba089d2034d910566bf6b40830af6b8ec732",
+        "rev": "ea26a82dda75bee6783baca6894040c8e6599728",
         "type": "github"
       },
       "original": {
@@ -122,12 +122,28 @@
         "type": "github"
       }
     },
+    "nixpkgs-unstable": {
+      "locked": {
+        "lastModified": 1742272065,
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "rev": "3549532663732bfd89993204d40543e9edaec4f2",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nixos",
+        "ref": "nixpkgs-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
     "root": {
       "inputs": {
         "devenv": "devenv",
         "git-hooks": "git-hooks",
         "nixpkgs": "nixpkgs",
         "nixpkgs-python": "nixpkgs-python",
+        "nixpkgs-unstable": "nixpkgs-unstable",
         "pre-commit-hooks": [
           "git-hooks"
         ]

devenv.nix

@@ -6,15 +6,19 @@
   ...
 }:
 
+let
+  pkgs-unstable = import inputs.nixpkgs-unstable { system = pkgs.stdenv.system; };
+in
 {
   packages = [
     pkgs.git
     pkgs.bash
+    pkgs.python312Packages.setuptools
   ];
 
   languages.python.enable = true;
   languages.python.uv.enable = true;
-  languages.python.uv.package = pkgs.uv;
+  languages.python.uv.package = pkgs-unstable.python312Packages.uv;
   languages.python.uv.sync.enable = true;
   languages.python.uv.sync.allExtras = true;
   languages.python.venv.enable = true;

devenv.yaml

@@ -1,6 +1,8 @@
 inputs:
   nixpkgs:
     url: github:cachix/devenv-nixpkgs/rolling
+  nixpkgs-unstable:
+    url: github:nixos/nixpkgs/nixpkgs-unstable
   nixpkgs-python:
     url: github:cachix/nixpkgs-python
     inputs: