Add llms.txt and llms-full.txt generation (#188)

Author: AdrianAtZyte

.github/workflows/main.yml

@@ -13,12 +13,6 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - python-version: "3.9"
-            env:
-              TOXENV: "msgpack"
-          - python-version: "3.9"
-            env:
-              TOXENV: "json"
           - python-version: "3.10"
             env:
               TOXENV: "msgpack"
@@ -43,6 +37,11 @@ jobs:
           - python-version: "3.13"
             env:
               TOXENV: "json"
+          # Keep in sync with pyproject.toml's
+          # tool.sphinx-scrapy.python-version.
+          - python-version: "3.14"
+            env:
+              TOXENV: "docs"
 
           # https://github.com/kevin1024/vcrpy/issues/885
           # - python-version: "pypy3.11-v7.3.20"

.pre-commit-config.yaml

@@ -0,0 +1,5 @@
+repos:
+- repo: https://github.com/scrapy/sphinx-scrapy
+  rev: 0.7.1
+  hooks:
+    - id: sphinx-scrapy

.readthedocs.yml

@@ -1,15 +1,10 @@
 version: 2
-formats: all
-sphinx:
-  configuration: docs/conf.py
-  fail_on_warning: true
 build:
-  os: ubuntu-22.04
+  os: ubuntu-24.04
   tools:
-    # For available versions, see:
-    # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-tools-python
-    python: "3.10"  # Keep in sync with .github/workflows/main.yml
-python:
-  install:
-    - requirements: docs/requirements.txt
-    - path: .
+    python: "3.14"
+  commands:
+    - pip install tox
+    - tox -e docs
+    - mkdir -p $READTHEDOCS_OUTPUT/html
+    - cp -a docs/_build/all/. $READTHEDOCS_OUTPUT/html/

README.rst

@@ -14,7 +14,7 @@ Client interface for Scrapinghub API
 .. image:: https://codecov.io/gh/scrapinghub/python-scrapinghub/branch/master/graph/badge.svg
    :target: https://app.codecov.io/gh/scrapinghub/python-scrapinghub
 
-The ``scrapinghub`` is a Python 3.9+ library for communicating with the
+The ``scrapinghub`` is a Python 3.10+ library for communicating with the
 `Scrapinghub API`_.
 
 

docs/Makefile

@@ -39,7 +39,7 @@
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
-extensions = ['sphinx.ext.autodoc']
+extensions = ['sphinx_scrapy']
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
@@ -160,9 +160,3 @@
      author, 'python-scrapinghub', 'One line description of project.',
      'Miscellaneous'),
 ]
-
-# Set Sphinx Read The Docs theme
-import sphinx_rtd_theme
-
-html_theme = 'sphinx_rtd_theme'
-html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]

docs/conf.py

@@ -4,7 +4,7 @@ Quickstart
 Requirements
 ------------
 
-* Python 3.8+
+* Python 3.10+
 
 
 Installation

docs/make.bat

@@ -0,0 +1,2 @@
+sphinx-rtd-theme
+sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.7.1

docs/quickstart.rst

@@ -1,2 +1,71 @@
-sphinx==7.2.6
-sphinx-rtd-theme==2.0.0
+# This file was autogenerated by uv via the following command:
+#    uv pip compile requirements.in -o requirements.txt
+alabaster==1.0.0
+    # via sphinx
+babel==2.18.0
+    # via sphinx
+certifi==2026.2.25
+    # via requests
+charset-normalizer==3.4.6
+    # via requests
+docutils==0.22.4
+    # via
+    #   sphinx
+    #   sphinx-markdown-builder
+    #   sphinx-rtd-theme
+idna==3.11
+    # via requests
+imagesize==2.0.0
+    # via sphinx
+jinja2==3.1.6
+    # via sphinx
+markupsafe==3.0.3
+    # via jinja2
+packaging==26.0
+    # via
+    #   sphinx
+    #   sphinx-scrapy
+pygments==2.19.2
+    # via sphinx
+requests==2.32.5
+    # via sphinx
+roman-numerals==4.1.0
+    # via sphinx
+snowballstemmer==3.0.1
+    # via sphinx
+sphinx==9.1.0
+    # via
+    #   sphinx-copybutton
+    #   sphinx-llms-txt
+    #   sphinx-markdown-builder
+    #   sphinx-rtd-theme
+    #   sphinx-scrapy
+    #   sphinxcontrib-jquery
+sphinx-copybutton==0.5.2
+    # via sphinx-scrapy
+sphinx-llms-txt @ git+https://github.com/zytedata/sphinx-llms-txt.git@5e8866cb0cc249aa2017ad9050b3b83a7ca16f69
+    # via sphinx-scrapy
+sphinx-markdown-builder @ git+https://github.com/zytedata/sphinx-markdown-builder.git@ac9f8babfe622e4300099ab44b96d9d9228e742e
+    # via sphinx-scrapy
+sphinx-rtd-theme==3.1.0
+    # via -r requirements.in
+sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@2b5f6c7de64c8317cb771fdeb2e5020d1c9c9dcf
+    # via -r requirements.in
+sphinxcontrib-applehelp==2.0.0
+    # via sphinx
+sphinxcontrib-devhelp==2.0.0
+    # via sphinx
+sphinxcontrib-htmlhelp==2.1.0
+    # via sphinx
+sphinxcontrib-jquery==4.1
+    # via sphinx-rtd-theme
+sphinxcontrib-jsmath==1.0.1
+    # via sphinx
+sphinxcontrib-qthelp==2.0.0
+    # via sphinx
+sphinxcontrib-serializinghtml==2.0.0
+    # via sphinx
+tabulate==0.10.0
+    # via sphinx-markdown-builder
+urllib3==2.6.3
+    # via requests