fix: update embedding extraction to use appropriate async method (#2068)

ahgraber · jjmachan · claude · web-flow · commit 169b37f8bb7c · 2025-06-16T21:20:43.000-07:00
Embedding extraction should be async and use langchain's `embed_text`
method rather than `embed_query`

---------

Co-authored-by: jjmachan &lt;jamesjithin97@gmail.com&gt;
Co-authored-by: Claude &lt;noreply@anthropic.com&gt;
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -31,16 +31,16 @@ jobs:
           token: ${{ github.token }}
           filters: |
             related: &related
-              - .github/workflows/ci.yml
+              - .github/workflows/ci.yaml
               - codecov.yml
               - pyproject.toml
               - requirements/test.txt
             ragas:
               - *related
-              - "src/ragas/**"
-              - "tests/**"
+              - "ragas/src/ragas/**"
+              - "ragas/tests/**"
             ragas_experimental:
-              - "src/experimental/**"
+              - "experimental/ragas_experimental/**"
             docs:
               - *related
               - requirements/docs-requirements.txt
@@ -85,7 +85,7 @@ jobs:
 
       - name: Install dependencies
         run: |
-          pip install "."
+          pip install "./ragas"
           pip install -r requirements/test.txt
 
 
@@ -97,7 +97,7 @@ jobs:
             OPTS=(--dist loadfile -n auto)
           fi
           # Now run the unit tests
-          pytest --nbmake tests/unit "${OPTS[@]}"
+          pytest --nbmake ragas/tests/unit "${OPTS[@]}"
         env:
           __RAGAS_DEBUG_TRACKING: true
           RAGAS_DO_NOT_TRACK: true
@@ -140,7 +140,7 @@ jobs:
 
       - name: Install dependencies
         run: |
-          pip install .
+          pip install ./ragas
           pip install -r requirements/dev.txt
 
       - name: Lint check
diff --git a/.gitignore b/.gitignore
@@ -167,7 +167,7 @@ cython_debug/
 # Ragas specific
 experiments/
 **/fil-result/
-src/ragas/_version.py
+ragas/src/ragas/_version.py
 experimental/ragas_experimental/_version.py
 .vscode
 .envrc
diff --git a/CLAUDE.md b/CLAUDE.md
@@ -187,4 +187,8 @@ console_handler.setFormatter(formatter)
 
 # Add the handler to the logger
 analytics_logger.addHandler(console_handler)
-```
+```
+
+## Memories
+
+- whenever you create such docs put in in /experiments because that is gitignored and you can use it as a scratchpad or tmp directory for storing these
diff --git a/Makefile b/Makefile
@@ -37,7 +37,7 @@ lint-all: lint lint-experimental ## Lint all code in the monorepo
 
 type: ## Running type checker for ragas
 	@echo "(pyright) Typechecking ragas codebase..."
-	PYRIGHT_PYTHON_FORCE_VERSION=latest pyright ragas/src/ragas
+	cd ragas && PYRIGHT_PYTHON_FORCE_VERSION=latest pyright src
 
 type-experimental: ## Running type checker for experimental
 	@echo "(pyright) Typechecking experimental codebase..."
diff --git a/docs/experimental/index.html.md b/docs/experimental/index.html.md
diff --git a/docs/experimental/index.md b/docs/experimental/index.md
@@ -0,0 +1,3 @@
+# Ragas Experimental
+
+Under the works but stay tuned :)
diff --git a/docs/howtos/applications/cost.ipynb b/docs/howtos/applications/cost.ipynb
@@ -24,33 +24,9 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "TokenUsage(input_tokens=9, output_tokens=9, model='')"
-      ]
-     },
-     "execution_count": 2,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "from langchain_openai.chat_models import ChatOpenAI\n",
-    "from langchain_core.prompt_values import StringPromptValue\n",
-    "\n",
-    "gpt4o = ChatOpenAI(model=\"gpt-4o\")\n",
-    "p = StringPromptValue(text=\"hai there\")\n",
-    "llm_result = gpt4o.generate_prompt([p])\n",
-    "\n",
-    "# lets import a parser for OpenAI\n",
-    "from ragas.cost import get_token_usage_for_openai\n",
-    "\n",
-    "get_token_usage_for_openai(llm_result)"
-   ]
+   "outputs": [],
+   "source": "from langchain_openai.chat_models import ChatOpenAI\nfrom langchain_core.prompt_values import StringPromptValue\n# lets import a parser for OpenAI\nfrom ragas.cost import get_token_usage_for_openai\n\ngpt4o = ChatOpenAI(model=\"gpt-4o\")\np = StringPromptValue(text=\"hai there\")\nllm_result = gpt4o.generate_prompt([p])\n\nget_token_usage_for_openai(llm_result)"
   },
   {
    "cell_type": "markdown",
@@ -284,4 +260,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 2
-}
+}
diff --git a/docs/howtos/customizations/metrics/cost.ipynb b/docs/howtos/customizations/metrics/cost.ipynb
@@ -35,41 +35,9 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "/opt/homebrew/Caskroom/miniforge/base/envs/ragas/lib/python3.9/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
-      "  from .autonotebook import tqdm as notebook_tqdm\n"
-     ]
-    },
-    {
-     "data": {
-      "text/plain": [
-       "TokenUsage(input_tokens=9, output_tokens=9, model='')"
-      ]
-     },
-     "execution_count": 5,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "from langchain_openai.chat_models import ChatOpenAI\n",
-    "from langchain_core.prompt_values import StringPromptValue\n",
-    "\n",
-    "gpt4o = ChatOpenAI(model=\"gpt-4o\")\n",
-    "p = StringPromptValue(text=\"hai there\")\n",
-    "llm_result = gpt4o.generate_prompt([p])\n",
-    "\n",
-    "# lets import a parser for OpenAI\n",
-    "from ragas.cost import get_token_usage_for_openai\n",
-    "\n",
-    "get_token_usage_for_openai(llm_result)"
-   ]
+   "outputs": [],
+   "source": "from langchain_openai.chat_models import ChatOpenAI\nfrom langchain_core.prompt_values import StringPromptValue\n# lets import a parser for OpenAI\nfrom ragas.cost import get_token_usage_for_openai\n\ngpt4o = ChatOpenAI(model=\"gpt-4o\")\np = StringPromptValue(text=\"hai there\")\nllm_result = gpt4o.generate_prompt([p])\n\nget_token_usage_for_openai(llm_result)"
   },
   {
    "cell_type": "markdown",
@@ -212,4 +180,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 2
-}
+}
diff --git a/docs/howtos/customizations/testgenerator/language_adaptation.ipynb b/docs/howtos/customizations/testgenerator/language_adaptation.ipynb
@@ -53,7 +53,7 @@
     }
    ],
    "source": [
-    "from langchain_community.document_loaders import DirectoryLoader, TextLoader\n",
+    "from langchain_community.document_loaders import DirectoryLoader\n",
     "\n",
     "\n",
     "path = \"Sample_non_english_corpus/\"\n",
diff --git a/docs/howtos/customizations/testgenerator/testgen-custom-single-hop.ipynb b/docs/howtos/customizations/testgenerator/testgen-custom-single-hop.ipynb
@@ -229,7 +229,6 @@
    "source": [
     "from ragas.testset.synthesizers.single_hop import (\n",
     "    SingleHopQuerySynthesizer,\n",
-    "    SingleHopScenario,\n",
     ")\n",
     "from dataclasses import dataclass\n",
     "from ragas.testset.synthesizers.prompts import (\n",
diff --git a/docs/howtos/customizations/testgenerator/testgen-customisation.ipynb b/docs/howtos/customizations/testgenerator/testgen-customisation.ipynb
@@ -40,7 +40,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from langchain_community.document_loaders import DirectoryLoader, TextLoader\n",
+    "from langchain_community.document_loaders import DirectoryLoader\n",
     "\n",
     "path = \"Sample_Docs_Markdown/\"\n",
     "loader = DirectoryLoader(path, glob=\"**/*.md\")\n",
@@ -136,7 +136,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from ragas.testset.transforms import Parallel, apply_transforms\n",
+    "from ragas.testset.transforms import apply_transforms\n",
     "from ragas.testset.transforms import (\n",
     "    HeadlinesExtractor,\n",
     "    HeadlineSplitter,\n",
diff --git a/docs/howtos/integrations/helicone.ipynb b/docs/howtos/integrations/helicone.ipynb
@@ -45,31 +45,9 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
    "metadata": {},
    "outputs": [],
-   "source": [
-    "import os\n",
-    "from datasets import Dataset\n",
-    "from ragas import evaluate\n",
-    "from ragas.metrics import faithfulness, answer_relevancy, context_precision\n",
-    "from ragas.integrations.helicone import helicone_config  # import helicone_config\n",
-    "\n",
-    "\n",
-    "# Set up Helicone\n",
-    "helicone_config.api_key = (\n",
-    "    \"your_helicone_api_key_here\"  # Replace with your actual Helicone API key\n",
-    ")\n",
-    "os.environ[\"OPENAI_API_KEY\"] = (\n",
-    "    \"your_openai_api_key_here\"  # Replace with your actual OpenAI API key\n",
-    ")\n",
-    "\n",
-    "# Verify Helicone API key is set\n",
-    "if HELICONE_API_KEY == \"your_helicone_api_key_here\":\n",
-    "    raise ValueError(\n",
-    "        \"Please replace 'your_helicone_api_key_here' with your actual Helicone API key.\"\n",
-    "    )"
-   ]
+   "source": "import os\nfrom datasets import Dataset\nfrom ragas import evaluate\nfrom ragas.metrics import faithfulness, answer_relevancy, context_precision\nfrom ragas.integrations.helicone import helicone_config  # import helicone_config\n\n\n# Set up Helicone\nHELICONE_API_KEY = \"your_helicone_api_key_here\"  # Replace with your actual Helicone API key\nhelicone_config.api_key = HELICONE_API_KEY\nos.environ[\"OPENAI_API_KEY\"] = (\n    \"your_openai_api_key_here\"  # Replace with your actual OpenAI API key\n)\n\n# Verify Helicone API key is set\nif HELICONE_API_KEY == \"your_helicone_api_key_here\":\n    raise ValueError(\n        \"Please replace 'your_helicone_api_key_here' with your actual Helicone API key.\"\n    )"
   },
   {
    "cell_type": "markdown",
@@ -175,4 +153,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 4
-}
+}
diff --git a/docs/howtos/integrations/langfuse.ipynb b/docs/howtos/integrations/langfuse.ipynb
@@ -149,7 +149,7 @@
    "source": [
     "# import metrics\n",
     "from ragas.metrics import faithfulness, answer_relevancy, context_precision\n",
-    "from ragas.metrics.critique import SUPPORTED_ASPECTS, harmfulness\n",
+    "from ragas.metrics.critique import harmfulness\n",
     "\n",
     "# metrics you chose\n",
     "metrics = [faithfulness, answer_relevancy, context_precision, harmfulness]"
diff --git a/docs/howtos/integrations/openlayer.ipynb b/docs/howtos/integrations/openlayer.ipynb
@@ -197,15 +197,10 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
    "id": "ced5f583-b849-4aae-8397-2bd9006bb69f",
    "metadata": {},
    "outputs": [],
-   "source": [
-    "from openlayer.tasks import TaskType\n",
-    "\n",
-    "client = openlayer.OpenlayerClient(\"YOUR_OPENLAYER_API_KEY_HERE\")"
-   ]
+   "source": "import openlayer\nfrom openlayer.tasks import TaskType\n\nclient = openlayer.OpenlayerClient(\"YOUR_OPENLAYER_API_KEY_HERE\")"
   },
   {
    "cell_type": "code",
@@ -298,4 +293,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 5
-}
+}
diff --git a/docs/howtos/integrations/opik.ipynb b/docs/howtos/integrations/opik.ipynb
@@ -298,7 +298,6 @@
     "from datasets import load_dataset\n",
     "from ragas.metrics import context_precision, answer_relevancy, faithfulness\n",
     "from ragas import evaluate\n",
-    "from ragas.integrations.opik import OpikTracer\n",
     "\n",
     "fiqa_eval = load_dataset(\"explodinggradients/fiqa\", \"ragas_eval\")\n",
     "\n",
diff --git a/mkdocs.yml b/mkdocs.yml
@@ -77,41 +77,7 @@ nav:
       - Feedback Intelligence:
           - concepts/feedback/index.md
   - 🧪 Experimental:
-    - Overview: experimental/index.html.md
-    - Core:
-      - Project:
-        - Core: experimental/project/core.html.md
-        - Experiments: experimental/project/experiments.html.md
-        - Naming: experimental/project/naming.html.md
-      - Dataset: experimental/dataset.html.md
-      - Experiment: experimental/experiment.html.md
-      - Utils: experimental/utils.html.md
-      - Typing: experimental/typing.html.md
-    - Models:
-      - Pydantic Models: experimental/model/pydantic_mode.html.md
-    - Components:
-      - LLM:
-        - Base: experimental/llm/llm.html.md
-      - Embedding:
-        - Base: experimental/embedding/base.md
-      - Prompt:
-        - Base: experimental/prompt/base.md
-        - Dynamic Few Shot: experimental/prompt/dynamic_few_shot.html.md
-      - Metric:
-        - Base: experimental/metric/base.html.md
-        - Decorator: experimental/metric/decorator.html.md
-        - Discrete: experimental/metric/discrete.html.md
-        - Numeric: experimental/metric/numeric.html.md
-        - Ranking: experimental/metric/ranking.html.md
-        - Result: experimental/metric/result.html.md
-    - Backends:
-      - Factory: experimental/backends/factory.html.md
-      - Ragas API Client: experimental/backends/ragas_api_client.html.md
-    - Tracing:
-      - Langfuse: experimental/tracing/langfuse.html.md
-      - MLflow: experimental/tracing/mlflow.html.md
-    - Exceptions: experimental/exceptions.html.md
-    - Init Module: experimental/init_module.md
+    - Overview: experimental/index.md
   - 🛠️ How-to Guides:
       - howtos/index.md
       - Customizations:
diff --git a/ragas/src/ragas/_version.py b/ragas/src/ragas/_version.py
diff --git a/ragas/src/ragas/embeddings/haystack_wrapper.py b/ragas/src/ragas/embeddings/haystack_wrapper.py
diff --git a/ragas/src/ragas/llms/haystack_wrapper.py b/ragas/src/ragas/llms/haystack_wrapper.py
diff --git a/ragas/src/ragas/testset/transforms/extractors/embeddings.py b/ragas/src/ragas/testset/transforms/extractors/embeddings.py
diff --git a/ragas/src/ragas/utils.py b/ragas/src/ragas/utils.py

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+# Ragas Experimental`
	`2`	`+`
	`3`	`+Under the works but stay tuned :)`
Original file line number	Diff line number	Diff line change
`@@ -53,7 +53,7 @@`
`53`	`53`	`}`
`54`	`54`	`],`
`55`	`55`	`"source": [`
`56`		`- "from langchain_community.document_loaders import DirectoryLoader, TextLoader\n",`
	`56`	`+ "from langchain_community.document_loaders import DirectoryLoader\n",`
`57`	`57`	`"\n",`
`58`	`58`	`"\n",`
`59`	`59`	`"path = \"Sample_non_english_corpus/\"\n",`