From bf3936bd061b2a423d39e501c5e7cea111e452c9 Mon Sep 17 00:00:00 2001 From: vladimirrotariu Date: Tue, 29 Jul 2025 16:11:29 -0600 Subject: [PATCH 01/39] add constant with endpoint for batch annotation --- apps/opik-frontend/src/api/api.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/opik-frontend/src/api/api.ts b/apps/opik-frontend/src/api/api.ts index 5b9e145ad91..7c3d7451ce8 100644 --- a/apps/opik-frontend/src/api/api.ts +++ b/apps/opik-frontend/src/api/api.ts @@ -17,6 +17,7 @@ export const EXPERIMENTS_REST_ENDPOINT = "/v1/private/experiments/"; export const FEEDBACK_DEFINITIONS_REST_ENDPOINT = "/v1/private/feedback-definitions/"; export const TRACES_REST_ENDPOINT = "/v1/private/traces/"; +export const TRACES_BATCH_FEEDBACK_SCORES_ENDPOINT = "/v1/private/traces/feedback-scores"; export const SPANS_REST_ENDPOINT = "/v1/private/spans/"; export const PROMPTS_REST_ENDPOINT = "/v1/private/prompts/"; export const PROVIDER_KEYS_REST_ENDPOINT = "/v1/private/llm-provider-key/"; From 1e7d14362b581cdca0a64bd4eb720eccd77a68d4 Mon Sep 17 00:00:00 2001 From: vladimirrotariu Date: Tue, 29 Jul 2025 16:22:52 -0600 Subject: [PATCH 02/39] add mutation for batch annotation --- .../useTracesBatchFeedbackScoresMutation.ts | 98 +++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 apps/opik-frontend/src/api/traces/useTracesBatchFeedbackScoresMutation.ts diff --git a/apps/opik-frontend/src/api/traces/useTracesBatchFeedbackScoresMutation.ts b/apps/opik-frontend/src/api/traces/useTracesBatchFeedbackScoresMutation.ts new file mode 100644 index 00000000000..458e2d2ff97 --- /dev/null +++ b/apps/opik-frontend/src/api/traces/useTracesBatchFeedbackScoresMutation.ts @@ -0,0 +1,98 @@ +import api, { + COMPARE_EXPERIMENTS_KEY, + TRACE_KEY, + TRACES_BATCH_FEEDBACK_SCORES_ENDPOINT, + TRACES_KEY, +} from "@/api/api"; +import { useToast } from "@/components/ui/use-toast"; +import { + generateUpdateMutation, + setExperimentsCompareCache, + setTraceCache, + setTracesCache, +} from "@/lib/feedback-scores"; +import { FEEDBACK_SCORE_TYPE } from "@/types/traces"; +import { useMutation, useQueryClient } from "@tanstack/react-query"; +import { AxiosError } from "axios"; +import get from "lodash/get"; + +type UseTracesBatchFeedbackScoresParams = { + projectId: string; + traceIds: string[]; + name: string; + value: number; + categoryName?: string; + reason?: string; +}; + +const useTracesBatchFeedbackScoresMutation = () => { + const queryClient = useQueryClient(); + const { toast } = useToast(); + + return useMutation({ + mutationFn: async ({ + projectId, + traceIds, + name, + value, + categoryName, + reason, + }: UseTracesBatchFeedbackScoresParams) => { + const body = { + scores: traceIds.map((id) => ({ + trace_id: id, + project_id: projectId, + name, + value, + ...(categoryName && { category_name: categoryName }), + ...(reason && { reason }), + source: FEEDBACK_SCORE_TYPE.ui, + })), + }; + + const { data } = await api.put(TRACES_BATCH_FEEDBACK_SCORES_ENDPOINT, body); + return data; + }, + onError: (error: AxiosError) => { + const message = get(error, ["response", "data", "message"], error.message); + toast({ + title: "Error", + description: message, + variant: "destructive", + }); + }, + onSuccess: (data, vars) => { + toast({ + title: "Success", + description: `Annotated ${vars.traceIds.length} traces`, + }); + }, + onMutate: async (params: UseTracesBatchFeedbackScoresParams) => { + const updateMutation = generateUpdateMutation({ + name: params.name, + category_name: params.categoryName, + value: params.value, + source: FEEDBACK_SCORE_TYPE.ui, + reason: params.reason, + }); + + await Promise.all( + params.traceIds.map(async (traceId) => { + const traceParams = { traceId }; + await setExperimentsCompareCache(queryClient, traceParams, updateMutation); + await setTracesCache(queryClient, traceParams, updateMutation); + await setTraceCache(queryClient, traceParams, updateMutation); + }), + ); + }, + onSettled: async () => { + await queryClient.invalidateQueries({ queryKey: [TRACES_KEY] }); + await queryClient.invalidateQueries({ queryKey: ["traces-columns"] }); + await queryClient.invalidateQueries({ queryKey: ["traces-statistic"] }); + await queryClient.invalidateQueries({ queryKey: [TRACE_KEY] }); + await queryClient.invalidateQueries({ queryKey: [COMPARE_EXPERIMENTS_KEY] }); + }, + }); +}; + +export default useTracesBatchFeedbackScoresMutation; \ No newline at end of file From e50abe28359644306559158c70d23d3cc6fe0566 Mon Sep 17 00:00:00 2001 From: vladimirrotariu Date: Tue, 29 Jul 2025 16:31:21 -0600 Subject: [PATCH 03/39] add modal window component --- .../useTracesBatchFeedbackScoresMutation.ts | 2 +- .../BatchAnnotateDialog.tsx | 127 ++++++++++++++++++ 2 files changed, 128 insertions(+), 1 deletion(-) create mode 100644 apps/opik-frontend/src/components/pages-shared/traces/BatchAnnotateDialog/BatchAnnotateDialog.tsx diff --git a/apps/opik-frontend/src/api/traces/useTracesBatchFeedbackScoresMutation.ts b/apps/opik-frontend/src/api/traces/useTracesBatchFeedbackScoresMutation.ts index 458e2d2ff97..a2364aef7ee 100644 --- a/apps/opik-frontend/src/api/traces/useTracesBatchFeedbackScoresMutation.ts +++ b/apps/opik-frontend/src/api/traces/useTracesBatchFeedbackScoresMutation.ts @@ -61,7 +61,7 @@ const useTracesBatchFeedbackScoresMutation = () => { variant: "destructive", }); }, - onSuccess: (data, vars) => { + onSuccess: (_, vars) => { toast({ title: "Success", description: `Annotated ${vars.traceIds.length} traces`, diff --git a/apps/opik-frontend/src/components/pages-shared/traces/BatchAnnotateDialog/BatchAnnotateDialog.tsx b/apps/opik-frontend/src/components/pages-shared/traces/BatchAnnotateDialog/BatchAnnotateDialog.tsx new file mode 100644 index 00000000000..11802ec435b --- /dev/null +++ b/apps/opik-frontend/src/components/pages-shared/traces/BatchAnnotateDialog/BatchAnnotateDialog.tsx @@ -0,0 +1,127 @@ +import useTracesBatchFeedbackScoresMutation from "@/api/traces/useTracesBatchFeedbackScoresMutation"; +import { Button } from "@/components/ui/button"; +import { + Dialog, + DialogContent, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { Input } from "@/components/ui/input"; +import { Textarea } from "@/components/ui/textarea"; +import { useToast } from "@/components/ui/use-toast"; +import { Trace } from "@/types/traces"; +import React, { useState } from "react"; + +type BatchAnnotateDialogProps = { + rows: Trace[]; + open: boolean | number; + setOpen: (o: boolean | number) => void; + projectId: string; + onSuccess?: () => void; +}; + +const MAX_ENTITIES = 10; + +const BatchAnnotateDialog: React.FunctionComponent = ({ + rows, + open, + setOpen, + projectId, + onSuccess, +}) => { + const { toast } = useToast(); + const [scoreName, setScoreName] = useState(""); + const [value, setValue] = useState(""); + const [categoryName, setCategoryName] = useState(""); + const [reason, setReason] = useState(""); + + const batchMutation = useTracesBatchFeedbackScoresMutation(); + + const handleClose = () => { + setOpen(false); + setScoreName(""); + setValue(""); + setCategoryName(""); + setReason(""); + }; + + const disabled = + !scoreName || value === "" || rows.length > MAX_ENTITIES || batchMutation.isPending; + + const handleAnnotate = async () => { + try { + await batchMutation.mutateAsync({ + projectId, + traceIds: rows.map((r) => r.id), + name: scoreName, + value: typeof value === "string" ? Number(value) : value, + categoryName: categoryName || undefined, + reason: reason || undefined, + }); + if (onSuccess) onSuccess(); + handleClose(); + } catch (e) { + toast({ + title: "Error", + description: "Failed to annotate traces", + variant: "destructive", + }); + } + }; + + return ( + + + + Annotate {rows.length} traces + + {rows.length > MAX_ENTITIES && ( +
+ You can annotate up to {MAX_ENTITIES} traces at a time. Please reduce your selection. +
+ )} +
+ setScoreName(e.target.value)} + disabled={rows.length > MAX_ENTITIES} + /> + { + const v = e.target.value; + setValue(v === "" ? "" : Number(v)); + }} + disabled={rows.length > MAX_ENTITIES} + /> + setCategoryName(e.target.value)} + disabled={rows.length > MAX_ENTITIES} + /> +