Skip to content

Commit ad8e2dd

Browse files
WevesJoachim RahmfeldOrbital-Webjoachim-danswer
authored
feat: frontend refactor + DR (onyx-dot-app#5225)
* squash: combine all DR commits into one Co-authored-by: Joachim Rahmfeld <joachim@onyx.app> Co-authored-by: Rei Meguro <rmeguro@umich.edu> * Fixes * show KG in Assistant only if available * KG only usable for KG Beta (for now) * base file upload * raise error if uploaded context is too long * improvements * More improvements * Fix citations * better decision making * improved decision-making in Orchestrator * generic_internal tools * Small tweak * tool use improvements * add on * More image gen stuff * fixes * Small color improvements * Markdown utils * fixed end conditions (incl early exit for image generation) * remove agent search + image fixes * Okta tool support for reload * Some cleanup * Stream back search tool results as they come * tool forcing * fixed no-Tool-Assistant * Support anthropic tool calling * Support anthropic models better * More stuff * prompt fixes and search step numbers * Fix hook ordering issue * internal search fix * Improve citation look * Small UI improvements * Improvements * Improve dot * Small chat fixes * Small UI tweaks * Small improvements * Remove un-used code * Fix * Remove test_answer.py for now * Fix * improvements * Add foreign keys * early forcing * Fix tests * Fix tests --------- Co-authored-by: Joachim Rahmfeld <joachim@onyx.app> Co-authored-by: Rei Meguro <rmeguro@umich.edu> Co-authored-by: joachim-danswer <joachim@danswer.ai>
1 parent b3e3ab5 commit ad8e2dd

File tree

305 files changed

+18032
-15970
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

305 files changed

+18032
-15970
lines changed

‎backend/alembic/versions/5ae8240accb3_add_research_agent_database_tables_and_.py‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
"""add research agent database tables and chat message research fields
2+
3+
Revision ID: 5ae8240accb3
4+
Revises: b558f51620b4
5+
Create Date: 2025-08-06 14:29:24.691388
6+
7+
"""
8+
9+
from alembic import op
10+
import sqlalchemy as sa
11+
from sqlalchemy.dialects import postgresql
12+
13+
14+
# revision identifiers, used by Alembic.
15+
revision = "5ae8240accb3"
16+
down_revision = "b558f51620b4"
17+
branch_labels = None
18+
depends_on = None
19+
20+
21+
def upgrade() -> None:
22+
# Add research_type and research_plan columns to chat_message table
23+
op.add_column(
24+
"chat_message",
25+
sa.Column("research_type", sa.String(), nullable=True),
26+
)
27+
op.add_column(
28+
"chat_message",
29+
sa.Column("research_plan", postgresql.JSONB(), nullable=True),
30+
)
31+
32+
# Create research_agent_iteration table
33+
op.create_table(
34+
"research_agent_iteration",
35+
sa.Column("id", sa.Integer(), autoincrement=True, nullable=False),
36+
sa.Column(
37+
"primary_question_id",
38+
sa.Integer(),
39+
sa.ForeignKey("chat_message.id", ondelete="CASCADE"),
40+
nullable=False,
41+
),
42+
sa.Column("iteration_nr", sa.Integer(), nullable=False),
43+
sa.Column(
44+
"created_at",
45+
sa.DateTime(timezone=True),
46+
server_default=sa.func.now(),
47+
nullable=False,
48+
),
49+
sa.Column("purpose", sa.String(), nullable=True),
50+
sa.Column("reasoning", sa.String(), nullable=True),
51+
sa.PrimaryKeyConstraint("id"),
52+
sa.UniqueConstraint(
53+
"primary_question_id",
54+
"iteration_nr",
55+
name="_research_agent_iteration_unique_constraint",
56+
),
57+
)
58+
59+
# Create research_agent_iteration_sub_step table
60+
op.create_table(
61+
"research_agent_iteration_sub_step",
62+
sa.Column("id", sa.Integer(), autoincrement=True, nullable=False),
63+
sa.Column(
64+
"primary_question_id",
65+
sa.Integer(),
66+
sa.ForeignKey("chat_message.id", ondelete="CASCADE"),
67+
nullable=False,
68+
),
69+
sa.Column(
70+
"parent_question_id",
71+
sa.Integer(),
72+
sa.ForeignKey("research_agent_iteration_sub_step.id", ondelete="CASCADE"),
73+
nullable=True,
74+
),
75+
sa.Column("iteration_nr", sa.Integer(), nullable=False),
76+
sa.Column("iteration_sub_step_nr", sa.Integer(), nullable=False),
77+
sa.Column(
78+
"created_at",
79+
sa.DateTime(timezone=True),
80+
server_default=sa.func.now(),
81+
nullable=False,
82+
),
83+
sa.Column("sub_step_instructions", sa.String(), nullable=True),
84+
sa.Column(
85+
"sub_step_tool_id",
86+
sa.Integer(),
87+
sa.ForeignKey("tool.id"),
88+
nullable=True,
89+
),
90+
sa.Column("reasoning", sa.String(), nullable=True),
91+
sa.Column("sub_answer", sa.String(), nullable=True),
92+
sa.Column("cited_doc_results", postgresql.JSONB(), nullable=True),
93+
sa.Column("claims", postgresql.JSONB(), nullable=True),
94+
sa.Column("generated_images", postgresql.JSONB(), nullable=True),
95+
sa.Column("additional_data", postgresql.JSONB(), nullable=True),
96+
sa.PrimaryKeyConstraint("id"),
97+
sa.ForeignKeyConstraint(
98+
["primary_question_id", "iteration_nr"],
99+
[
100+
"research_agent_iteration.primary_question_id",
101+
"research_agent_iteration.iteration_nr",
102+
],
103+
ondelete="CASCADE",
104+
),
105+
)
106+
107+
108+
def downgrade() -> None:
109+
# Drop tables in reverse order
110+
op.drop_table("research_agent_iteration_sub_step")
111+
op.drop_table("research_agent_iteration")
112+
113+
# Remove columns from chat_message table
114+
op.drop_column("chat_message", "research_plan")
115+
op.drop_column("chat_message", "research_type")

‎backend/alembic/versions/bd7c3bf8beba_migrate_agent_responses_to_research_.py‎

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
"""migrate_agent_sub_questions_to_research_iterations
2+
3+
Revision ID: bd7c3bf8beba
4+
Revises: f8a9b2c3d4e5
5+
Create Date: 2025-08-18 11:33:27.098287
6+
7+
"""
8+
9+
from alembic import op
10+
import sqlalchemy as sa
11+
12+
13+
# revision identifiers, used by Alembic.
14+
revision = "bd7c3bf8beba"
15+
down_revision = "f8a9b2c3d4e5"
16+
branch_labels = None
17+
depends_on = None
18+
19+
20+
def upgrade() -> None:
21+
# Get connection to execute raw SQL
22+
connection = op.get_bind()
23+
24+
# First, insert data into research_agent_iteration table
25+
# This creates one iteration record per primary_question_id using the earliest time_created
26+
connection.execute(
27+
sa.text(
28+
"""
29+
INSERT INTO research_agent_iteration (primary_question_id, created_at, iteration_nr, purpose, reasoning)
30+
SELECT
31+
primary_question_id,
32+
MIN(time_created) as created_at,
33+
1 as iteration_nr,
34+
'Generating and researching subquestions' as purpose,
35+
'(No previous reasoning)' as reasoning
36+
FROM agent__sub_question
37+
JOIN chat_message on agent__sub_question.primary_question_id = chat_message.id
38+
WHERE primary_question_id IS NOT NULL
39+
AND chat_message.is_agentic = true
40+
GROUP BY primary_question_id
41+
ON CONFLICT DO NOTHING;
42+
"""
43+
)
44+
)
45+
46+
# Then, insert data into research_agent_iteration_sub_step table
47+
# This migrates each sub-question as a sub-step
48+
connection.execute(
49+
sa.text(
50+
"""
51+
INSERT INTO research_agent_iteration_sub_step (
52+
primary_question_id,
53+
iteration_nr,
54+
iteration_sub_step_nr,
55+
created_at,
56+
sub_step_instructions,
57+
sub_step_tool_id,
58+
sub_answer,
59+
cited_doc_results
60+
)
61+
SELECT
62+
primary_question_id,
63+
1 as iteration_nr,
64+
level_question_num as iteration_sub_step_nr,
65+
time_created as created_at,
66+
sub_question as sub_step_instructions,
67+
1 as sub_step_tool_id,
68+
sub_answer,
69+
sub_question_doc_results as cited_doc_results
70+
FROM agent__sub_question
71+
JOIN chat_message on agent__sub_question.primary_question_id = chat_message.id
72+
WHERE chat_message.is_agentic = true
73+
AND primary_question_id IS NOT NULL
74+
ON CONFLICT DO NOTHING;
75+
"""
76+
)
77+
)
78+
79+
# Update chat_message records: set legacy agentic type and answer purpose for existing agentic messages
80+
connection.execute(
81+
sa.text(
82+
"""
83+
UPDATE chat_message
84+
SET research_answer_purpose = 'ANSWER'
85+
WHERE is_agentic = true
86+
AND research_type IS NULL and
87+
message_type = 'ASSISTANT';
88+
"""
89+
)
90+
)
91+
connection.execute(
92+
sa.text(
93+
"""
94+
UPDATE chat_message
95+
SET research_type = 'LEGACY_AGENTIC'
96+
WHERE is_agentic = true
97+
AND research_type IS NULL;
98+
"""
99+
)
100+
)
101+
102+
103+
def downgrade() -> None:
104+
# Get connection to execute raw SQL
105+
connection = op.get_bind()
106+
107+
# Note: This downgrade removes all research agent iteration data
108+
# There's no way to perfectly restore the original agent__sub_question data
109+
# if it was deleted after this migration
110+
111+
# Delete all research_agent_iteration_sub_step records that were migrated
112+
connection.execute(
113+
sa.text(
114+
"""
115+
DELETE FROM research_agent_iteration_sub_step
116+
USING chat_message
117+
WHERE research_agent_iteration_sub_step.primary_question_id = chat_message.id
118+
AND chat_message.research_type = 'LEGACY_AGENTIC';
119+
"""
120+
)
121+
)
122+
123+
# Delete all research_agent_iteration records that were migrated
124+
connection.execute(
125+
sa.text(
126+
"""
127+
DELETE FROM research_agent_iteration
128+
USING chat_message
129+
WHERE research_agent_iteration.primary_question_id = chat_message.id
130+
AND chat_message.research_type = 'LEGACY_AGENTIC';
131+
"""
132+
)
133+
)
134+
135+
# Revert chat_message updates: clear research fields for legacy agentic messages
136+
connection.execute(
137+
sa.text(
138+
"""
139+
UPDATE chat_message
140+
SET research_type = NULL,
141+
research_answer_purpose = NULL
142+
WHERE is_agentic = true
143+
AND research_type = 'LEGACY_AGENTIC'
144+
AND message_type = 'ASSISTANT';
145+
"""
146+
)
147+
)

‎backend/alembic/versions/f8a9b2c3d4e5_add_research_answer_purpose_to_chat_message.py‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
"""add research_answer_purpose to chat_message
2+
3+
Revision ID: f8a9b2c3d4e5
4+
Revises: 5ae8240accb3
5+
Create Date: 2025-01-27 12:00:00.000000
6+
7+
"""
8+
9+
from alembic import op
10+
import sqlalchemy as sa
11+
12+
13+
# revision identifiers, used by Alembic.
14+
revision = "f8a9b2c3d4e5"
15+
down_revision = "5ae8240accb3"
16+
branch_labels = None
17+
depends_on = None
18+
19+
20+
def upgrade() -> None:
21+
# Add research_answer_purpose column to chat_message table
22+
op.add_column(
23+
"chat_message",
24+
sa.Column("research_answer_purpose", sa.String(), nullable=True),
25+
)
26+
27+
28+
def downgrade() -> None:
29+
# Remove research_answer_purpose column from chat_message table
30+
op.drop_column("chat_message", "research_answer_purpose")

‎backend/ee/onyx/chat/process_message.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,17 @@
11
from ee.onyx.server.query_and_chat.models import OneShotQAResponse
22
from onyx.chat.models import AllCitations
3+
from onyx.chat.models import AnswerStream
34
from onyx.chat.models import LLMRelevanceFilterResponse
45
from onyx.chat.models import OnyxAnswerPiece
56
from onyx.chat.models import QADocsResponse
67
from onyx.chat.models import StreamingError
7-
from onyx.chat.process_message import ChatPacketStream
88
from onyx.server.query_and_chat.models import ChatMessageDetail
99
from onyx.utils.timing import log_function_time
1010

1111

1212
@log_function_time()
1313
def gather_stream_for_answer_api(
14-
packets: ChatPacketStream,
14+
packets: AnswerStream,
1515
) -> OneShotQAResponse:
1616
response = OneShotQAResponse()
1717

0 commit comments

Comments
 (0)