Skip to content

Commit 2a61ed7

Browse files
trial-danswertrial-danswer
committed
tables/migration for permission syncing attempts
1 parent 2b1c5a0 commit 2a61ed7

File tree

3 files changed

+414
-13
lines changed

3 files changed

+414
-13
lines changed

‎backend/alembic/versions/03d710ccf29c_add_permission_sync_attempt_tables.py‎

Lines changed: 205 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,205 @@
1+
"""add permission sync attempt tables
2+
3+
Revision ID: 03d710ccf29c
4+
Revises: b7ec9b5b505f
5+
Create Date: 2025-09-11 13:30:00.000000
6+
7+
"""
8+
9+
from alembic import op
10+
import sqlalchemy as sa
11+
12+
13+
# revision identifiers, used by Alembic.
14+
revision = "03d710ccf29c" # Generate a new unique ID
15+
down_revision = "b7ec9b5b505f"
16+
branch_labels = None
17+
depends_on = None
18+
19+
20+
def upgrade() -> None:
21+
# Create connector_permission_sync_attempt table
22+
op.create_table(
23+
"connector_permission_sync_attempt",
24+
sa.Column("id", sa.Integer(), nullable=False),
25+
sa.Column("connector_credential_pair_id", sa.Integer(), nullable=False),
26+
sa.Column(
27+
"status",
28+
sa.Enum(
29+
"not_started",
30+
"in_progress",
31+
"success",
32+
"canceled",
33+
"failed",
34+
"completed_with_errors",
35+
name="permissionsyncstatus",
36+
native_enum=False,
37+
),
38+
nullable=False,
39+
),
40+
sa.Column(
41+
"sync_type",
42+
sa.Enum(
43+
"doc_permissions",
44+
"group_permissions",
45+
"full_sync",
46+
name="permissionsynctype",
47+
native_enum=False,
48+
),
49+
nullable=False,
50+
),
51+
sa.Column("total_docs_synced", sa.Integer(), nullable=True),
52+
sa.Column("total_groups_synced", sa.Integer(), nullable=True),
53+
sa.Column("docs_with_permission_errors", sa.Integer(), nullable=True),
54+
sa.Column("error_msg", sa.Text(), nullable=True),
55+
sa.Column("full_exception_trace", sa.Text(), nullable=True),
56+
sa.Column("celery_task_id", sa.String(), nullable=True),
57+
sa.Column(
58+
"cancellation_requested",
59+
sa.Boolean(),
60+
nullable=False,
61+
server_default=sa.text("false"),
62+
),
63+
sa.Column(
64+
"time_created",
65+
sa.DateTime(timezone=True),
66+
server_default=sa.text("now()"),
67+
nullable=False,
68+
),
69+
sa.Column("time_started", sa.DateTime(timezone=True), nullable=True),
70+
sa.Column(
71+
"time_updated",
72+
sa.DateTime(timezone=True),
73+
server_default=sa.text("now()"),
74+
nullable=False,
75+
),
76+
sa.ForeignKeyConstraint(
77+
["connector_credential_pair_id"],
78+
["connector_credential_pair.id"],
79+
),
80+
sa.PrimaryKeyConstraint("id"),
81+
)
82+
83+
# Create indexes for connector_permission_sync_attempt
84+
op.create_index(
85+
"ix_connector_permission_sync_attempt_time_created",
86+
"connector_permission_sync_attempt",
87+
["time_created"],
88+
unique=False,
89+
)
90+
op.create_index(
91+
"ix_permission_sync_attempt_latest_for_cc_pair",
92+
"connector_permission_sync_attempt",
93+
["connector_credential_pair_id", "time_created"],
94+
unique=False,
95+
)
96+
op.create_index(
97+
"ix_permission_sync_attempt_status_time",
98+
"connector_permission_sync_attempt",
99+
["status", sa.text("time_updated DESC")],
100+
unique=False,
101+
)
102+
103+
# Create external_group_permission_sync_attempt table
104+
op.create_table(
105+
"external_group_permission_sync_attempt",
106+
sa.Column("id", sa.Integer(), nullable=False),
107+
sa.Column("connector_credential_pair_id", sa.Integer(), nullable=True),
108+
sa.Column(
109+
"status",
110+
sa.Enum(
111+
"not_started",
112+
"in_progress",
113+
"success",
114+
"canceled",
115+
"failed",
116+
"completed_with_errors",
117+
name="permissionsyncstatus",
118+
native_enum=False,
119+
),
120+
nullable=False,
121+
),
122+
sa.Column("total_users_processed", sa.Integer(), nullable=True),
123+
sa.Column("total_groups_processed", sa.Integer(), nullable=True),
124+
sa.Column("total_group_memberships_synced", sa.Integer(), nullable=True),
125+
sa.Column("users_with_sync_errors", sa.Integer(), nullable=True),
126+
sa.Column("error_msg", sa.Text(), nullable=True),
127+
sa.Column("full_exception_trace", sa.Text(), nullable=True),
128+
sa.Column("celery_task_id", sa.String(), nullable=True),
129+
sa.Column(
130+
"cancellation_requested",
131+
sa.Boolean(),
132+
nullable=False,
133+
server_default=sa.text("false"),
134+
),
135+
sa.Column(
136+
"time_created",
137+
sa.DateTime(timezone=True),
138+
server_default=sa.text("now()"),
139+
nullable=False,
140+
),
141+
sa.Column("time_started", sa.DateTime(timezone=True), nullable=True),
142+
sa.Column(
143+
"time_updated",
144+
sa.DateTime(timezone=True),
145+
server_default=sa.text("now()"),
146+
nullable=False,
147+
),
148+
sa.ForeignKeyConstraint(
149+
["connector_credential_pair_id"],
150+
["connector_credential_pair.id"],
151+
),
152+
sa.PrimaryKeyConstraint("id"),
153+
)
154+
155+
# Create indexes for external_group_permission_sync_attempt
156+
op.create_index(
157+
"ix_external_group_permission_sync_attempt_time_created",
158+
"external_group_permission_sync_attempt",
159+
["time_created"],
160+
unique=False,
161+
)
162+
op.create_index(
163+
"ix_group_sync_attempt_cc_pair_time",
164+
"external_group_permission_sync_attempt",
165+
["connector_credential_pair_id", "time_created"],
166+
unique=False,
167+
)
168+
op.create_index(
169+
"ix_group_sync_attempt_status_time",
170+
"external_group_permission_sync_attempt",
171+
["status", sa.text("time_updated DESC")],
172+
unique=False,
173+
)
174+
175+
176+
def downgrade() -> None:
177+
# Drop indexes
178+
op.drop_index(
179+
"ix_group_sync_attempt_status_time",
180+
table_name="external_group_permission_sync_attempt",
181+
)
182+
op.drop_index(
183+
"ix_group_sync_attempt_cc_pair_time",
184+
table_name="external_group_permission_sync_attempt",
185+
)
186+
op.drop_index(
187+
"ix_external_group_permission_sync_attempt_time_created",
188+
table_name="external_group_permission_sync_attempt",
189+
)
190+
op.drop_index(
191+
"ix_permission_sync_attempt_status_time",
192+
table_name="connector_permission_sync_attempt",
193+
)
194+
op.drop_index(
195+
"ix_permission_sync_attempt_latest_for_cc_pair",
196+
table_name="connector_permission_sync_attempt",
197+
)
198+
op.drop_index(
199+
"ix_connector_permission_sync_attempt_time_created",
200+
table_name="connector_permission_sync_attempt",
201+
)
202+
203+
# Drop tables
204+
op.drop_table("external_group_permission_sync_attempt")
205+
op.drop_table("connector_permission_sync_attempt")

‎backend/onyx/db/enums.py‎

Lines changed: 38 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,55 @@
11
from enum import Enum as PyEnum
22

33

4-
class IndexingStatus(str, PyEnum):
5-
NOT_STARTED = "not_started"
6-
IN_PROGRESS = "in_progress"
7-
SUCCESS = "success"
8-
CANCELED = "canceled"
9-
FAILED = "failed"
10-
COMPLETED_WITH_ERRORS = "completed_with_errors"
4+
class AttemptStatusMixin:
5+
"""Mixin providing common methods for attempt status enums"""
116

127
def is_terminal(self) -> bool:
138
terminal_states = {
14-
IndexingStatus.SUCCESS,
15-
IndexingStatus.COMPLETED_WITH_ERRORS,
16-
IndexingStatus.CANCELED,
17-
IndexingStatus.FAILED,
9+
self.__class__.SUCCESS,
10+
self.__class__.COMPLETED_WITH_ERRORS,
11+
self.__class__.CANCELED,
12+
self.__class__.FAILED,
1813
}
1914
return self in terminal_states
2015

2116
def is_successful(self) -> bool:
2217
return (
23-
self == IndexingStatus.SUCCESS
24-
or self == IndexingStatus.COMPLETED_WITH_ERRORS
18+
self == self.__class__.SUCCESS
19+
or self == self.__class__.COMPLETED_WITH_ERRORS
2520
)
2621

2722

23+
class IndexingStatus(AttemptStatusMixin, str, PyEnum):
24+
"""Status enum for indexing attempts"""
25+
26+
NOT_STARTED = "not_started"
27+
IN_PROGRESS = "in_progress"
28+
SUCCESS = "success"
29+
CANCELED = "canceled"
30+
FAILED = "failed"
31+
COMPLETED_WITH_ERRORS = "completed_with_errors"
32+
33+
34+
class PermissionSyncStatus(AttemptStatusMixin, str, PyEnum):
35+
"""Status enum for permission sync attempts"""
36+
37+
NOT_STARTED = "not_started"
38+
IN_PROGRESS = "in_progress"
39+
SUCCESS = "success"
40+
CANCELED = "canceled"
41+
FAILED = "failed"
42+
COMPLETED_WITH_ERRORS = "completed_with_errors"
43+
44+
45+
class PermissionSyncType(str, PyEnum):
46+
"""Types of permission sync operations"""
47+
48+
DOCUMENT_PERMISSIONS = "doc_permissions"
49+
GROUP_PERMISSIONS = "group_permissions"
50+
FULL_SYNC = "full_sync" # Both document and group permissions
51+
52+
2853
class IndexingMode(str, PyEnum):
2954
UPDATE = "update"
3055
REINDEX = "reindex"

0 commit comments

Comments
 (0)