Skip to content

Commit b7b43e6

Browse files
trial-danswertrial-danswer
committed
tables/migration for permission syncing attempts
1 parent 36c96f2 commit b7b43e6

File tree

3 files changed

+324
-0
lines changed

3 files changed

+324
-0
lines changed

‎backend/alembic/versions/03d710ccf29c_add_permission_sync_attempt_tables.py‎

Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
"""add permission sync attempt tables
2+
3+
Revision ID: 03d710ccf29c
4+
Revises: b7ec9b5b505f
5+
Create Date: 2025-09-11 13:30:00.000000
6+
7+
"""
8+
9+
from alembic import op
10+
import sqlalchemy as sa
11+
12+
13+
# revision identifiers, used by Alembic.
14+
revision = "03d710ccf29c" # Generate a new unique ID
15+
down_revision = "b7ec9b5b505f"
16+
branch_labels = None
17+
depends_on = None
18+
19+
20+
def upgrade() -> None:
21+
# Create the permission sync status enum
22+
permission_sync_status_enum = sa.Enum(
23+
"not_started",
24+
"in_progress",
25+
"success",
26+
"canceled",
27+
"failed",
28+
"completed_with_errors",
29+
name="permissionsyncstatus",
30+
native_enum=False,
31+
)
32+
33+
# Create doc_permission_sync_attempt table
34+
op.create_table(
35+
"doc_permission_sync_attempt",
36+
sa.Column("id", sa.Integer(), nullable=False),
37+
sa.Column("connector_credential_pair_id", sa.Integer(), nullable=False),
38+
sa.Column("status", permission_sync_status_enum, nullable=False),
39+
sa.Column("total_docs_synced", sa.Integer(), nullable=True),
40+
sa.Column("docs_with_permission_errors", sa.Integer(), nullable=True),
41+
sa.Column("error_message", sa.Text(), nullable=True),
42+
sa.Column(
43+
"time_created",
44+
sa.DateTime(timezone=True),
45+
server_default=sa.text("now()"),
46+
nullable=False,
47+
),
48+
sa.Column("time_started", sa.DateTime(timezone=True), nullable=True),
49+
sa.Column("time_finished", sa.DateTime(timezone=True), nullable=True),
50+
sa.ForeignKeyConstraint(
51+
["connector_credential_pair_id"],
52+
["connector_credential_pair.id"],
53+
ondelete="CASCADE",
54+
),
55+
sa.PrimaryKeyConstraint("id"),
56+
)
57+
58+
# Create indexes for doc_permission_sync_attempt
59+
op.create_index(
60+
"ix_doc_permission_sync_attempt_time_created",
61+
"doc_permission_sync_attempt",
62+
["time_created"],
63+
unique=False,
64+
)
65+
op.create_index(
66+
"ix_permission_sync_attempt_latest_for_cc_pair",
67+
"doc_permission_sync_attempt",
68+
["connector_credential_pair_id", "time_created"],
69+
unique=False,
70+
)
71+
op.create_index(
72+
"ix_permission_sync_attempt_status_time",
73+
"doc_permission_sync_attempt",
74+
["status", sa.text("time_finished DESC")],
75+
unique=False,
76+
)
77+
78+
# Create external_group_permission_sync_attempt table
79+
# connector_credential_pair_id is nullable - group syncs can be global (e.g., Confluence)
80+
op.create_table(
81+
"external_group_permission_sync_attempt",
82+
sa.Column("id", sa.Integer(), nullable=False),
83+
sa.Column("connector_credential_pair_id", sa.Integer(), nullable=True),
84+
sa.Column("status", permission_sync_status_enum, nullable=False),
85+
sa.Column("total_users_processed", sa.Integer(), nullable=True),
86+
sa.Column("total_groups_processed", sa.Integer(), nullable=True),
87+
sa.Column("total_group_memberships_synced", sa.Integer(), nullable=True),
88+
sa.Column("error_message", sa.Text(), nullable=True),
89+
sa.Column(
90+
"time_created",
91+
sa.DateTime(timezone=True),
92+
server_default=sa.text("now()"),
93+
nullable=False,
94+
),
95+
sa.Column("time_started", sa.DateTime(timezone=True), nullable=True),
96+
sa.Column("time_finished", sa.DateTime(timezone=True), nullable=True),
97+
sa.ForeignKeyConstraint(
98+
["connector_credential_pair_id"],
99+
["connector_credential_pair.id"],
100+
ondelete="CASCADE",
101+
),
102+
sa.PrimaryKeyConstraint("id"),
103+
)
104+
105+
# Create indexes for external_group_permission_sync_attempt
106+
op.create_index(
107+
"ix_external_group_permission_sync_attempt_time_created",
108+
"external_group_permission_sync_attempt",
109+
["time_created"],
110+
unique=False,
111+
)
112+
op.create_index(
113+
"ix_group_sync_attempt_cc_pair_time",
114+
"external_group_permission_sync_attempt",
115+
["connector_credential_pair_id", "time_created"],
116+
unique=False,
117+
)
118+
op.create_index(
119+
"ix_group_sync_attempt_status_time",
120+
"external_group_permission_sync_attempt",
121+
["status", sa.text("time_finished DESC")],
122+
unique=False,
123+
)
124+
125+
126+
def downgrade() -> None:
127+
# Drop indexes
128+
op.drop_index(
129+
"ix_group_sync_attempt_status_time",
130+
table_name="external_group_permission_sync_attempt",
131+
)
132+
op.drop_index(
133+
"ix_group_sync_attempt_cc_pair_time",
134+
table_name="external_group_permission_sync_attempt",
135+
)
136+
op.drop_index(
137+
"ix_external_group_permission_sync_attempt_time_created",
138+
table_name="external_group_permission_sync_attempt",
139+
)
140+
op.drop_index(
141+
"ix_permission_sync_attempt_status_time",
142+
table_name="doc_permission_sync_attempt",
143+
)
144+
op.drop_index(
145+
"ix_permission_sync_attempt_latest_for_cc_pair",
146+
table_name="doc_permission_sync_attempt",
147+
)
148+
op.drop_index(
149+
"ix_doc_permission_sync_attempt_time_created",
150+
table_name="doc_permission_sync_attempt",
151+
)
152+
153+
# Drop tables
154+
op.drop_table("external_group_permission_sync_attempt")
155+
op.drop_table("doc_permission_sync_attempt")

‎backend/onyx/db/enums.py‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,32 @@ def is_successful(self) -> bool:
2525
)
2626

2727

28+
class PermissionSyncStatus(str, PyEnum):
29+
"""Status enum for permission sync attempts"""
30+
31+
NOT_STARTED = "not_started"
32+
IN_PROGRESS = "in_progress"
33+
SUCCESS = "success"
34+
CANCELED = "canceled"
35+
FAILED = "failed"
36+
COMPLETED_WITH_ERRORS = "completed_with_errors"
37+
38+
def is_terminal(self) -> bool:
39+
terminal_states = {
40+
PermissionSyncStatus.SUCCESS,
41+
PermissionSyncStatus.COMPLETED_WITH_ERRORS,
42+
PermissionSyncStatus.CANCELED,
43+
PermissionSyncStatus.FAILED,
44+
}
45+
return self in terminal_states
46+
47+
def is_successful(self) -> bool:
48+
return (
49+
self == PermissionSyncStatus.SUCCESS
50+
or self == PermissionSyncStatus.COMPLETED_WITH_ERRORS
51+
)
52+
53+
2854
class IndexingMode(str, PyEnum):
2955
UPDATE = "update"
3056
REINDEX = "reindex"

‎backend/onyx/db/models.py‎

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,7 @@
7171
from onyx.db.enums import ConnectorCredentialPairStatus
7272
from onyx.db.enums import IndexingStatus
7373
from onyx.db.enums import IndexModelStatus
74+
from onyx.db.enums import PermissionSyncStatus
7475
from onyx.db.enums import TaskStatus
7576
from onyx.db.pydantic_type import PydanticListType, PydanticType
7677
from onyx.kg.models import KGEntityTypeAttributes
@@ -3593,3 +3594,145 @@ class MCPConnectionConfig(Base):
35933594
Index("ix_mcp_connection_config_user_email", "user_email"),
35943595
Index("ix_mcp_connection_config_server_user", "mcp_server_id", "user_email"),
35953596
)
3597+
3598+
3599+
"""
3600+
Permission Sync Tables
3601+
"""
3602+
3603+
3604+
class DocPermissionSyncAttempt(Base):
3605+
"""
3606+
Represents an attempt to sync document permissions for a connector credential pair.
3607+
Similar to IndexAttempt but specifically for document permission syncing operations.
3608+
"""
3609+
3610+
__tablename__ = "doc_permission_sync_attempt"
3611+
3612+
id: Mapped[int] = mapped_column(primary_key=True)
3613+
3614+
connector_credential_pair_id: Mapped[int] = mapped_column(
3615+
ForeignKey("connector_credential_pair.id", ondelete="CASCADE"),
3616+
nullable=False,
3617+
)
3618+
3619+
# Status of the sync attempt
3620+
status: Mapped[PermissionSyncStatus] = mapped_column(
3621+
Enum(PermissionSyncStatus, native_enum=False, index=True)
3622+
)
3623+
3624+
# Counts for tracking progress
3625+
total_docs_synced: Mapped[int | None] = mapped_column(Integer, default=0)
3626+
docs_with_permission_errors: Mapped[int | None] = mapped_column(Integer, default=0)
3627+
3628+
# Error message if sync fails
3629+
error_message: Mapped[str | None] = mapped_column(Text, default=None)
3630+
3631+
# Timestamps
3632+
time_created: Mapped[datetime.datetime] = mapped_column(
3633+
DateTime(timezone=True),
3634+
server_default=func.now(),
3635+
index=True,
3636+
)
3637+
time_started: Mapped[datetime.datetime | None] = mapped_column(
3638+
DateTime(timezone=True), default=None
3639+
)
3640+
time_finished: Mapped[datetime.datetime | None] = mapped_column(
3641+
DateTime(timezone=True), default=None
3642+
)
3643+
3644+
# Relationships
3645+
connector_credential_pair: Mapped[ConnectorCredentialPair] = relationship(
3646+
"ConnectorCredentialPair"
3647+
)
3648+
3649+
__table_args__ = (
3650+
Index(
3651+
"ix_permission_sync_attempt_latest_for_cc_pair",
3652+
"connector_credential_pair_id",
3653+
"time_created",
3654+
),
3655+
Index(
3656+
"ix_permission_sync_attempt_status_time",
3657+
"status",
3658+
desc("time_finished"),
3659+
),
3660+
)
3661+
3662+
def __repr__(self) -> str:
3663+
return f"<DocPermissionSyncAttempt(id={self.id!r}, " f"status={self.status!r})>"
3664+
3665+
def is_finished(self) -> bool:
3666+
return self.status.is_terminal()
3667+
3668+
3669+
class ExternalGroupPermissionSyncAttempt(Base):
3670+
"""
3671+
Represents an attempt to sync external group memberships for users.
3672+
This tracks the syncing of user-to-external-group mappings across connectors.
3673+
"""
3674+
3675+
__tablename__ = "external_group_permission_sync_attempt"
3676+
3677+
id: Mapped[int] = mapped_column(primary_key=True)
3678+
3679+
# Can be tied to a specific connector or be a global group sync
3680+
connector_credential_pair_id: Mapped[int | None] = mapped_column(
3681+
ForeignKey("connector_credential_pair.id", ondelete="CASCADE"),
3682+
nullable=True, # Nullable for global group syncs across all connectors
3683+
)
3684+
3685+
# Status of the group sync attempt
3686+
status: Mapped[PermissionSyncStatus] = mapped_column(
3687+
Enum(PermissionSyncStatus, native_enum=False, index=True)
3688+
)
3689+
3690+
# Counts for tracking progress
3691+
total_users_processed: Mapped[int | None] = mapped_column(Integer, default=0)
3692+
total_groups_processed: Mapped[int | None] = mapped_column(Integer, default=0)
3693+
total_group_memberships_synced: Mapped[int | None] = mapped_column(
3694+
Integer, default=0
3695+
)
3696+
3697+
# Error message if sync fails
3698+
error_message: Mapped[str | None] = mapped_column(Text, default=None)
3699+
3700+
# Timestamps
3701+
time_created: Mapped[datetime.datetime] = mapped_column(
3702+
DateTime(timezone=True),
3703+
server_default=func.now(),
3704+
index=True,
3705+
)
3706+
time_started: Mapped[datetime.datetime | None] = mapped_column(
3707+
DateTime(timezone=True), default=None
3708+
)
3709+
time_finished: Mapped[datetime.datetime | None] = mapped_column(
3710+
DateTime(timezone=True), default=None
3711+
)
3712+
3713+
# Relationships
3714+
connector_credential_pair: Mapped[ConnectorCredentialPair | None] = relationship(
3715+
"ConnectorCredentialPair"
3716+
)
3717+
3718+
__table_args__ = (
3719+
Index(
3720+
"ix_group_sync_attempt_cc_pair_time",
3721+
"connector_credential_pair_id",
3722+
"time_created",
3723+
),
3724+
Index(
3725+
"ix_group_sync_attempt_status_time",
3726+
"status",
3727+
desc("time_finished"),
3728+
),
3729+
)
3730+
3731+
def __repr__(self) -> str:
3732+
return (
3733+
f"<ExternalGroupPermissionSyncAttempt(id={self.id!r}, "
3734+
f"status={self.status!r})>"
3735+
)
3736+
3737+
def is_finished(self) -> bool:
3738+
return self.status.is_terminal()

0 commit comments

Comments
 (0)