Skip to content

Commit 98eb58a

Browse files
Subash-MohanSubash-Mohan
committed
add sharepoint integration tests
1 parent bf2ac11 commit 98eb58a

File tree

5 files changed

+358
-7
lines changed

5 files changed

+358
-7
lines changed

‎backend/ee/onyx/external_permissions/sharepoint/utils.py‎

Lines changed: 37 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@
1212

1313
from ee.onyx.db.external_perm import ExternalUserGroup
1414
from onyx.access.models import ExternalAccess
15+
from onyx.access.utils import build_ext_group_name_for_onyx
16+
from onyx.configs.constants import DocumentSource
1517
from onyx.utils.logger import setup_logger
1618

1719
logger = setup_logger()
@@ -475,6 +477,7 @@ def get_external_access_from_sharepoint(
475477
graph_client: GraphClient,
476478
drive_name: str,
477479
drive_item: DriveItem,
480+
add_prefix: bool = False,
478481
) -> ExternalAccess:
479482
"""
480483
Get external access information from SharePoint.
@@ -510,6 +513,19 @@ def add_user_and_group_to_sets(
510513
) -> None:
511514
nonlocal user_emails, groups
512515
for assignment in role_assignments:
516+
logger.info(f"Assignment: {assignment.to_json()}")
517+
if assignment.role_definition_bindings:
518+
role_names = [
519+
role_definition_binding.name
520+
for role_definition_binding in assignment.role_definition_bindings
521+
]
522+
523+
# Skip if the role is only Limited Access, because this is not a actual permission its a travel through permission
524+
if role_names == ["Limited Access"]:
525+
logger.info(
526+
"Skipping assignment because it has only Limited Access role"
527+
)
528+
continue
513529
if assignment.member:
514530
member = assignment.member
515531
if hasattr(member, "principal_type"):
@@ -533,7 +549,7 @@ def add_user_and_group_to_sets(
533549
)
534550

535551
_sleep_and_retry(
536-
item.role_assignments.expand(["Member"]).get_all(
552+
item.role_assignments.expand(["Member", "RoleDefinitionBindings"]).get_all(
537553
page_loaded=add_user_and_group_to_sets
538554
),
539555
"get_external_access_from_sharepoint",
@@ -542,6 +558,10 @@ def add_user_and_group_to_sets(
542558
client_context, graph_client, groups
543559
)
544560
for group_name, _ in groups_and_members.items():
561+
if add_prefix:
562+
group_name = build_ext_group_name_for_onyx(
563+
group_name, DocumentSource.SHAREPOINT
564+
)
545565
group_ids.add(group_name.lower())
546566

547567
return ExternalAccess(
@@ -565,6 +585,19 @@ def get_sharepoint_external_groups(
565585
def add_group_to_sets(role_assignments: RoleAssignmentCollection) -> None:
566586
nonlocal groups
567587
for assignment in role_assignments:
588+
logger.info(f"Assignment: {assignment.to_json()}")
589+
if assignment.role_definition_bindings:
590+
role_names = [
591+
role_definition_binding.name
592+
for role_definition_binding in assignment.role_definition_bindings
593+
]
594+
595+
# Skip if the role is only Limited Access, because this is not a actual permission its a travel through permission
596+
if role_names == ["Limited Access"]:
597+
logger.info(
598+
"Skipping assignment because it has only Limited Access role"
599+
)
600+
continue
568601
if assignment.member:
569602
member = assignment.member
570603
if hasattr(member, "principal_type"):
@@ -581,9 +614,9 @@ def add_group_to_sets(role_assignments: RoleAssignmentCollection) -> None:
581614
)
582615

583616
_sleep_and_retry(
584-
client_context.web.role_assignments.expand(["Member"]).get_all(
585-
page_loaded=add_group_to_sets
586-
),
617+
client_context.web.role_assignments.expand(
618+
["Member", "RoleDefinitionBindings"]
619+
).get_all(page_loaded=add_group_to_sets),
587620
"get_sharepoint_external_groups",
588621
)
589622
groups_and_members: dict[str, set[str]] = _get_groups_and_members_recursively(

‎backend/onyx/connectors/sharepoint/connector.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -131,7 +131,7 @@ def _convert_driveitem_to_document_with_permissions(
131131
raise ValueError("DriveItem ID is required")
132132
if include_permissions and ctx is None:
133133
raise ValueError("ClientContext is required for permissions")
134-
134+
logger.info(f"Getting drive item content for drive item: {driveitem.name}")
135135
content = driveitem.get_content().execute_query().value
136136

137137
if content is None:
@@ -154,7 +154,7 @@ def _convert_driveitem_to_document_with_permissions(
154154
if include_permissions and ctx is not None:
155155
try:
156156
external_access = get_sharepoint_external_access(
157-
driveitem, drive_name, ctx, graph_client
157+
driveitem, drive_name, ctx, graph_client, True
158158
)
159159
except Exception as e:
160160
logger.warning(

‎backend/onyx/connectors/sharepoint/utils.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ def get_sharepoint_external_access(
1515
drive_name: str,
1616
ctx: ClientContext,
1717
graph_client: GraphClient,
18+
add_prefix: bool = False,
1819
) -> ExternalAccess:
1920
if drive_item.id is None:
2021
raise ValueError("DriveItem ID is required")
@@ -30,7 +31,7 @@ def noop_fallback(*args: Any, **kwargs: Any) -> ExternalAccess:
3031
)
3132

3233
external_access = get_external_access_func(
33-
ctx, graph_client, drive_name, drive_item
34+
ctx, graph_client, drive_name, drive_item, add_prefix
3435
)
3536

3637
return external_access

‎backend/tests/integration/connector_job_tests/sharepoint/conftest.py‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
import os
2+
from collections.abc import Generator
3+
from datetime import datetime
4+
from datetime import timezone
5+
6+
import pytest
7+
8+
from onyx.configs.constants import DocumentSource
9+
from onyx.connectors.models import InputType
10+
from onyx.connectors.sharepoint.connector import SharepointAuthMethod
11+
from onyx.db.enums import AccessType
12+
from tests.integration.common_utils.managers.cc_pair import CCPairManager
13+
from tests.integration.common_utils.managers.connector import ConnectorManager
14+
from tests.integration.common_utils.managers.credential import CredentialManager
15+
from tests.integration.common_utils.managers.llm_provider import LLMProviderManager
16+
from tests.integration.common_utils.managers.user import UserManager
17+
from tests.integration.common_utils.reset import reset_all
18+
from tests.integration.common_utils.test_models import DATestCCPair
19+
from tests.integration.common_utils.test_models import DATestConnector
20+
from tests.integration.common_utils.test_models import DATestCredential
21+
from tests.integration.common_utils.test_models import DATestUser
22+
23+
SharepointTestEnvSetupTuple = tuple[
24+
DATestUser, # admin_user
25+
DATestUser, # regular_user_1
26+
DATestUser, # regular_user_2
27+
DATestCredential,
28+
DATestConnector,
29+
DATestCCPair,
30+
]
31+
32+
33+
@pytest.fixture(scope="module")
34+
def sharepoint_test_env_setup() -> Generator[SharepointTestEnvSetupTuple]:
35+
# Reset all data before running the test
36+
reset_all()
37+
# Required environment variables for SharePoint certificate authentication
38+
sp_client_id = os.environ["PERM_SYNC_SHAREPOINT_CLIENT_ID"]
39+
sp_private_key = os.environ[
40+
"PERM_SYNC_SHAREPOINT_PRIVATE_KEY"
41+
] # Base64 encoded PFX data
42+
sp_certificate_password = os.environ["PERM_SYNC_SHAREPOINT_CERTIFICATE_PASSWORD"]
43+
sp_directory_id = os.environ["PERM_SYNC_SHAREPOINT_DIRECTORY_ID"]
44+
sp_tenant_domain = os.environ["PERM_SYNC_SHAREPOINT_TENANT_DOMAIN"]
45+
sharepoint_sites = "https://danswerai.sharepoint.com/sites/Permisisonsync"
46+
admin_email = "admin@onyx.app"
47+
user1_email = "subash@onyx.app"
48+
user2_email = "raunak@onyx.app"
49+
50+
# Certificate-based credentials
51+
credentials = {
52+
"authentication_method": SharepointAuthMethod.CERTIFICATE.value,
53+
"sp_client_id": sp_client_id,
54+
"sp_private_key": sp_private_key,
55+
"sp_certificate_password": sp_certificate_password,
56+
"sp_directory_id": sp_directory_id,
57+
"sp_tenant_domain": sp_tenant_domain,
58+
}
59+
60+
# Create users
61+
admin_user: DATestUser = UserManager.create(email=admin_email)
62+
regular_user_1: DATestUser = UserManager.create(email=user1_email)
63+
regular_user_2: DATestUser = UserManager.create(email=user2_email)
64+
65+
# Create LLM provider for search functionality
66+
LLMProviderManager.create(user_performing_action=admin_user)
67+
68+
# Create credential
69+
credential: DATestCredential = CredentialManager.create(
70+
source=DocumentSource.SHAREPOINT,
71+
credential_json=credentials,
72+
user_performing_action=admin_user,
73+
)
74+
75+
# Create connector with SharePoint-specific configuration
76+
connector: DATestConnector = ConnectorManager.create(
77+
name="SharePoint Test",
78+
input_type=InputType.POLL,
79+
source=DocumentSource.SHAREPOINT,
80+
connector_specific_config={
81+
"sites": sharepoint_sites.split(","),
82+
},
83+
access_type=AccessType.SYNC, # Enable permission sync
84+
user_performing_action=admin_user,
85+
)
86+
87+
# Create CC pair with permission sync enabled
88+
cc_pair: DATestCCPair = CCPairManager.create(
89+
credential_id=credential.id,
90+
connector_id=connector.id,
91+
access_type=AccessType.SYNC, # Enable permission sync
92+
user_performing_action=admin_user,
93+
)
94+
95+
# Wait for both indexing and permission sync to complete
96+
before = datetime.now(tz=timezone.utc)
97+
CCPairManager.wait_for_indexing_completion(
98+
cc_pair=cc_pair,
99+
after=before,
100+
user_performing_action=admin_user,
101+
timeout=float("inf"),
102+
)
103+
104+
# Wait for permission sync completion specifically
105+
CCPairManager.wait_for_sync(
106+
cc_pair=cc_pair,
107+
after=before,
108+
user_performing_action=admin_user,
109+
timeout=float("inf"),
110+
)
111+
112+
yield admin_user, regular_user_1, regular_user_2, credential, connector, cc_pair

0 commit comments

Comments
 (0)