Skip to content

Iris: Add integration test for rewriting and consistency pipeline #10931

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 9 commits into
base: develop
Choose a base branch
from
+344 −154
Open

Iris: Add integration test for rewriting and consistency pipeline #10931

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
7251a29
Add rewriting tests
cremertim May 27, 2025
0c97b08
Add consistency tests as well
cremertim May 27, 2025
9141411
Merge branch 'develop' into chore/iris/add-rewriting-test
cremertim May 30, 2025
34a4e9b
Merge branch 'develop' into chore/iris/add-rewriting-test
cremertim Jun 1, 2025
3300d7a
Refactor tests for better code quality
cremertim Jun 2, 2025
099fc04
Remove var
cremertim Jun 2, 2025
dd85970
Add private constructor and name magic numbers
cremertim Jun 3, 2025
6f979c4
Merge branch 'develop' into chore/iris/add-rewriting-test
cremertim Jun 5, 2025
2c89b6a
Add doc
cremertim Jun 6, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
194 changes: 51 additions & 143 deletions src/test/java/de/tum/cit/aet/artemis/core/connector/IrisRequestMockProvider.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
import de.tum.cit.aet.artemis.iris.service.pyris.dto.chat.textexercise.PyrisTextExerciseChatPipelineExecutionDTO;
import de.tum.cit.aet.artemis.iris.service.pyris.dto.chat.tutorsuggestion.PyrisTutorSuggestionPipelineExecutionDTO;
import de.tum.cit.aet.artemis.iris.service.pyris.dto.competency.PyrisCompetencyExtractionPipelineExecutionDTO;
import de.tum.cit.aet.artemis.iris.service.pyris.dto.consistencyCheck.PyrisConsistencyCheckPipelineExecutionDTO;
import de.tum.cit.aet.artemis.iris.service.pyris.dto.faqingestionwebhook.PyrisWebhookFaqIngestionExecutionDTO;
import de.tum.cit.aet.artemis.iris.service.pyris.dto.lectureingestionwebhook.PyrisWebhookLectureIngestionExecutionDTO;
import de.tum.cit.aet.artemis.iris.service.pyris.dto.rewriting.PyrisRewritingPipelineExecutionDTO;
Expand Down Expand Up @@ -95,17 +96,7 @@ public void reset() throws Exception {
}

public void mockProgrammingExerciseChatResponse(Consumer<PyrisExerciseChatPipelineExecutionDTO> responseConsumer) {
// @formatter:off
mockServer
.expect(ExpectedCount.once(), requestTo(pipelinesApiURL + "/programming-exercise-chat/run"))
.andExpect(method(HttpMethod.POST))
.andRespond(request -> {
var mockRequest = (MockClientHttpRequest) request;
var dto = mapper.readValue(mockRequest.getBodyAsString(), PyrisExerciseChatPipelineExecutionDTO.class);
responseConsumer.accept(dto);
return MockRestResponseCreators.withRawStatus(HttpStatus.ACCEPTED.value()).createResponse(request);
});
// @formatter:on
mockPostRequest("/programming-exercise-chat/run", PyrisExerciseChatPipelineExecutionDTO.class, responseConsumer);
}

public void mockProgrammingExerciseChatResponseExpectingSubmissionId(Consumer<PyrisExerciseChatPipelineExecutionDTO> responseConsumer, long submissionId) {
Expand Down Expand Up @@ -141,184 +132,75 @@ public void mockProgrammingExerciseChatResponseExpectingSubmissionId(Consumer<Py
}

public void mockTextExerciseChatResponse(Consumer<PyrisTextExerciseChatPipelineExecutionDTO> responseConsumer) {
// @formatter:off
mockServer
.expect(ExpectedCount.once(), requestTo(pipelinesApiURL + "/text-exercise-chat/run"))
.andExpect(method(HttpMethod.POST))
.andRespond(request -> {
var mockRequest = (MockClientHttpRequest) request;
var dto = mapper.readValue(mockRequest.getBodyAsString(), PyrisTextExerciseChatPipelineExecutionDTO.class);
responseConsumer.accept(dto);
return MockRestResponseCreators.withRawStatus(HttpStatus.ACCEPTED.value()).createResponse(request);
});
// @formatter:on
mockPostRequest("/text-exercise-chat/run", PyrisTextExerciseChatPipelineExecutionDTO.class, responseConsumer);
}

public void mockLectureChatResponse(Consumer<PyrisLectureChatPipelineExecutionDTO> responseConsumer) {
// @formatter:off
mockServer
.expect(ExpectedCount.once(), requestTo(pipelinesApiURL + "/lecture-chat/run"))
.andExpect(method(HttpMethod.POST))
.andRespond(request -> {
var mockRequest = (MockClientHttpRequest) request;
var dto = mapper.readValue(mockRequest.getBodyAsString(), PyrisLectureChatPipelineExecutionDTO.class);
responseConsumer.accept(dto);
return MockRestResponseCreators.withRawStatus(HttpStatus.ACCEPTED.value()).createResponse(request);
});
// @formatter:on
mockPostRequest("/lecture-chat/run", PyrisLectureChatPipelineExecutionDTO.class, responseConsumer);
}

public void mockTutorSuggestionResponse(Consumer<PyrisTutorSuggestionPipelineExecutionDTO> responseConsumer) {
// @formatter:off
mockServer
.expect(ExpectedCount.once(), requestTo(pipelinesApiURL + "/tutor-suggestion/run"))
.andExpect(method(HttpMethod.POST))
.andRespond(request -> {
var mockRequest = (MockClientHttpRequest) request;
assert mockRequest != null;
var dto = mapper.readValue(mockRequest.getBodyAsString(), PyrisTutorSuggestionPipelineExecutionDTO.class);
responseConsumer.accept(dto);
return MockRestResponseCreators.withRawStatus(HttpStatus.ACCEPTED.value()).createResponse(request);
});
// @formatter:on
mockPostRequest("/tutor-suggestion/run", PyrisTutorSuggestionPipelineExecutionDTO.class, responseConsumer);
}

public void mockRunCompetencyExtractionResponseAnd(Consumer<PyrisCompetencyExtractionPipelineExecutionDTO> executionDTOConsumer) {
// @formatter:off
mockServer
.expect(ExpectedCount.once(), requestTo(pipelinesApiURL + "/competency-extraction/run"))
.andExpect(method(HttpMethod.POST))
.andRespond(request -> {
var mockRequest = (MockClientHttpRequest) request;
var dto = mapper.readValue(mockRequest.getBodyAsString(), PyrisCompetencyExtractionPipelineExecutionDTO.class);
executionDTOConsumer.accept(dto);
return MockRestResponseCreators.withRawStatus(HttpStatus.ACCEPTED.value()).createResponse(request);
});
// @formatter:on
public void mockRunCompetencyExtractionResponseAnd(Consumer<PyrisCompetencyExtractionPipelineExecutionDTO> responseConsumer) {
mockPostRequest("/competency-extraction/run", PyrisCompetencyExtractionPipelineExecutionDTO.class, responseConsumer);
}

public void mockRunRewritingResponseAnd(Consumer<PyrisRewritingPipelineExecutionDTO> executionDTOConsumer) {
// @formatter:off
mockServer
.expect(ExpectedCount.once(), requestTo(pipelinesApiURL + "/rewriting/faq/run"))
.andExpect(method(HttpMethod.POST))
.andRespond(request -> {
var mockRequest = (MockClientHttpRequest) request;
var dto = mapper.readValue(mockRequest.getBodyAsString(), PyrisRewritingPipelineExecutionDTO.class);
executionDTOConsumer.accept(dto);
return MockRestResponseCreators.withRawStatus(HttpStatus.ACCEPTED.value()).createResponse(request);
});
// @formatter:on
public void mockRewritingPipelineResponse(Consumer<PyrisRewritingPipelineExecutionDTO> responseConsumer) {
mockPostRequest("/rewriting/run", PyrisRewritingPipelineExecutionDTO.class, responseConsumer);
}

public void mockProgrammingConsistencyCheckResponse(Consumer<PyrisConsistencyCheckPipelineExecutionDTO> responseConsumer) {
mockPostRequest("/inconsistency-check/run", PyrisConsistencyCheckPipelineExecutionDTO.class, responseConsumer);
}

public void mockIngestionWebhookRunResponse(Consumer<PyrisWebhookLectureIngestionExecutionDTO> responseConsumer) {
mockServer.expect(ExpectedCount.once(), requestTo(webhooksApiURL + "/lectures/ingest")).andExpect(method(HttpMethod.POST)).andRespond(request -> {
var mockRequest = (MockClientHttpRequest) request;
var dto = mapper.readValue(mockRequest.getBodyAsString(), PyrisWebhookLectureIngestionExecutionDTO.class);
responseConsumer.accept(dto);
return MockRestResponseCreators.withRawStatus(HttpStatus.ACCEPTED.value()).createResponse(request);
});
mockWebhookPost("/lectures/ingest", PyrisWebhookLectureIngestionExecutionDTO.class, responseConsumer);
}

public void mockTranscriptionIngestionWebhookRunResponse(Consumer<PyrisWebhookTranscriptionIngestionExecutionDTO> responseConsumer) {
mockServer.expect(ExpectedCount.once(), requestTo(webhooksApiURL + "/transcriptions/ingest")).andExpect(method(HttpMethod.POST)).andRespond(request -> {
var mockRequest = (MockClientHttpRequest) request;
var dto = mapper.readValue(mockRequest.getBodyAsString(), PyrisWebhookTranscriptionIngestionExecutionDTO.class);
responseConsumer.accept(dto);
return MockRestResponseCreators.withRawStatus(HttpStatus.ACCEPTED.value()).createResponse(request);
});
mockWebhookPost("/transcriptions/ingest", PyrisWebhookTranscriptionIngestionExecutionDTO.class, responseConsumer);
}

public void mockTranscriptionDeletionWebhookRunResponse(Consumer<PyrisWebhookTranscriptionDeletionExecutionDTO> responseConsumer) {
mockServer.expect(ExpectedCount.once(), requestTo(webhooksApiURL + "/transcriptions/delete")).andExpect(method(HttpMethod.POST)).andRespond(request -> {
var mockRequest = (MockClientHttpRequest) request;
var dto = mapper.readValue(mockRequest.getBodyAsString(), PyrisWebhookTranscriptionDeletionExecutionDTO.class);
responseConsumer.accept(dto);
return MockRestResponseCreators.withRawStatus(HttpStatus.ACCEPTED.value()).createResponse(request);
});
mockWebhookPost("/transcriptions/delete", PyrisWebhookTranscriptionDeletionExecutionDTO.class, responseConsumer);
}

public void mockFaqIngestionWebhookRunResponse(Consumer<PyrisWebhookFaqIngestionExecutionDTO> responseConsumer) {
mockServer.expect(ExpectedCount.once(), requestTo(webhooksApiURL + "/faqs/ingest")).andExpect(method(HttpMethod.POST)).andRespond(request -> {
var mockRequest = (MockClientHttpRequest) request;
var dto = mapper.readValue(mockRequest.getBodyAsString(), PyrisWebhookFaqIngestionExecutionDTO.class);
responseConsumer.accept(dto);
return MockRestResponseCreators.withRawStatus(HttpStatus.ACCEPTED.value()).createResponse(request);
});
mockWebhookPost("/faqs/ingest", PyrisWebhookFaqIngestionExecutionDTO.class, responseConsumer);
}

public void mockDeletionWebhookRunResponse(Consumer<PyrisWebhookLectureIngestionExecutionDTO> responseConsumer) {
mockServer.expect(ExpectedCount.once(), requestTo(webhooksApiURL + "/lectures/delete")).andExpect(method(HttpMethod.POST)).andRespond(request -> {
var mockRequest = (MockClientHttpRequest) request;
var dto = mapper.readValue(mockRequest.getBodyAsString(), PyrisWebhookLectureIngestionExecutionDTO.class);
responseConsumer.accept(dto);
return MockRestResponseCreators.withRawStatus(HttpStatus.ACCEPTED.value()).createResponse(request);
});
mockWebhookPost("/lectures/delete", PyrisWebhookLectureIngestionExecutionDTO.class, responseConsumer);
}

public void mockFaqDeletionWebhookRunResponse(Consumer<PyrisWebhookFaqIngestionExecutionDTO> responseConsumer) {
mockServer.expect(ExpectedCount.once(), requestTo(webhooksApiURL + "/faqs/delete")).andExpect(method(HttpMethod.POST)).andRespond(request -> {
var mockRequest = (MockClientHttpRequest) request;
var dto = mapper.readValue(mockRequest.getBodyAsString(), PyrisWebhookFaqIngestionExecutionDTO.class);
responseConsumer.accept(dto);
return MockRestResponseCreators.withRawStatus(HttpStatus.ACCEPTED.value()).createResponse(request);
});
mockWebhookPost("/faqs/delete", PyrisWebhookFaqIngestionExecutionDTO.class, responseConsumer);
}

public void mockBuildFailedRunResponse(Consumer<PyrisExerciseChatPipelineExecutionDTO> responseConsumer) {
mockServer.expect(ExpectedCount.max(2), requestTo(pipelinesApiURL + "/programming-exercise-chat/run?event=build_failed")).andExpect(method(HttpMethod.POST))
.andRespond(request -> {
var mockRequest = (MockClientHttpRequest) request;
var dto = mapper.readValue(mockRequest.getBodyAsString(), PyrisExerciseChatPipelineExecutionDTO.class);
responseConsumer.accept(dto);
return MockRestResponseCreators.withRawStatus(HttpStatus.ACCEPTED.value()).createResponse(request);
});
mockPostRequest("/programming-exercise-chat/run?event=build_failed", PyrisExerciseChatPipelineExecutionDTO.class, responseConsumer, ExpectedCount.max(2));
}

public void mockProgressStalledEventRunResponse(Consumer<PyrisCourseChatPipelineExecutionDTO> responseConsumer) {
mockServer.expect(ExpectedCount.max(2), requestTo(pipelinesApiURL + "/programming-exercise-chat/run?event=progress_stalled")).andExpect(method(HttpMethod.POST))
.andRespond(request -> {
var mockRequest = (MockClientHttpRequest) request;
var dto = mapper.readValue(mockRequest.getBodyAsString(), PyrisCourseChatPipelineExecutionDTO.class);
responseConsumer.accept(dto);
return MockRestResponseCreators.withRawStatus(HttpStatus.ACCEPTED.value()).createResponse(request);
});
mockPostRequest("/programming-exercise-chat/run?event=progress_stalled", PyrisCourseChatPipelineExecutionDTO.class, responseConsumer, ExpectedCount.max(2));
}

public void mockJolEventRunResponse(Consumer<PyrisCourseChatPipelineExecutionDTO> responseConsumer) {
mockServer.expect(ExpectedCount.once(), requestTo(pipelinesApiURL + "/course-chat/run?event=jol")).andExpect(method(HttpMethod.POST)).andRespond(request -> {
var mockRequest = (MockClientHttpRequest) request;
var dto = mapper.readValue(mockRequest.getBodyAsString(), PyrisCourseChatPipelineExecutionDTO.class);
responseConsumer.accept(dto);
return MockRestResponseCreators.withRawStatus(HttpStatus.ACCEPTED.value()).createResponse(request);
});
mockPostRequest("/course-chat/run?event=jol", PyrisCourseChatPipelineExecutionDTO.class, responseConsumer);
}

public void mockRunError(int httpStatus) {
// @formatter:off
mockServer
.expect(ExpectedCount.once(), requestTo(pipelinesApiURL + "/programming-exercise-chat/run"))
.andExpect(method(HttpMethod.POST))
.andRespond(withStatus(HttpStatus.valueOf(httpStatus)));
// @formatter:on
mockPostError(pipelinesApiURL.toString(), "/programming-exercise-chat/run", httpStatus);
}

public void mockIngestionWebhookRunError(int httpStatus) {
// @formatter:off
mockServer
.expect(ExpectedCount.once(), requestTo(webhooksApiURL + "/lectures/ingest"))
.andExpect(method(HttpMethod.POST))
.andRespond(withStatus(HttpStatus.valueOf(httpStatus)));
// @formatter:on
mockPostError(webhooksApiURL.toString(), "/lectures/ingest", httpStatus);
}

public void mockDeletionWebhookRunError(int httpStatus) {
// @formatter:off
mockServer
.expect(ExpectedCount.once(), requestTo(webhooksApiURL + "/lectures/delete"))
.andExpect(method(HttpMethod.POST))
.andRespond(withStatus(HttpStatus.valueOf(httpStatus)));
// @formatter:on
mockPostError(webhooksApiURL.toString(), "/lectures/delete", httpStatus);
}

public void mockVariantsResponse(IrisSubSettingsType feature) throws JsonProcessingException {
Expand Down Expand Up @@ -348,6 +230,32 @@ public void mockStatusResponses() throws JsonProcessingException {
// @formatter:on
}

private void mockPostError(String baseUrl, String path, int httpStatus) {
mockServer.expect(ExpectedCount.once(), requestTo(baseUrl + path)).andExpect(method(HttpMethod.POST)).andRespond(withStatus(HttpStatus.valueOf(httpStatus)));
}

private <T> void mockPostRequest(String path, Class<T> dtoClass, Consumer<T> responseConsumer) {
mockPostRequest(path, dtoClass, responseConsumer, ExpectedCount.once());
}

private <T> void mockPostRequest(String path, Class<T> dtoClass, Consumer<T> responseConsumer, ExpectedCount count) {
mockServer.expect(count, requestTo(pipelinesApiURL + path)).andExpect(method(HttpMethod.POST)).andRespond(request -> {
var mockRequest = (MockClientHttpRequest) request;
var dto = mapper.readValue(mockRequest.getBodyAsString(), dtoClass);
responseConsumer.accept(dto);
return MockRestResponseCreators.withRawStatus(HttpStatus.ACCEPTED.value()).createResponse(request);
});
}

private <T> void mockWebhookPost(String path, Class<T> dtoClass, Consumer<T> responseConsumer) {
mockServer.expect(ExpectedCount.once(), requestTo(webhooksApiURL + path)).andExpect(method(HttpMethod.POST)).andRespond(request -> {
var mockRequest = (MockClientHttpRequest) request;
var dto = mapper.readValue(mockRequest.getBodyAsString(), dtoClass);
responseConsumer.accept(dto);
return MockRestResponseCreators.withRawStatus(HttpStatus.ACCEPTED.value()).createResponse(request);
});
}

/**
* Mocks a get model error from the Pyris models endpoint
*/
Expand Down
9 changes: 1 addition & 8 deletions src/test/java/de/tum/cit/aet/artemis/iris/IrisChatTokenTrackingIntegrationTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package de.tum.cit.aet.artemis.iris;

import static de.tum.cit.aet.artemis.iris.utils.IrisLLMMock.getMockLLMCosts;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatNoException;
import static org.awaitility.Awaitility.await;
Expand Down Expand Up @@ -205,14 +206,6 @@ void testTokenTrackingExerciseChatWithPipelineFail() throws Exception {
}
}

private List<LLMRequest> getMockLLMCosts() {
List<LLMRequest> costs = new ArrayList<>();
for (int i = 0; i < 5; i++) {
costs.add(new LLMRequest("test-llm", i * 10 + 5, i * 0.5f, i * 3 + 5, i * 0.12f, "IRIS_CHAT_EXERCISE_MESSAGE"));
}
return costs;
}

private IrisMessage createDefaultMockMessage(IrisSession irisSession) {
var messageToSend = irisSession.newMessage();
messageToSend.addContent(createMockTextContent(), createMockTextContent(), createMockTextContent());
Expand Down
Loading
Loading