Skip to content

Commit 9660326

Browse files
authored
CONDEC-1027: Remove non-existing code files from knowledge graph (#697)
* Fix that feature branches including "masterdata" were recognized as master branch * Enable to filter for unlinked elements (max degree is zero) * Add new metrics and boxplots that shows number of linked Jira issues per code file and for lines of code (LOC) * Speed up adding commits to diff * Use PathSuffixFilter in DiffFormatter to exclude files in diff * Improve recognition of test files * Use tree walk path to get file path * Fix that vis timeline nodes were in the wrong group * Update screenshot of dashboard * Update dependencies in pom.xml
1 parent b1b5da6 commit 9660326

25 files changed

+410
-125
lines changed

.eslintrc_with_warnings.json

Lines changed: 0 additions & 34 deletions
This file was deleted.

doc/features/dashboard.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,12 +45,17 @@ In particular, metrics are shown to answer the following questions:
4545
This dashboard item presents the following **metrics on the knowledge graph data structure** after the graph was filtered with the given filter settings:
4646
- Number of comments per Jira issue
4747
- Number of commits per Jira issue
48+
- Number of linked Jira issues per code file (via commits with a Jira issue key in the commit message), number of unlinked code files that are not reachable from Jira tickets
49+
- Number of lines of code (LOC) per code file and total number of lines of code
4850
- Number of code files and requirements in the project
4951
- Number of rationale elements per origin/documentation location
5052
- Number of comments with and without decision knowledge
5153
- Number of decision knowledge elements per decision knowledge type
5254
- Number of knowledge elements fulfilling and violating the definition of done (DoD)
5355

56+
The metrics involving code files work on those files that are added to the knowledge graph.
57+
The rationale manager can configure which file types (e.g. java, js, ts, ...) to include in the knowledge graph.
58+
5459
![General metrics dashboard item](../screenshots/dashboard_general_metrics.png)
5560

5661
*General metrics dashboard item showing metrics using boxplots and pie charts*
993 Bytes
Loading

pom.xml

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -141,7 +141,7 @@
141141
<dependency>
142142
<groupId>com.atlassian.templaterenderer</groupId>
143143
<artifactId>atlassian-template-renderer-api</artifactId>
144-
<version>5.0.2</version>
144+
<version>5.0.3</version>
145145
<scope>provided</scope>
146146
</dependency>
147147
<dependency>
@@ -185,7 +185,7 @@
185185
<dependency>
186186
<groupId>com.google.code.gson</groupId>
187187
<artifactId>gson</artifactId>
188-
<version>2.9.0</version>
188+
<version>2.9.1</version>
189189
</dependency>
190190
<dependency>
191191
<groupId>com.fasterxml.jackson.core</groupId>
@@ -196,7 +196,7 @@
196196
<dependency>
197197
<groupId>org.slf4j</groupId>
198198
<artifactId>slf4j-api</artifactId>
199-
<version>1.7.36</version>
199+
<version>2.0.0</version>
200200
<scope>provided</scope>
201201
</dependency>
202202
<!-- Active Objects to store model objects via object relational mapping -->
@@ -223,7 +223,7 @@
223223
<dependency>
224224
<groupId>org.mockito</groupId>
225225
<artifactId>mockito-core</artifactId>
226-
<version>4.6.1</version>
226+
<version>4.7.0</version>
227227
<scope>test</scope>
228228
</dependency>
229229
<dependency>
@@ -284,7 +284,7 @@
284284
<dependency>
285285
<groupId>com.github.javaparser</groupId>
286286
<artifactId>javaparser-core</artifactId>
287-
<version>3.24.2</version>
287+
<version>3.24.4</version>
288288
</dependency>
289289
<!-- Webhook -->
290290
<dependency>
@@ -303,7 +303,7 @@
303303
<dependency>
304304
<groupId>org.apache.jena</groupId>
305305
<artifactId>jena-arq</artifactId>
306-
<version>4.5.0</version>
306+
<version>4.6.0</version>
307307
<exclusions>
308308
<exclusion>
309309
<groupId>org.slf4j</groupId>
@@ -445,7 +445,7 @@
445445
<amps.version>8.3.0-128eead6d</amps.version>
446446
<ao.version>4.0.0-m02</ao.version>
447447
<jacoco-version>0.8.8</jacoco-version>
448-
<atlassian.spring.scanner.version>3.0.1</atlassian.spring.scanner.version>
448+
<atlassian.spring.scanner.version>3.0.2</atlassian.spring.scanner.version>
449449
<!-- This key is also used in atlassian-plugin.xml -->
450450
<atlassian.plugin.key>${project.groupId}.${project.artifactId}</atlassian.plugin.key>
451451
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>

src/main/java/de/uhd/ifi/se/decision/management/jira/filtering/FilteringManager.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -340,7 +340,7 @@ public boolean isElementMatchingDecisionGroupFilter(KnowledgeElement element) {
340340
* minDegree and maxDegree in the {@link FilterSettings}.
341341
*/
342342
public boolean isElementMatchingDegreeFilter(KnowledgeElement element) {
343-
if (filterSettings.getMinDegree() > 0) {
343+
if (filterSettings.getMinDegree() > 0 || filterSettings.getMaxDegree() == 0) {
344344
int degree = element.getLinks().size();
345345
return degree >= filterSettings.getMinDegree() && degree <= filterSettings.getMaxDegree();
346346
}
@@ -357,7 +357,7 @@ public boolean isElementMatchingIsTestCodeFilter(KnowledgeElement element) {
357357
if (element.getType() != KnowledgeType.CODE) {
358358
return true;
359359
}
360-
return filterSettings.isTestCodeShown() || !element.getSummary().contains("Test");
360+
return filterSettings.isTestCodeShown() || !ChangedFile.isTestCodeFile(element.getDescription());
361361
}
362362

363363
/**

src/main/java/de/uhd/ifi/se/decision/management/jira/git/CodeFileExtractorAndMaintainer.java

Lines changed: 49 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package de.uhd.ifi.se.decision.management.jira.git;
22

33
import java.util.List;
4+
import java.util.stream.Collectors;
45

56
import org.eclipse.jgit.diff.DiffEntry;
67

@@ -27,51 +28,32 @@
2728
* improvement and maintenance by developers. Developers can manually change
2829
* links. 3) Automatic trace link maintenance during git fetch based on recent
2930
* changes.
31+
*
32+
* @issue Which files should be integrated into the knowledge graph?
33+
* @decision Integrate all Java files into the knowledge graph and link them to
34+
* the respective Jira issues (e.g., work items or requirements)!
3035
*/
3136
public class CodeFileExtractorAndMaintainer {
3237

33-
private String projectKey;
3438
private CodeClassPersistenceManager codeFilePersistenceManager;
39+
private KnowledgeGraph graph;
3540

3641
public CodeFileExtractorAndMaintainer(String projectKey) {
37-
this.projectKey = projectKey;
3842
this.codeFilePersistenceManager = KnowledgePersistenceManager.getInstance(projectKey)
3943
.getCodeClassPersistenceTextManager();
40-
}
41-
42-
/**
43-
* Extracts all code files and the decision knowledge from code comments within
44-
* the {@link Diff}. Links the files to the respective Jira Jira issues (e.g.,
45-
* work items or requirements). Extracting means: 1) Adding code files to the
46-
* {@link CodeClassInDatabase}, 2) adding links to the {@link LinkInDatabase},
47-
* 3) adding code files and links to the {@link KnowledgeGraph}.
48-
*
49-
* @param diff
50-
* {@link Diff} object with added, updated, or deleted
51-
* {@link ChangedFile}s.
52-
*
53-
* @issue Which files should be integrated into the knowledge graph?
54-
* @decision Integrate all Java files into the knowledge graph and link them to
55-
* the respective Jira issues (e.g., work items or requirements)!
56-
*/
57-
public void extractAllChangedFiles(Diff diff) {
58-
KnowledgeGraph graph = KnowledgeGraph.getInstance(projectKey);
59-
for (ChangedFile changedFile : diff.getChangedFiles()) {
60-
if (!changedFile.isCodeFileToExtract()) {
61-
continue;
62-
}
63-
List<DecisionKnowledgeElementInCodeComment> decisionKnowledgeInCodeComments = changedFile
64-
.getRationaleElementsFromCodeComments();
65-
KnowledgeElement source = codeFilePersistenceManager.insertKnowledgeElement(changedFile, null);
66-
graph.updateElement(source);
67-
graph.addElementsNotInDatabase(source, decisionKnowledgeInCodeComments);
68-
}
44+
this.graph = KnowledgeGraph.getInstance(projectKey);
6945
}
7046

7147
/**
7248
* Either inserts, updates, or deletes the code files in the diff in the
7349
* database depending on its change type (see {@link ChangedFile#getDiffEntry()}
74-
* and {@link DiffEntry#getChangeType()}.
50+
* and {@link DiffEntry#getChangeType()}. Also extracts the decision knowledge
51+
* from code comments within the {@link Diff}. Links the files to the respective
52+
* Jira issues (e.g., work items or requirements).
53+
*
54+
* Code extraction means: 1) Adding code files to the
55+
* {@link CodeClassInDatabase}, 2) adding links to the {@link LinkInDatabase},
56+
* 3) adding code files and links to the {@link KnowledgeGraph}.
7557
*
7658
* @param diff
7759
* {@link Diff} object with recently added, updated, or deleted
@@ -105,22 +87,49 @@ public void addUpdateOrDeleteChangedFileInDatabase(ChangedFile changedFile) {
10587
if (!changedFile.isCodeFileToExtract()) {
10688
return;
10789
}
90+
10891
DiffEntry diffEntry = changedFile.getDiffEntry();
10992
switch (diffEntry.getChangeType()) {
110-
case ADD:
111-
codeFilePersistenceManager.insertKnowledgeElement(changedFile, null);
93+
case DELETE:
94+
codeFilePersistenceManager.deleteKnowledgeElement(changedFile, null);
11295
break;
113-
case MODIFY:
114-
// new links could have been added
115-
// same as rename, thus, no break after add to fall through
11696
case RENAME:
11797
codeFilePersistenceManager.updateKnowledgeElement(changedFile, null);
11898
break;
119-
case DELETE:
120-
codeFilePersistenceManager.deleteKnowledgeElement(changedFile, null);
121-
break;
12299
default:
100+
case MODIFY:
101+
// rationale elements in code comments could have been added
102+
// no break after modify to fall through
103+
case ADD:
104+
List<DecisionKnowledgeElementInCodeComment> decisionKnowledgeInCodeComments = changedFile
105+
.getRationaleElementsFromCodeComments();
106+
KnowledgeElement source = codeFilePersistenceManager.insertKnowledgeElement(changedFile, null);
107+
if (!graph.updateElement(source)) {
108+
graph.addVertex(source);
109+
}
110+
graph.addElementsNotInDatabase(source, decisionKnowledgeInCodeComments);
123111
break;
124112
}
125113
}
114+
115+
/**
116+
* Deletes code files from database and the knowledge graph that do not exist in
117+
* the latest version (tagged with HEAD) in the git repository anymore.
118+
*
119+
* @param diff
120+
* for the current version in git.
121+
* @return true if any old code file was deleted.
122+
*/
123+
public boolean deleteOldFiles(Diff diff) {
124+
List<String> fileNamesInDiff = diff.getChangedFiles().stream().map(file -> file.getName())
125+
.collect(Collectors.toList());
126+
boolean isAnyFileDeleted = false;
127+
for (KnowledgeElement codeFileInDatabase : codeFilePersistenceManager.getKnowledgeElements()) {
128+
if (!fileNamesInDiff.contains(codeFileInDatabase.getSummary())) {
129+
codeFilePersistenceManager.deleteKnowledgeElement(codeFileInDatabase, null);
130+
isAnyFileDeleted = true;
131+
}
132+
}
133+
return isAnyFileDeleted;
134+
}
126135
}

src/main/java/de/uhd/ifi/se/decision/management/jira/git/GitClient.java

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,10 @@ public static GitClient getInstance(String projectKey) {
8787
}
8888
if (extractAllCodeKnowledge) {
8989
Diff diff = gitClient.getDiffOfEntireDefaultBranch();
90-
new CodeFileExtractorAndMaintainer(projectKey).extractAllChangedFiles(diff);
90+
CodeFileExtractorAndMaintainer codeFileExtractorAndMaintainer = new CodeFileExtractorAndMaintainer(
91+
projectKey);
92+
codeFileExtractorAndMaintainer.deleteOldFiles(diff);
93+
codeFileExtractorAndMaintainer.maintainChangedFilesInDatabase(diff);
9194
}
9295
return gitClient;
9396
}
@@ -246,4 +249,5 @@ public Diff getDiffOfEntireDefaultBranchFromKnowledgeGraph() {
246249
}
247250
return diffForDefaultBranches;
248251
}
252+
249253
}

0 commit comments

Comments
 (0)