Skip to content

Commit c1115e8

Browse files
authored
Merge pull request #37 from balmaster/feature/22
Feature/22
2 parents 0dae87d + dd84ce7 commit c1115e8

19 files changed

+363
-60
lines changed

src/main/groovy/repo/build/ComponentDependencyGraph.groovy

Lines changed: 34 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,12 @@ package repo.build
22

33
import groovy.transform.CompileStatic
44
import org.jgrapht.DirectedGraph
5+
import org.jgrapht.alg.ConnectivityInspector
56
import org.jgrapht.alg.CycleDetector
67
import org.jgrapht.graph.DefaultDirectedGraph
78
import org.jgrapht.graph.DefaultEdge
89
import org.jgrapht.traverse.TopologicalOrderIterator
10+
import repo.build.maven.MavenArtifact
911
import repo.build.maven.MavenArtifactRef
1012
import repo.build.maven.MavenComponent
1113

@@ -15,13 +17,14 @@ import repo.build.maven.MavenComponent
1517
class ComponentDependencyGraph {
1618
private final Map<MavenArtifactRef, MavenComponent> componentsMap
1719
private final DirectedGraph<MavenComponent, DefaultEdge> graph = new DefaultDirectedGraph<>(DefaultEdge.class)
18-
private final Map<MavenComponent, Set<MavenComponent>> cycleRefs = new HashMap<>()
20+
private final CycleDetector<MavenComponent, DefaultEdge> cycleDetector = new CycleDetector<>(graph)
1921

2022
ComponentDependencyGraph(Map<MavenArtifactRef, MavenComponent> componentsMap) {
2123
this.componentsMap = componentsMap
2224
}
2325

24-
static ComponentDependencyGraph build(Map<MavenArtifactRef, MavenComponent> componentsMap) {
26+
static ComponentDependencyGraph build(Collection<MavenComponent> components) {
27+
def componentsMap = getModuleToComponentMap(components)
2528
ComponentDependencyGraph result = new ComponentDependencyGraph(componentsMap)
2629
for (MavenComponent c : componentsMap.values()) {
2730
result.add(c)
@@ -52,24 +55,24 @@ class ComponentDependencyGraph {
5255

5356
private void addComponentRef(MavenComponent component, MavenArtifactRef ref) {
5457
MavenComponent refComponent = componentsMap.get(ref)
55-
if (refComponent) {
58+
if (refComponent && !component.equals(refComponent)) {
5659
add(refComponent)
57-
DefaultEdge e = graph.addEdge(component, refComponent)
58-
if (hasCycles()) {
59-
graph.removeEdge(e)
60-
if (!cycleRefs.containsKey(component)) {
61-
cycleRefs.put(component, new HashSet<MavenComponent>())
62-
}
63-
cycleRefs.get(component).add(refComponent)
64-
}
60+
graph.addEdge(component, refComponent)
6561
}
6662
}
6763

6864
boolean hasCycles() {
69-
CycleDetector<MavenComponent, DefaultEdge> cycleDetector = new CycleDetector<>(graph)
7065
return cycleDetector.detectCycles()
7166
}
7267

68+
Set<MavenComponent> findCycles() {
69+
return cycleDetector.findCycles()
70+
}
71+
72+
Set<MavenComponent> findCycles(MavenComponent v) {
73+
return cycleDetector.findCyclesContainingVertex(v)
74+
}
75+
7376
List<MavenComponent> sort() {
7477
TopologicalOrderIterator<MavenComponent, DefaultEdge> i = new TopologicalOrderIterator<>(graph)
7578
List<MavenComponent> items = new ArrayList<>()
@@ -80,8 +83,25 @@ class ComponentDependencyGraph {
8083
return items
8184
}
8285

83-
Map<MavenComponent, Set<MavenComponent>> getCycleRefs() {
84-
return cycleRefs
86+
List<MavenComponent> getIncoming(MavenComponent component) {
87+
return graph.incomingEdgesOf(component)
88+
.collect { graph.getEdgeSource(it) }
8589
}
8690

91+
List<MavenComponent> getOutgoing(MavenComponent component) {
92+
return graph.outgoingEdgesOf(component)
93+
.collect { graph.getEdgeSource(it) }
94+
}
95+
96+
@CompileStatic
97+
static Map<MavenArtifactRef, MavenComponent> getModuleToComponentMap(Collection<MavenComponent> components) {
98+
Map<MavenArtifactRef, MavenComponent> result = new HashMap<>()
99+
for (MavenComponent c : components) {
100+
for (MavenArtifact m : c.getModules()) {
101+
// map all component modules into host component
102+
result.put(new MavenArtifactRef(m.getGroupId(), m.getArtifactId()), c)
103+
}
104+
}
105+
return result
106+
}
87107
}

src/main/groovy/repo/build/ExecuteProcess.groovy

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ class ExecuteProcess {
4242

4343
def exitValue = process.waitFor()
4444
// for read process output
45-
Thread.sleep(100)
45+
Thread.sleep(100L)
4646

4747
def outStream = new AnsiOutputStream(System.out)
4848
if (bufferStream.size() > 0) {
@@ -83,7 +83,7 @@ class ExecuteProcess {
8383

8484
def exitValue = process.waitFor()
8585
// for read process output
86-
Thread.sleep(100)
86+
Thread.sleep(100L)
8787

8888
if (checkErrorCode && exitValue != 0) {
8989
throw new RepoBuildException("name '$cmd' has exit code $exitValue");

src/main/groovy/repo/build/Maven.groovy

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,12 @@ class Maven {
4747
}
4848
)
4949
}
50-
}
5150

51+
@CompileStatic
52+
static void execute(ActionContext context,
53+
File pomFile,
54+
List<String> goals) {
55+
execute(context, pomFile, goals, [:])
56+
}
5257

58+
}

src/main/groovy/repo/build/MavenFeature.groovy

Lines changed: 17 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package repo.build
33
import groovy.transform.CompileStatic
44
import org.apache.log4j.Logger
55
import org.apache.maven.shared.invoker.InvocationRequest
6+
import repo.build.maven.Build
67
import repo.build.maven.MavenArtifact
78
import repo.build.maven.MavenArtifactRef
89
import repo.build.maven.MavenComponent
@@ -267,10 +268,8 @@ class MavenFeature {
267268
def context = parentContext.newChild(ACTION_UPDATE_VERSIONS)
268269
Pom.generateXml(context, featureBranch, new File(context.env.basedir, 'pom.xml'))
269270

270-
// получаем компоненты и зависимости
271-
def componentsMap = getModuleToComponentMap(context)
272271
// формируем граф зависимостей
273-
List<MavenComponent> sortedComponents = sortComponents(componentsMap)
272+
List<MavenComponent> sortedComponents = sortComponents(getComponents(context))
274273
context.writeOut("sort component by dependency tree\n")
275274
sortedComponents.each {
276275
context.writeOut(it.groupId + ':' + it.artifactId + '\n')
@@ -342,28 +341,11 @@ class MavenFeature {
342341

343342

344343
@CompileStatic
345-
static List<MavenComponent> sortComponents(Map<MavenArtifactRef, MavenComponent> modulesMap) {
346-
def graph = ComponentDependencyGraph.build(modulesMap)
344+
static List<MavenComponent> sortComponents(Collection<MavenComponent> components) {
345+
def graph = ComponentDependencyGraph.build(components)
347346
return graph.sort()
348347
}
349348

350-
@CompileStatic
351-
static Map<MavenArtifactRef, MavenComponent> getModuleToComponentMap(ActionContext context) {
352-
return getModuleToComponentMap(getComponents(context))
353-
}
354-
355-
@CompileStatic
356-
static Map<MavenArtifactRef, MavenComponent> getModuleToComponentMap(List<MavenComponent> components) {
357-
Map<MavenArtifactRef, MavenComponent> result = new HashMap<>()
358-
for (MavenComponent c : components) {
359-
for (MavenArtifact m : c.getModules()) {
360-
// map all component modules into host component
361-
result.put(new MavenArtifactRef(m.getGroupId(), m.getArtifactId()), c)
362-
}
363-
}
364-
return result
365-
}
366-
367349
static List<MavenComponent> getComponents(ActionContext context) {
368350
List<MavenComponent> result = new ArrayList<>()
369351
forEachWithPom(context, { ActionContext actionContext, project ->
@@ -480,11 +462,10 @@ class MavenFeature {
480462
def context = parentContext.newChild(ACTION_BUILD_PARENTS)
481463

482464
// получаем компоненты и зависимости
483-
def componentsMap = getModuleToComponentMap(
484-
getParentComponents(getComponents(context)))
465+
def components = getParentComponents(getComponents(context))
485466
// формируем граф зависимостей
486-
List<MavenComponent> sortedComponents = sortComponents(componentsMap)
487-
context.writeOut("sort component by dependency tree\n")
467+
List<MavenComponent> sortedComponents = sortComponents(components)
468+
context.writeOut("sort parents by dependency tree\n")
488469
sortedComponents.each {
489470
context.writeOut(it.groupId + ':' + it.artifactId + '\n')
490471
}
@@ -495,4 +476,14 @@ class MavenFeature {
495476
}
496477
}
497478

479+
static final String ACTION_BUILD_PARALLEL = 'mavenFeatureBuildParallel'
480+
481+
@CompileStatic
482+
static boolean buildParallel(ActionContext parentContext) {
483+
def context = parentContext.newChild(ACTION_BUILD_PARALLEL)
484+
context.withCloseable {
485+
def build = new Build(context, getComponents(context))
486+
return build.execute(context)
487+
}
488+
}
498489
}

src/main/groovy/repo/build/RepoBuild.groovy

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import repo.build.command.GrepCommand
1414
import repo.build.command.InitCommand
1515
import repo.build.command.MergeAbortCommand
1616
import repo.build.command.MvnBuildCommand
17+
import repo.build.command.MvnBuildParallelCommand
1718
import repo.build.command.PrepareMergeCommand
1819
import repo.build.command.PushFeatureCommand
1920
import repo.build.command.PushManifestCommand
@@ -54,6 +55,7 @@ class RepoBuild {
5455
commandRegistry.registerCommand(new InitCommand())
5556
commandRegistry.registerCommand(new MergeAbortCommand())
5657
commandRegistry.registerCommand(new MvnBuildCommand())
58+
commandRegistry.registerCommand(new MvnBuildParallelCommand())
5759
commandRegistry.registerCommand(new PrepareMergeCommand())
5860
commandRegistry.registerCommand(new PushFeatureCommand())
5961
commandRegistry.registerCommand(new PushManifestCommand())

src/main/groovy/repo/build/RepoManifest.groovy

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ class RepoManifest {
6161
}
6262
}
6363
catch (Exception e) {
64-
def componentError = new RepoBuildException("Project ${project.@path} error ${e.message}", e)
64+
def componentError = new RepoBuildException("Component ${project.@path} error ${e.message}", e)
6565
if (actionContext.options.hasFae()) {
6666
actionContext.addError(componentError)
6767
} else {
Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
package repo.build.command
2+
3+
import repo.build.*
4+
5+
class MvnBuildParallelCommand extends AbstractCommand {
6+
MvnBuildParallelCommand() {
7+
super('mvn-build-parallel', 'Execute mvn clean install for topology sorted compoents of project in parallel')
8+
}
9+
10+
public static final String ACTION_EXECUTE = 'mvnBuildParallelExecute'
11+
12+
void execute(RepoEnv env, CliOptions options) {
13+
def context = new ActionContext(env, ACTION_EXECUTE, options, new DefaultActionHandler())
14+
context.withCloseable {
15+
MavenFeature.buildParallel(context)
16+
}
17+
}
18+
}
Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
package repo.build.maven
2+
3+
import groovy.transform.CompileStatic
4+
import repo.build.ActionContext
5+
import repo.build.ComponentDependencyGraph
6+
import repo.build.Maven
7+
import repo.build.RepoBuildException
8+
9+
import java.util.concurrent.ConcurrentHashMap
10+
import java.util.concurrent.ConcurrentMap
11+
import java.util.concurrent.Executors
12+
import java.util.concurrent.LinkedBlockingQueue
13+
import java.util.concurrent.ThreadPoolExecutor
14+
import java.util.concurrent.TimeUnit
15+
16+
/**
17+
* @author Markelov Ruslan markelov@jet.msk.su
18+
*/
19+
//@CompileStatic
20+
class Build {
21+
Collection<MavenComponent> components
22+
ActionContext context
23+
24+
Build(ActionContext context, Collection<MavenComponent> components) {
25+
this.context = context
26+
this.components = components
27+
}
28+
29+
boolean execute(ActionContext context) {
30+
// check circular dependencies
31+
def graph = ComponentDependencyGraph.build(components)
32+
if (graph.hasCycles()) {
33+
for (def component : graph.findCycles()) {
34+
def componentCycle = graph.findCycles(component).collect { it.path }
35+
def error = new RepoBuildException("Component ${component.@path} has circular refs in cycle ${componentCycle}")
36+
context.addError(error)
37+
}
38+
throw new RepoBuildException("project has circular dependencies")
39+
}
40+
41+
Map<MavenArtifactRef, MavenComponent> componentMap = components.collectEntries {
42+
[new MavenArtifactRef(it), it]
43+
}
44+
45+
Map<MavenArtifactRef, BuildState> buildStates = componentMap.collectEntries {
46+
[it.key, BuildState.NEW]
47+
} as Map<MavenArtifactRef, BuildState>
48+
49+
Map<MavenArtifactRef, MavenComponent> moduleToComponentMap =
50+
ComponentDependencyGraph.getModuleToComponentMap(components)
51+
52+
ConcurrentMap<MavenArtifactRef, List<MavenArtifactRef>> buildDeps = new ConcurrentHashMap<>()
53+
buildStates.keySet().forEach { MavenArtifactRef key ->
54+
def depsTasks = componentMap.get(key).modules
55+
.collectMany { it.dependencies }
56+
.findAll { moduleToComponentMap.containsKey(it) }
57+
.unique()
58+
.collect { new MavenArtifactRef(moduleToComponentMap.get(it)) }
59+
.findAll { !key.equals(it) }
60+
buildDeps.put(key, depsTasks)
61+
return
62+
}
63+
64+
Set<MavenArtifactRef> tasks = new HashSet<>(buildStates.keySet())
65+
66+
def executionQueue = new LinkedBlockingQueue<Runnable>()
67+
def pool = new ThreadPoolExecutor(context.getParallel(), context.getParallel(),
68+
0L, TimeUnit.MILLISECONDS,
69+
executionQueue)
70+
try {
71+
while (!tasks.isEmpty()) {
72+
def iter = tasks.iterator()
73+
while (iter.hasNext()) {
74+
def key = iter.next()
75+
def component = componentMap.get(key)
76+
def deps = buildDeps.get(key)
77+
if (!deps.any { buildStates.get(it) != BuildState.SUCCESS }) {
78+
pool.execute {
79+
// build component
80+
try {
81+
def pomFile = new File(component.basedir, 'pom.xml')
82+
try {
83+
Maven.execute(context, pomFile, ['clean'])
84+
} catch (RepoBuildException ignore) {
85+
}
86+
Maven.execute(context, pomFile, ['install'])
87+
buildStates.put(key, BuildState.SUCCESS)
88+
} catch (Exception e) {
89+
buildStates.put(key, BuildState.ERROR)
90+
context.setErrorFlag()
91+
context.addError(new RepoBuildException(" component ${component.path} build ERROR", e))
92+
}
93+
}
94+
iter.remove()
95+
} else if (deps.any {
96+
buildStates.get(it) == BuildState.DEPS_ERROR ||
97+
buildStates.get(it) == BuildState.ERROR
98+
}) {
99+
buildStates.put(key, BuildState.DEPS_ERROR)
100+
context.setErrorFlag()
101+
context.addError(new RepoBuildException(" component ${component.path} build DEPS_ERROR"))
102+
iter.remove()
103+
}
104+
}
105+
// throttle cpu
106+
Thread.sleep(500L)
107+
}
108+
} finally {
109+
pool.shutdown()
110+
}
111+
return !context.getErrorFlag()
112+
}
113+
}
Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
package repo.build.maven;
2+
3+
/**
4+
* @author Markelov Ruslan markelov@jet.msk.su
5+
*/
6+
public enum BuildState {
7+
NEW,
8+
SUCCESS,
9+
ERROR,
10+
DEPS_ERROR
11+
}

0 commit comments

Comments
 (0)