Skip to content

Commit f10a9e6

Browse files
kikass13kikass13
committed
refined plugin based system; implemented first real plugin for local file contributions scanning; 'blacked' my code as far as possible; still work in progress protontypes#164 
1 parent 9bea708 commit f10a9e6

File tree

5 files changed

+360
-77
lines changed

5 files changed

+360
-77
lines changed

‎libreselery/contribution_action_plugins/git_file_contribution_action.py

Lines changed: 210 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,210 @@
1+
#! /usr/bin/python3
2+
3+
from libreselery.contribution_distribution_engine_types import (
4+
ContributionAction,
5+
ContributionActionPlugin,
6+
)
7+
8+
### Start User Imports
9+
### specialzed plugin imports can be added here
10+
##################################################################################
11+
import subprocess
12+
from datetime import datetime
13+
14+
### End User Imports
15+
##################################################################################
16+
17+
18+
class GitFileContributionAction(ContributionActionPlugin):
19+
"""
20+
This class is a plugin containing the implementation of a single ContributorAction.
21+
It is responsible for gathering contributor information and evaluating scores
22+
for each contributor based on configurated metrics
23+
24+
Plugin description:
25+
This plugin will evaluate the line-of-code contributions made from all
26+
contributors of local files (under git version control).
27+
"""
28+
29+
_alias_ = "git_file_contribution_action"
30+
GITBLAMESEPERATOR = "***\n"
31+
32+
def __init__(self):
33+
super(GitFileContributionAction, self).__init__()
34+
35+
def initialize_(self, action):
36+
"""
37+
Overload of abstract method which is responsible for initializing this plugin
38+
39+
Parameters:
40+
action (ContributionAction):
41+
action object which contains all necessary information of what
42+
a contributor has to doto be scored and recognized as such
43+
44+
Returns:
45+
bool: True if successfully initialized
46+
"""
47+
return True
48+
49+
def gather_(self, cachedContributors=[]):
50+
"""
51+
Overload of abstract method which is responsible for gathering
52+
contributor information and scoring contributors based on the action defined
53+
54+
Parameters:
55+
[optional] cachedContributors (list):
56+
list of contributors from various external (remote) sources which had been chached earlier
57+
so that plugins don't need to do expensive lookups all the time
58+
59+
Returns:
60+
tuple: (list of contributors, list of scores)
61+
"""
62+
contributors = []
63+
scores = []
64+
65+
fileContributions = self.execGit()
66+
for filename, fileContributorDict in fileContributions.items():
67+
print("%s" % filename)
68+
self.printFileContributorDict(fileContributorDict)
69+
70+
return contributors, scores
71+
72+
### Start User Methods
73+
### specialzed plugin methods can be added here
74+
##################################################################################
75+
76+
def execGit(self):
77+
cmd = [
78+
"git",
79+
"ls-files",
80+
"|",
81+
"while read f;",
82+
'do echo "%s$f";' % self.GITBLAMESEPERATOR,
83+
"git blame -CCC --line-porcelain $f;",
84+
"done",
85+
]
86+
87+
ps = subprocess.Popen(
88+
" ".join(cmd), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
89+
)
90+
stdout, stderr = ps.communicate()
91+
92+
fileContributions = {}
93+
if not stderr:
94+
encoded = stdout.decode("utf-8")
95+
### split output for each file (we added "\n" manually to seperate each file output)
96+
fileblames = encoded.split(self.GITBLAMESEPERATOR)[
97+
1:
98+
] ### 0 entry is empty because reasons ... :D
99+
for blame in fileblames:
100+
### seperate lines into array
101+
lines = blame.split("\n")
102+
filename = lines[0]
103+
lines = lines[1:]
104+
#print("Blame > %s [%s]" % (filename, len(lines)))
105+
### put lines through blameParser
106+
fileContributorDict = self.parseBlame(lines)
107+
fileContributions[filename] = fileContributorDict
108+
return fileContributions
109+
110+
def printFileContributorDict(self, fcDict):
111+
for author, data in fcDict.items():
112+
print(" %s [%s]" % (author, data["count"]))
113+
for stamp, count in data["stamps"].items():
114+
datetimeStr = datetime.fromtimestamp(float(stamp)).strftime(
115+
"%Y-%m-%d/%H:%M:%S"
116+
)
117+
print(" -- %s [%s]" % (datetimeStr, count))
118+
119+
def parseBlame(self, lines):
120+
lineDescriptions = []
121+
lineDescription = {}
122+
currentCommitSha = None
123+
headerLength = None
124+
newEntry = True
125+
for line in lines:
126+
if newEntry:
127+
# print("######################################")
128+
newEntry = False
129+
### commit hash extraction
130+
key = "commit"
131+
val = line.split(" ")[0]
132+
else:
133+
splits = line.split(" ")
134+
key = splits[0]
135+
val = " ".join(splits[1:])
136+
137+
### the amount of lines per entry is not consistent inside git blame
138+
### so parsing is nearly impossible
139+
### the only proper criteria in this stupid pocelain implementation is when we have a codeline which allways starts with \t
140+
### thanks git ... idiots -.-
141+
if line and line[0] == "\t":
142+
### valuable lines done
143+
lineDescriptions.append(lineDescription)
144+
lineDescription = {}
145+
newEntry = True
146+
147+
lineDescription[key] = val
148+
149+
fileContributions = {}
150+
for d in lineDescriptions:
151+
author = d["author-mail"]
152+
timestamp = d["committer-time"]
153+
key = author
154+
dd = fileContributions.get(author, None)
155+
if dd:
156+
c = dd["count"]
157+
stamps = dd["stamps"]
158+
c += 1
159+
if timestamp in stamps:
160+
stamps[timestamp] += 1
161+
else:
162+
stamps[timestamp] = 1
163+
### rewrite the dict data
164+
fileContributions[key] = {"count": c, "stamps": stamps}
165+
else:
166+
fileContributions[key] = {"count": 1, "stamps": {timestamp: 1}}
167+
return fileContributions
168+
169+
### End User Methods
170+
##################################################################################
171+
172+
173+
def test():
174+
success = False
175+
print("This is a Test!")
176+
### define our input configuration (action) which normally comes from .yml configuration
177+
d = {
178+
"contributions_to_code": {
179+
"type": "git_file_contribution_action", ### type of action (also the name of the plugin _alias_ used!)
180+
"applies_to": [
181+
"*.md",
182+
"docs/",
183+
], ### simple filter, not really thought out yet
184+
"metrics": [ ### metrics applied to this action, what gets score and what doesnt
185+
{
186+
"UNIFORM": { ### metric identifier
187+
"degradation_type": "linear",
188+
"degradation_value": 1,
189+
}
190+
}
191+
],
192+
}
193+
}
194+
### create an action object
195+
action = ContributionAction(d)
196+
### initialize the action
197+
### which will in turn use this specific plugin
198+
### if configured correctly
199+
init = action.initialize_()
200+
if init:
201+
### let us do our work
202+
data = action.gather_()
203+
### visualize and evaluate test data
204+
print(data)
205+
success = True
206+
return success
207+
208+
209+
if __name__ == "__main__":
210+
assert test() == True

‎libreselery/contribution_action_plugins/test_action.py

Lines changed: 87 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,28 +1,103 @@
11
#! /usr/bin/python3
22

3-
from libreselery.contribution_distribution_engine_types import ContributionActionPlugin as actionplugin
3+
from libreselery.contribution_distribution_engine_types import (
4+
ContributionAction,
5+
ContributionActionPlugin,
6+
)
47

8+
### Start User Imports
9+
### specialzed plugin imports can be added here
10+
##################################################################################
11+
import sys
12+
13+
### End User Imports
14+
##################################################################################
15+
16+
17+
class MY_TEST_ACTION_PLUGIN_CLASS(ContributionActionPlugin):
18+
"""
19+
This class is a plugin containing the implementation of a single ContributorAction.
20+
It is responsible for gathering contributor information and evaluating scores
21+
for each contributor based on configurated metrics
22+
23+
Plugin description:
24+
This plugin does nothing special, it's just for testing and showcasing how
25+
to use and implement plugins in the action lifecycle of the CDE.
26+
It will just return a random contributor list and some randome scores.
27+
"""
28+
29+
_alias_ = "test_action"
530

6-
class MY_TEST_ACTION_PLUGIN_CLASS(actionplugin):
7-
_alias_ = 'test_action'
8-
931
def __init__(self):
1032
super(MY_TEST_ACTION_PLUGIN_CLASS, self).__init__()
1133

1234
def initialize_(self, action):
35+
"""
36+
Overload of abstract method which is responsible for initializing this plugin
37+
38+
Parameters:
39+
action (ContributionAction):
40+
action object which contains all necessary information of what
41+
a contributor has to doto be scored and recognized as such
42+
43+
Returns:
44+
bool: True if successfully initialized
45+
"""
1346
print(" > PLUGIN - INIT")
14-
pass
47+
return True
1548

1649
def gather_(self, cachedContributors=[]):
50+
"""
51+
Overload of abstract method which is responsible for gathering
52+
contributor information and scoring contributors based on the action defined
53+
54+
Parameters:
55+
[optional] cachedContributors (list):
56+
list of contributors from various external (remote) sources which had been chached earlier
57+
so that plugins don't need to do expensive lookups all the time
58+
59+
Returns:
60+
tuple: (list of contributors, list of scores)
61+
"""
1762
print(" > PLUGIN - GATHER")
1863
contributors = ["kikass13", "otherUser"]
1964
scores = [1337.0, 500.0]
20-
2165
return contributors, scores
2266

23-
#def weight_(self, actionContributors):
24-
# print(" > PLUGIN - WEIGHTS")
25-
# contributors = ["kikass13"]
26-
# weights = [1.0]
27-
# return contributors, weights
28-
67+
### Start User Methods
68+
### specialzed plugin methods can be added here
69+
##################################################################################
70+
###
71+
### def work(self):
72+
### pass
73+
###
74+
### Énd User Methods
75+
##################################################################################
76+
77+
78+
def test():
79+
success = False
80+
print("This is a Test!")
81+
### define our input configuration (action) which normally comes from .yml configuration
82+
d = {
83+
"test_action_id": {
84+
"type": "test_action", ### type of action (also the name of the plugin _alias_ used!)
85+
}
86+
}
87+
### create an action object
88+
action = ContributionAction(d)
89+
### initialize the action
90+
### which will in turn use this specific plugin
91+
### if configured correctly
92+
init = action.initialize_()
93+
if init:
94+
### let us do our work
95+
data = action.gather_()
96+
### visualize and evaluate test data
97+
print(data)
98+
success = True
99+
return success
100+
101+
102+
if __name__ == "__main__":
103+
assert test() == True

‎libreselery/contribution_distribution_engine.py

Lines changed: 10 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -11,30 +11,30 @@ def __init__(self, config):
1111
super(ContributionDistributionEngine, self).__init__()
1212
###grab relevant entries from selery cfg
1313
self.domains = self._extractContributionDomains(config)
14-
14+
1515
def _extractContributionDomains(self, config):
16-
### read the config and parse usable objects for each domain configured
16+
### read the config and parse usable objects for each domain configured
1717
domains = []
1818
for domainDict in config.contribution_domains:
19-
domain = cdetypes.ContributionDomain(domainDict)
19+
domain = cdetypes.ContributionDomain(domainDict)
2020
domains.append(domain)
2121
return domains
2222

2323
def gather_(self):
2424
### our task is to apply whatever ContributionType was configured
25-
### for a specific domain and extract all
25+
### for a specific domain and extract all
2626
### contributors + their weights that fit into this domain
2727
print("\n\nLOOK, BUT DONT TOUCH!")
2828
cachedContributors = []
2929

30-
contributorData = {"gather" : {}}
30+
contributorData = {"gather": {}}
3131
for domain in self.domains:
3232
### execute all actions of every domain
3333
### this should identify the contributos that
34-
### fit the action description /
34+
### fit the action description /
3535
### that have done the configured action successfully
3636
contributorScores = domain.gather_(cachedContributors=cachedContributors)
37-
### every domain has to weight it's actions
37+
### every domain has to weight it's actions
3838
contributorData["gather"][domain.name] = contributorScores
3939
###
4040
return contributorData
@@ -43,11 +43,9 @@ def weight_(self, contributorData):
4343
### domains have to weight action scores in relation to each other
4444
contributorData["weight"] = {}
4545
for domain in self.domains:
46-
print("\n_____ weight %s" % domain.name)
4746
domainContent = contributorData.get("gather").get(domain.name)
4847
### normalize contributor weights based on contributor scores
4948
contributors, weights = domain.weight_(domainContent)
50-
print("weights: %s" % weights)
5149
contributorData["weight"][domain.name] = (contributors, weights)
5250
return contributorData
5351

@@ -64,18 +62,14 @@ def merge_(self, contributorData):
6462
contributorData["merge"][contributor] += weight * domain.weight
6563
else:
6664
contributorData["merge"][contributor] = weight * domain.weight
67-
68-
### because we potentially downgraded our weights by multiplying with
65+
66+
### because we potentially downgraded our weights by multiplying with
6967
### the given domain weight ... we have to re-normalize the weights
7068
### of every contributor to be within [0 ... 1] again
7169
contributorData["merge_norm"] = {}
7270
blob = [*contributorData.get("merge").items()]
73-
contributors, weights = ([c for c,w in blob], [w for c,w in blob])
71+
contributors, weights = ([c for c, w in blob], [w for c, w in blob])
7472
newWeights = cdetypes.normalizeSum(weights)
7573
for contributor, weight in zip(contributors, newWeights):
7674
contributorData["merge_norm"][contributor] = weight
7775
return contributorData
78-
79-
80-
81-

0 commit comments

Comments
 (0)