Skip to content

Commit d52fa7e

Browse files
authored
Merge pull request #24 from armingh2000/fix/fact_scorer_knowledge
Fix/fact scorer knowledge
2 parents 35dae2a + 535443c commit d52fa7e

File tree

6 files changed

+22
-12
lines changed

6 files changed

+22
-12
lines changed

CHANGELOG.md

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,12 @@ All notable changes to this project will be documented in this file.
8080

8181
- Fix knowledge source in fact scorer prompt (used to send all knowledge sources in one prompt)
8282

83+
## v 1.3.0 - 2024-04-22
84+
85+
- Refactor knowledge source passing in FactScore
86+
- Update FactScorer prompt (improve performance)
87+
- Update README.md to contain the new prompt.
88+
8389
<!--
8490
### Added
8591

FactScoreLite/fact_scorer.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,8 @@ def get_instructions(self) -> str:
3434
str: The instructions for the prompt generation.
3535
"""
3636

37-
instructions = "Evaluate the truthfulness of the statement based solely on the provided context and provide the reason for your decision.\n\n"
38-
instructions += "Instruction:\nOnly consider the statement true if it can be directly verified by the information in the context. If the information in the statement cannot be found in the context or differs from it, label it as false.\n\n"
37+
# instructions = "Evaluate the truthfulness of the statement based solely on the provided context.\n\n"
38+
instructions = "Instruction:\nOnly consider the statement true if it can be directly verified by the information in the context. If the information in the statement cannot be found in the context or differs from it, label it as false.\n\n"
3939
true_example = self.demons[0]
4040
false_example = random.choice(self.demons[1:])
4141

FactScoreLite/factscore.py

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -98,18 +98,27 @@ def get_decisions(
9898
decisions = self.decisions_handler.load()
9999
scores = []
100100
init_scores = []
101-
cur_knw_idx = 0
102101

103102
for entry in decisions:
104103
score, init_score = self.calculate_score(entry["decision"])
105104
init_scores.append(init_score)
106105
scores.append(score)
107-
cur_knw_idx += 1
108106

109-
for entry in tqdm(generation_facts_pairs[len(decisions) :]):
107+
assert len(generation_facts_pairs) == len(
108+
knowledge_sources
109+
), "Number of generation-facts pairs and knowledge sources should be the same."
110+
111+
current_index = len(decisions)
112+
113+
for entry, knowledge_source in tqdm(
114+
zip(
115+
generation_facts_pairs[current_index:],
116+
knowledge_sources[current_index:],
117+
)
118+
):
110119
generation, facts = entry["generation"], entry["facts"]
111120

112-
decision = self.fact_scorer.get_score(facts, knowledge_sources[cur_knw_idx])
121+
decision = self.fact_scorer.get_score(facts, knowledge_source)
113122
score, init_score = self.calculate_score(decision)
114123

115124
init_scores.append(init_score)

README.md

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -170,13 +170,9 @@ The following prompt template is used to instruct GPT for scoring facts:
170170
```
171171
# fact_scorer.py
172172

173-
Evaluate the truthfulness of the statement based solely on the provided context and provide the reason for your decision.
174-
175-
176173
Instruction:
177174
Only consider the statement true if it can be directly verified by the information in the context. If the information in the statement cannot be found in the context or differs from it, label it as false.
178175

179-
180176
Context:
181177
knw 1
182178
Statement:

setup.cfg

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[metadata]
22
name = FactScoreLite
3-
version = 1.2.0
3+
version = 1.3.0
44
author = armingh2000
55
author_email =
66
license = MIT

tests/test_fact_scorer.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,6 @@ def test_get_instructions_true_false_demons(fact_scorer):
122122
# Test case for a single demon in self.demons
123123
fact_scorer.demons = mock_demons_data
124124
expected_instructions = (
125-
"Evaluate the truthfulness of the statement based solely on the provided context and provide the reason for your decision.\n\n"
126125
"Instruction:\nOnly consider the statement true if it can be directly verified by the information in the context. If the information in the statement cannot be found in the context or differs from it, label it as false.\n\n"
127126
"Context:\nknw 1\n"
128127
"Statement:\nfact 1 True or False?\n"

0 commit comments

Comments
 (0)