Optimize prompt for entire learn loop (#1589)

* Adjust prompt and fix cases
* adjust summarizeTask & learn prompts;
* fix typos & drop duplicate task method;

* adjust learn prompts;

Author: Fivele-Li

qlib/finco/knowledge.py

@@ -1,8 +1,10 @@
 from pathlib import Path
+from jinja2 import Template
+from typing import List
+
 from qlib.workflow import R
 from qlib.finco.log import FinCoLog
 from qlib.finco.llm import APIBackend
-from jinja2 import Template
 
 
 class Knowledge:
@@ -95,7 +97,7 @@ class KnowledgeBase:
     Load knowledge, offer brief information of knowledge and common handle interfaces
     """
 
-    def __init__(self, init_path=None, topics: list[Topic] = None):
+    def __init__(self, init_path=None, topics: List[Topic] = None):
         self.logger = FinCoLog()
         init_path = init_path if init_path else Path.cwd()
 
@@ -111,7 +113,7 @@ def __init__(self, init_path=None, topics: list[Topic] = None):
 
         self.topics = topics if topics else []
 
-    def load(self, path) -> list:
+    def load(self, path) -> List:
         if isinstance(path, str):
             path = Path(path)
 
@@ -131,7 +133,7 @@ def update(self, path):
         self.docs = self.brief(self.knowledge)
         self.logger.plain_info(f"Update knowledge finished.")
 
-    def brief(self, knowledge: list[Knowledge]) -> list:
+    def brief(self, knowledge: List[Knowledge]) -> List:
         docs = []
         for k in knowledge:
             docs.extend(k.brief())

qlib/finco/prompt_template.yaml

@@ -605,24 +605,36 @@ SummarizeTask_system : |-
   You can add subheadings and paragraphs in Markdown for readability.
   You can bold or use other formatting options to highlight keywords in the main text.
   You should display images I offered in markdown using the appropriate image format.
+  Don't list data user doesn't provide.
 
 SummarizeTask_user : |-
   Here is my information: '{{information}}'
   My intention is: {{user_prompt}}. Please provide me with a summary and   recommendation based on my intention and the information I have provided. There are some figures which absolute path are: {{figure_path}}, You must display these images in markdown using the appropriate image format.
 
 SummarizeTask_context_system : |-
-  Your purpose is to find the important information offered by user and summarize it.
+  Your purpose is to find out the important information offered by user. You can just show the data provided by user in markdown format.
 
 SummarizeTask_context_user : |-
   Here is my information: '{{key}}:{{value}}'
+
+SummarizeTask_metrics_system : |-
+  Your purpose is to summarize the information by metrics in markdown format.
+
+SummarizeTask_metrics_user : |-
+  Here is my information: '{{information}}'
   Please summarize it.
 
 LearnManager_system : |-
-  Your task is adjusting system prompt in each task to fulfill user's intention
+  Your task is adjusting system prompt in each task to fulfill user's intention. If you have no idea how to optimize the system prompt, you can just return the original system prompt.
 
 LearnManager_user : |-
-  Here is the final summary:\n{{summary}}\n. Brief of this workflow is:{{brief}}\n 
-  Tasks I have run are: {{task_finished}}, \n{{task}}'s system prompt is: {{system}}. \nUser's intention is: {{user_prompt}}. you will adjust it to:
+  Here is the final summary:\n{{summary}}\n. 
+  Brief of this workflow is:{{brief}}\n 
+  Tasks I have run are: {{task_finished}},\n
+  {{task}}'s system prompt is: {{system}}.\n
+  User's intention is: {{user_prompt}}. 
+  If you have no idea how to optimize the system prompt, you can just return the original system prompt.
+  you will adjust {{task}}'s system prompt to:
 
 Topic_IC : |-
   Summarize the influence of parameters on IC: {{docs}}

qlib/finco/task.py

@@ -776,6 +776,14 @@ def summarize_context_system(self):
     def summarize_context_user(self):
         return self.prompt_template.get(self.__class__.__name__ + "_context_user")
 
+    @property
+    def summarize_metrics_system(self):
+        return self.prompt_template.get(self.__class__.__name__ + "_metrics_system")
+
+    @property
+    def summarize_metrics_user(self):
+        return self.prompt_template.get(self.__class__.__name__ + "_metrics_user")
+
     def execute(self) -> Any:
         workspace = self._context_manager.get_context("workspace")
         user_prompt = self._context_manager.get_context("user_prompt")
@@ -811,8 +819,15 @@ def _get_value_from_info(info: list, k: str):
         recorder = R.get_recorder(experiment_name=workflow_yaml["experiment_name"])
         recorder.save_objects(context_summary=context_summary)
 
+        prompt_workflow_selection = self.summarize_metrics_user.render(
+            information=_get_value_from_info(info=record_info, k="metrics"), user_prompt=user_prompt
+        )
+        metrics_response = be.build_messages_and_create_chat_completion(
+            user_prompt=prompt_workflow_selection, system_prompt=self.summarize_metrics_system.render()
+        )
+
         prompt_workflow_selection = self.user.render(
-            information=file_info + record_info, figure_path=figure_path, user_prompt=user_prompt
+            information=file_info + [{"metrics": metrics_response}], figure_path=figure_path, user_prompt=user_prompt
         )
         response = be.build_messages_and_create_chat_completion(
             user_prompt=prompt_workflow_selection, system_prompt=self.system.render()
@@ -856,23 +871,15 @@ def get_info_from_file(self, path) -> List:
 
     def get_info_from_context(self):
         context = []
-        # TODO: get all keys from context?
-        for key in [
-            "user_prompt",
-            "chat_history",
-            "Dataset_plan",
-            "Model_plan",
-            "Record_plan",
-            "Strategy_plan",
-            "Backtest_plan",
-        ]:
-            c = self._context_manager.get_context(key=key)
-            if c is not None:
-                c = str(c)
-                context.append({key: c[: self.__MAX_LENGTH_OF_FILE]})
+        for key, v in self._context_manager.context.items():
+            if v is not None:
+                v = str(v)
+                context.append({key: v[: self.__MAX_LENGTH_OF_FILE]})
         return context
 
-    def get_info_from_recorder(self, path, exp_name) -> list:
+    @staticmethod
+    def get_info_from_recorder(path, exp_name) -> list:
+        path = Path(path)
         path = path if path.name == "mlruns" else path.joinpath("mlruns")
 
         R.set_uri(Path(path).as_uri())

qlib/finco/workflow.py

@@ -2,6 +2,7 @@
 import copy
 import shutil
 from pathlib import Path
+from typing import List
 
 from qlib.finco.task import WorkflowTask, SummarizeTask, TrainTask
 from qlib.finco.prompt_template import PromptTemplate, Template
@@ -188,22 +189,31 @@ def run(self, prompt):
 
     def learn(self):
         workspace = self.wm.context.get_context("workspace")
+
+        def _drop_duplicate_task(_task: List):
+            unique_task = {}
+            for obj in _task:
+                task_name = obj.__class__.__name__
+                if task_name not in unique_task:
+                    unique_task[task_name] = obj
+            return list(unique_task.values())
+
         # one task maybe run several times in workflow
-        task_finished = list(set(self.wm.context.get_context("task_finished")))
+        task_finished = _drop_duplicate_task(self.wm.context.get_context("task_finished"))
 
         user_prompt = self.wm.context.get_context("user_prompt")
         summary = self.wm.context.get_context("summary")
 
         for task in task_finished:
             prompt_workflow_selection = self.wm.prompt_template.get(f"{self.__class__.__name__}_user").render(
                 summary=summary, brief=self.knowledge_base.query_topics(),
-                task_finished=[str(task) for task in task_finished],
-                task=task.__class__, system=task.system, user_prompt=user_prompt
+                task_finished=[str(t) for t in task_finished],
+                task=task.__class__.__name__, system=task.system.render(), user_prompt=user_prompt
             )
 
             response = APIBackend().build_messages_and_create_chat_completion(
                 user_prompt=prompt_workflow_selection,
-                system_prompt=self.wm.prompt_template.get(f"{self.__class__.__name__}_user").render()
+                system_prompt=self.wm.prompt_template.get(f"{self.__class__.__name__}_system").render()
             )
 
             # todo: response assertion