Dev/backend Add modules to preprocess, modules, config for GraphCypherQAChain

be_repo/app.py

@@ -38,6 +38,11 @@
     user_state_collection = user_database.get_collection("user_chat_state")
     query = {"user_id": "test"}
     user_state = user_state_collection.find_one(query)
+    # update all user state to 0
+    update_result = user_state_collection.update_many(
+        {},
+        {"$set": {"state": 0}}
+    )
 except Exception as e:
     raise Exception("Unable to find the document due to the following error: ", e)
 
@@ -160,6 +165,7 @@ def ask_question():
 
     return jsonify({"response": response}), 200
 
+
 @app.route('/suggest/interiew_question', methods=['POST', 'OPTIONS'])
 def interview_question_suggestion():
     if request.method == 'OPTIONS':

be_repo/configs/openai_key.py

@@ -0,0 +1,27 @@
+# configs/openai_key.py
+
+from configs.database import get_key_database
+
+def get_openai_api_key():
+    """
+    Retrieve the OpenAI API key from the MongoDB database.
+
+    Returns:
+        str: The OpenAI API key.
+
+    Raises:
+        ValueError: If the API key is not found or is empty.
+    """
+    db = get_key_database()
+    keys_collection = db["keys"]
+    openai_key_doc = keys_collection.find_one({"_id": "chatgpt_api"})
+
+    if not openai_key_doc:
+        raise ValueError("OpenAI API key not found in the database.")
+
+    openai_key = openai_key_doc.get("api_key")
+
+    if not openai_key:
+        raise ValueError("OpenAI API key is empty.")
+
+    return openai_key

be_repo/modules/job_recommendation_system.py

@@ -0,0 +1,75 @@
+# job_recommendation_system.py
+
+from neo4j_model import Neo4jModel
+from resume_processor import ResumeProcessor
+from retrieval_engine import RetrievalEngine
+from recommendation_generator import RecommendationGenerator
+from view import CLIView
+import sys
+
+def main():
+
+
+    # Redirect standard output to a file
+    sys.stdout = open('output.log', 'w')
+
+    # Your code here
+    print("Lots of output")
+
+
+    # Setup Logging
+    import logging
+    logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+    logger = logging.getLogger(__name__)
+
+    # Neo4j Connection Details
+    NEO4J_URI = "neo4j+ssc://7bf5a48e.databases.neo4j.io"  # Replace with your Neo4j URI
+    NEO4J_USERNAME = "neo4j"             # Replace with your Neo4j username
+    NEO4J_PASSWORD = "oxsK7V5_86emZlYQlvCfQHfVWS95wXz29OhtU8GAdFc"          # Replace with your Neo4j password
+
+    # Initialize Model
+    neo4j_model = Neo4jModel(
+        uri=NEO4J_URI,
+        username=NEO4J_USERNAME,
+        password=NEO4J_PASSWORD
+    )
+
+    # Initialize Controller Components
+    resume_processor = ResumeProcessor()
+    retrieval_engine = RetrievalEngine(resume_processor, neo4j_model)
+    recommendation_generator = RecommendationGenerator()
+
+    # Initialize View
+    view = CLIView()
+
+    # Get Resume Input from User
+    resume_text = view.get_resume_input()
+
+    if not resume_text.strip():
+        logger.error("No resume text provided.")
+        print("Error: No resume text provided.")
+        return
+
+    # Perform Mixed Retrieval for 'JD' Node Label
+    node_label = "JD"  # Adjust as needed; could be dynamic based on user input or other criteria
+    similar_docs, graph_results = retrieval_engine.perform_mixed_retrieval(resume_text, node_label=node_label)
+
+    if not similar_docs and not graph_results:
+        print("No job recommendations found based on your resume.")
+        return
+
+    # Generate Recommendations
+    try:
+        recommendations = recommendation_generator.generate_recommendations(similar_docs, graph_results)
+    except Exception as e:
+        print("Error: Failed to generate job recommendations.")
+        return
+
+    # Display Recommendations
+    view.display_recommendations(recommendations)
+
+    # Close the file
+    sys.stdout.close()
+
+if __name__ == "__main__":
+    main()

be_repo/modules/neo4j_model.py

@@ -0,0 +1,136 @@
+# neo4j_model.py
+from langchain_neo4j import GraphCypherQAChain, Neo4jGraph
+from langchain_community.vectorstores import Neo4jVector
+from langchain_openai import OpenAIEmbeddings, ChatOpenAI
+from configs.openai_key import get_openai_api_key  # New import
+from langchain.prompts import PromptTemplate
+
+custom_cypher_prompt = PromptTemplate(
+    input_variables=["question", "schema"],
+    template="""
+    You are an expert Cypher query writer for a Neo4j graph database.
+
+    The database has the following schema:
+
+    {schema}
+
+    Given the user's question, generate an efficient Cypher query that:
+
+    - Retrieves relevant job recommendations based on the user's resume.
+    - Excludes the 'embedding' property to avoid exceeding context limits.
+    - Limits the number of results to avoid duplicates and improve performance.
+    - Returns relevant job recommendations based on the user's resume.
+
+    Question:
+    {question}
+
+    Cypher Query:
+    """
+)
+
+
+class Neo4jModel:
+    def __init__(self, uri, username, password):
+
+        # Initialize Neo4j Graph connection
+        self.graph = Neo4jGraph(
+            url=uri,
+            username=username,
+            password=password,
+            # enhanced_schema=True,  # Optional: Provides more detailed schema information
+        )
+
+        # Initialize the embedding model with the API key
+        api_key = get_openai_api_key()
+        self.embeddings = OpenAIEmbeddings(openai_api_key=api_key)
+
+        # Initialize Neo4jVector for each node label
+        self.vector_store_jd = Neo4jVector.from_existing_index(
+            embedding=self.embeddings,
+            url=uri,
+            username=username,
+            password=password,
+            index_name="jd_embedding_index",
+        )
+
+
+        self.vector_store_jtitle = Neo4jVector.from_existing_index(
+            embedding=self.embeddings,
+            url=uri,
+            username=username,
+            password=password,
+            index_name="jtitle_embedding_index",
+        )
+
+
+        self.vector_store_jkeyword = Neo4jVector.from_existing_index(
+            embedding=self.embeddings,
+            url=uri,
+            username=username,
+            password=password,
+            index_name="jkeyword_embedding_index",
+        )
+
+        # Initialize Language Model for QA Chain
+        self.llm = ChatOpenAI(model="gpt-4o-mini", temperature=0, openai_api_key=api_key)
+
+        # Initialize GraphCypherQAChain
+        self.graph_chain = GraphCypherQAChain.from_llm(
+            graph=self.graph,
+            llm=self.llm,
+            cypher_prompt=custom_cypher_prompt,
+            return_intermediate_steps=True,
+            verbose=True,
+            validate_cypher=True,  # Ensures correct relationship directions
+            allow_dangerous_requests=True,
+        )
+
+    def store_documents(self, docs, node_label="JD"):
+        """
+        Store documents in Neo4jVector with embeddings.
+        """
+        # Ensure that 'docs' is a list of Document objects
+        if node_label == "JD":
+            self.vector_store_jd.add_documents(docs)
+
+        elif node_label == "JTitle":
+            self.vector_store_jtitle.add_documents(docs)
+
+        elif node_label == "JKeyword":
+            self.vector_store_jkeyword.add_documents(docs)
+
+        else:
+
+            raise ValueError(f"Invalid node_label '{node_label}'. Must be 'JD', 'JTitle', or 'JKeyword'.")
+
+    def query_graph(self, cypher_query, parameters=None):
+        """
+        Execute a Cypher query against the Neo4j graph.
+        """
+        results = self.graph.query(cypher_query, parameters)
+        return results
+
+    def get_retriever(self, node_label="JD"):
+        """
+        Get a retriever from the Neo4jVector for vector similarity searches.
+        """
+        try:
+            if node_label == "JD":
+                return self.vector_store_jd.as_retriever()
+            elif node_label == "JTitle":
+                return self.vector_store_jtitle.as_retriever()
+            elif node_label == "JKeyword":
+                return self.vector_store_jkeyword.as_retriever()
+            else:
+                raise ValueError(f"Invalid node_label '{node_label}'. Must be 'JD', 'JTitle', or 'JKeyword'.")
+        except Exception as e:
+            raise e
+
+    def get_graph_chain(self):
+        """
+        Get the GraphCypherQAChain instance.
+
+        Returns:
+            GraphCypherQAChain: The QA chain instance.
+        """
+        return self.graph_chain

be_repo/modules/recommendation_generator.py

@@ -0,0 +1,45 @@
+# recommendation_generator.py
+
+class RecommendationGenerator:
+    def __init__(self):
+        pass
+
+    def merge_results(self, vector_docs, graph_results):
+        combined_jobs = {}
+
+        # Process vector similarity results
+        for doc in vector_docs:
+            comp = doc.metadata.get("comp", "")
+            resp = doc.metadata.get("resp", "")
+            job_title = f"{resp} at {comp}".strip()
+            if job_title:
+                combined_jobs[job_title] = combined_jobs.get(job_title, 0) + 1
+
+        # Process graph traversal results
+        # Access the context from intermediate steps
+        intermediate_steps = graph_results.get('intermediate_steps', [])
+        if len(intermediate_steps) > 1:
+            context = intermediate_steps[1].get('context', [])
+            for job in context:
+                job_title = job.get('job_title', '')
+                company = job.get('company', '')
+                if job_title and company:
+                    combined_job = f"{job_title} at {company}"
+                    combined_jobs[combined_job] = combined_jobs.get(combined_job, 0) + 1
+
+        # Convert to sorted list based on combined score
+        sorted_jobs = sorted(combined_jobs.items(), key=lambda item: item[1], reverse=True)
+        return [job for job, score in sorted_jobs]
+
+    def generate_recommendations(self, vector_docs, graph_results):
+        """
+        Generate a ranked list of job recommendations by merging vector and graph results.
+
+        Parameters:
+            vector_docs (List[Document]): Documents from vector similarity search.
+            graph_results (dict): Results from graph traversal.
+
+        Returns:
+            List[str]: Ranked list of unique job recommendations.
+        """
+        return self.merge_results(vector_docs, graph_results)

be_repo/modules/resume_processor.py

@@ -0,0 +1,25 @@
+# resume_processor.py
+
+from langchain.docstore.document import Document
+
+class ResumeProcessor:
+    def __init__(self):
+        """
+        Initialize the Resume Processor.
+        """
+
+    def process_resume(self, resume_text):
+        """
+        Process the user's resume to create a Document object.
+
+        Parameters:
+            resume_text (str): The user's resume text.
+
+        Returns:
+            Document or None: The processed resume as a LangChain Document or None if failed.
+        """
+        try:
+            doc = Document(page_content=resume_text, metadata={})
+            return doc
+        except Exception as e:
+            return None