feat: taxonomy patch instead of re-generating (#554)

The goal is to be able to patch taxonomy text files instead of
re-generating them completely.

This will avoid having a lot of changes that are not related to the real
modifications made by a contributor.

For this we need to
* add a "modified" property to entries (to track modified entries)
  * a parent id change marks an entry as changed (as we need to regenerate it in the file to have parent id right)
* track lines location of entries (to know where to change the original
file)
* keep removed entries (because we need to remove them from the
original file) by prefixing their type label by REMOVED (REMOVED_ENTRY for example)
* added integration tests
* new entries are put next to their parents or at the end of the file

Relates to: #541 and #366

Author: alexgarel

.github/workflows/github-projects-for-openfoodfacts-design.yml

@@ -59,13 +59,13 @@ jobs:
           project-url: https://github.com/orgs/openfoodfacts/projects/5 # Add issue to the folksonomy project
           github-token: ${{ secrets.ADD_TO_PROJECT_PAT }}
           labeled: 🏷️ Folksonomy Project
-          label-operator: OR         
+          label-operator: OR
       - uses: actions/add-to-project@main
         with:
           project-url: https://github.com/orgs/openfoodfacts/projects/44 # Add issue to the data quality project
           github-token: ${{ secrets.ADD_TO_PROJECT_PAT }}
           labeled: 🧽 Data quality
-          label-operator: OR    
+          label-operator: OR
       - uses: actions/add-to-project@main
         with:
           project-url: https://github.com/orgs/openfoodfacts/projects/82 # Add issue to the search project
@@ -77,19 +77,19 @@ jobs:
           project-url: https://github.com/orgs/openfoodfacts/projects/41 # Add issue to the producer platform project
           github-token: ${{ secrets.ADD_TO_PROJECT_PAT }}
           labeled: 🏭 Producers Platform
-          label-operator: OR    
+          label-operator: OR
       - uses: actions/add-to-project@main
         with:
           project-url: https://github.com/orgs/openfoodfacts/projects/19 # Add issue to the infrastructure project
           github-token: ${{ secrets.ADD_TO_PROJECT_PAT }}
           labeled: infrastructure
-          label-operator: OR   
+          label-operator: OR
       - uses: actions/add-to-project@main
         with:
           project-url: https://github.com/orgs/openfoodfacts/projects/92 # Add issue to the Nutri-Score project
           github-token: ${{ secrets.ADD_TO_PROJECT_PAT }}
           labeled: 🚦 Nutri-Score
-          label-operator: OR   
+          label-operator: OR
       - uses: actions/add-to-project@main
         with:
           project-url: https://github.com/orgs/openfoodfacts/projects/132 # Add issue to the Top upvoted issues board
@@ -107,4 +107,4 @@ jobs:
           project-url: https://github.com/orgs/openfoodfacts/projects/35 # Add issue to the ♿️ accessibility project
           github-token: ${{ secrets.ADD_TO_PROJECT_PAT }}
           labeled: ♿️ accessibility
-          label-operator: OR   
+          label-operator: OR

Makefile

@@ -99,9 +99,14 @@ local_config_quality: ## Run on lint configuration files
 
 build: ## Build docker images
 	@echo "🍜 Building docker images"
-	${DOCKER_COMPOSE} build
+	${DOCKER_COMPOSE} build ${args}
 	@echo "🍜 Project setup done"
 
+backend_poetry_update: ## Update poetry.lock file
+	@echo "🍜 Updating poetry.lock file"
+	${DOCKER_COMPOSE} run --user root --rm taxonomy_api bash -c "pip install poetry==1.4.2 && poetry lock --no-update"
+
+
 up: ## Run the project
 	@echo "🍜 Running project (ctrl+C to stop)"
 	@echo "🍜 The React app will be available on http://ui.taxonomy.localhost:8091"
@@ -177,11 +182,11 @@ config_quality: ## Run quality checks on configuration files
 
 tests: backend_tests ## Run all tests
 
-backend_tests: ## Run python tests
+backend_tests: ## Run python tests, you might provide additional arguments with args="…"
 	@echo "🍜 Running python tests"
 	${DOCKER_COMPOSE_TEST} up -d neo4j
-	${DOCKER_COMPOSE_TEST}  run --rm taxonomy_api pytest /parser
-	${DOCKER_COMPOSE_TEST}  run --rm taxonomy_api pytest /code/tests
+	${DOCKER_COMPOSE_TEST}  run --rm taxonomy_api pytest /parser ${args}
+	${DOCKER_COMPOSE_TEST}  run --rm taxonomy_api pytest /code/tests ${args}
 	${DOCKER_COMPOSE_TEST} stop neo4j
 
 checks: quality tests ## Run all checks (quality + tests)

backend/editor/api.py

@@ -184,7 +184,7 @@ async def find_all_nodes(response: Response, branch: str, taxonomy_name: str):
     Get all nodes within taxonomy
     """
     taxonomy = TaxonomyGraph(branch, taxonomy_name)
-    all_nodes = await taxonomy.get_all_nodes("")
+    all_nodes = await taxonomy.get_all_nodes()
     return all_nodes
 
 
@@ -235,7 +235,7 @@ async def find_one_synonym(response: Response, branch: str, taxonomy_name: str,
     Get synonym corresponding to id within taxonomy
     """
     taxonomy = TaxonomyGraph(branch, taxonomy_name)
-    one_synonym = await taxonomy.get_nodes("SYNONYMS", synonym)
+    one_synonym = await taxonomy.get_nodes(NodeType.SYNONYMS, synonym)
 
     check_single(one_synonym)
 
@@ -248,7 +248,7 @@ async def find_all_synonyms(response: Response, branch: str, taxonomy_name: str)
     Get all synonyms within taxonomy
     """
     taxonomy = TaxonomyGraph(branch, taxonomy_name)
-    all_synonyms = await taxonomy.get_all_nodes("SYNONYMS")
+    all_synonyms = await taxonomy.get_all_nodes(NodeType.SYNONYMS)
     return all_synonyms
 
 
@@ -258,7 +258,7 @@ async def find_one_stopword(response: Response, branch: str, taxonomy_name: str,
     Get stopword corresponding to id within taxonomy
     """
     taxonomy = TaxonomyGraph(branch, taxonomy_name)
-    one_stopword = await taxonomy.get_nodes("STOPWORDS", stopword)
+    one_stopword = await taxonomy.get_nodes(NodeType.STOPWORDS, stopword)
 
     check_single(one_stopword)
 
@@ -271,7 +271,7 @@ async def find_all_stopwords(response: Response, branch: str, taxonomy_name: str
     Get all stopwords within taxonomy
     """
     taxonomy = TaxonomyGraph(branch, taxonomy_name)
-    all_stopwords = await taxonomy.get_all_nodes("STOPWORDS")
+    all_stopwords = await taxonomy.get_all_nodes(NodeType.STOPWORDS)
     return all_stopwords
 
 
@@ -281,7 +281,7 @@ async def find_header(response: Response, branch: str, taxonomy_name: str):
     Get __header__ within taxonomy
     """
     taxonomy = TaxonomyGraph(branch, taxonomy_name)
-    header = await taxonomy.get_nodes("TEXT", "__header__")
+    header = await taxonomy.get_nodes(NodeType.TEXT, "__header__")
     return header[0]
 
 
@@ -291,7 +291,7 @@ async def find_footer(response: Response, branch: str, taxonomy_name: str):
     Get __footer__ within taxonomy
     """
     taxonomy = TaxonomyGraph(branch, taxonomy_name)
-    footer = await taxonomy.get_nodes("TEXT", "__footer__")
+    footer = await taxonomy.get_nodes(NodeType.TEXT, "__footer__")
     return footer[0]
 
 
@@ -395,7 +395,7 @@ async def upload_taxonomy(
     """
     taxonomy = TaxonomyGraph(branch, taxonomy_name)
     if not taxonomy.is_valid_branch_name():
-        raise HTTPException(status_code=422, detail="branch_name: Enter a valid branch name!")
+        raise HTTPException(status_code=422, detail="branch_name: Enter a valid branch name!")
     if await taxonomy.does_project_exist():
         raise HTTPException(status_code=409, detail="Project already exists!")
     if not await taxonomy.is_branch_unique(from_github=False):
@@ -431,7 +431,7 @@ async def edit_entry(request: Request, branch: str, taxonomy_name: str, entry: s
     incoming_data = await request.json()
     incoming_data["id"] = entry
     new_entry = EntryNode(**incoming_data)
-    updated_entry = await taxonomy.update_node("ENTRY", new_entry)
+    updated_entry = await taxonomy.update_node(NodeType.ENTRY, new_entry)
     return updated_entry
 
 
@@ -467,5 +467,5 @@ async def delete_project(branch: str, taxonomy_name: str):
     """
     Delete a project
     """
-    taxonomy = TaxonomyGraph(branch, taxonomy_name)
-    await project_controller.delete_project(taxonomy.project_name)
+    project_id = project_controller.get_project_id(branch, taxonomy_name)
+    await project_controller.delete_project(project_id)

backend/editor/controllers/node_controller.py

@@ -1,3 +1,4 @@
+import datetime
 import logging
 
 from openfoodfacts_taxonomy_parser import utils as parser_utils
@@ -12,7 +13,7 @@
 async def delete_project_nodes(project_id: str):
     """
     Remove all nodes for project.
-    This includes entries, stopwords, synonyms and errors
+    This includes entries, stopwords, synonyms, errors and removed entries
     """
 
     query = f"""
@@ -38,6 +39,8 @@ async def create_entry_node(
         "id": language_code + ":" + normalized_name,
         f"tags_{language_code}": [name],
         f"tags_ids_{language_code}": [normalized_name],
+        "modified": datetime.datetime.now().timestamp(),
+        "is_external": False,
     }
     params = {"entry_node": entry_node_data}
 

backend/editor/controllers/project_controller.py

@@ -4,6 +4,10 @@
 from .utils.result_utils import get_unique_record
 
 
+def get_project_id(branch_name: str, taxonomy_name: str) -> str:
+    return "p_" + taxonomy_name + "_" + branch_name
+
+
 async def get_project(project_id: str) -> Project:
     """
     Get project by id
@@ -78,3 +82,42 @@ async def delete_project(project_id: str):
     params = {"project_id": project_id}
     await get_current_transaction().run(query, params)
     await delete_project_nodes(project_id)
+
+
+async def clone_project(source_branch: str, taxonomy_name: str, target_branch: str):
+    """
+    Clone a project using a new branch name
+
+    Currently used for tests only.
+    """
+    source_id = get_project_id(source_branch, taxonomy_name)
+    target_id = get_project_id(target_branch, taxonomy_name)
+    # clone project node
+    query = """
+        MATCH (p:PROJECT {id: $project_id})
+        WITH p
+        CALL apoc.refactor.cloneNodes([p], true, ['id', 'branch'] )
+        YIELD output as new_node
+        WITH new_node
+        SET new_node.created_at = datetime(),
+            new_node.branch_name = $target_branch,
+            new_node.id = $target_id
+        RETURN new_node
+    """
+    params = {
+        "project_id": source_id,
+        "target_branch": target_branch,
+        "target_id": get_project_id(target_branch, taxonomy_name),
+    }
+    await get_current_transaction().run(query, params)
+    # clone nodes thanks to apoc.refactor.cloneSubgraph
+    query = f"""
+        MATCH (n:{source_id})
+        WITH collect(n) AS source_nodes
+        CALL apoc.refactor.cloneSubgraph(source_nodes)
+        YIELD output as new_node
+        WITH new_node
+        REMOVE new_node:{source_id}
+        SET new_node:{target_id}
+    """
+    await get_current_transaction().run(query)