Skip to content

Commit deb920a

Browse files
committed
fixing json and example
1 parent 3eacc6f commit deb920a

File tree

4 files changed

+8
-4
lines changed

4 files changed

+8
-4
lines changed

examples/openai/result.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{"books": [{"title": "XML Developer's Guide", "author": "Unknown", "genre": "Unknown"}]}

scrapegraphai/graphs/json_scraper_graph.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ def _create_graph(self):
3030
Creates the graph of nodes representing the workflow for web scraping.
3131
"""
3232
fetch_node = FetchNode(
33-
input="json | json_dir",
33+
input="json_dir",
3434
output=["doc"],
3535
)
3636
parse_node = ParseNode(

scrapegraphai/graphs/xml_scraper_graph.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ def _create_graph(self):
3030
Creates the graph of nodes representing the workflow for web scraping.
3131
"""
3232
fetch_node = FetchNode(
33-
input="xml | xml_dir",
33+
input="xml_dir",
3434
output=["doc"],
3535
)
3636
parse_node = ParseNode(

scrapegraphai/nodes/fetch_node.py

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -70,9 +70,12 @@ def execute(self, state):
7070
input_data = [state[key] for key in input_keys]
7171

7272
source = input_data[0]
73-
73+
if self.input == "json_dir" or self.input == "xml_dir":
74+
compressed_document = [Document(page_content=source, metadata={
75+
"source": "local_dir"
76+
})]
7477
# if it is a local directory
75-
if not source.startswith("http"):
78+
elif not source.startswith("http"):
7679
compressed_document = [Document(page_content=remover(source), metadata={
7780
"source": "local_dir"
7881
})]

0 commit comments

Comments
 (0)