Merge pull request #4248 from masatake/toml--disabling

masatake · web-flow · commit 23c6e3683ccd · 2025-05-10T15:49:56.000+09:00
TOML,Cargo: disable the parsers temporarily
diff --git a/.github/workflows/building-with-pegof.yml b/.github/workflows/building-with-pegof.yml
@@ -45,8 +45,8 @@ jobs:
       - run: ./ctags --list-features | grep pegof
 
       - run: |
-          echo "Targets: $(./ctags --list-languages=_packcc |  tr '\n' ',')"
-      - run: make units LANGUAGES="$(./ctags --list-languages=_packcc |  tr '\n' ',')"
+          echo "Targets: $(./ctags --list-languages=_packcc |  while read LANG REST; do echo "$LANG"; done | tr '\n' ',')"
+      - run: make units LANGUAGES="$(./ctags --list-languages=_packcc |  while read LANG REST; do echo "$LANG"; done | tr '\n' ',')"
       - run: make dist
         # See EXTRA_DIST in Makefile.am.
         # Currently we add .pego files.
diff --git a/Units/parser-cargo.r/simple.d/args.ctags b/Units/parser-cargo.r/simple.d/args.ctags
@@ -1,4 +1,6 @@
 --languages=-TOML
+--languages=+Cargo
+
 --map-Cargo=.cargo
 --sort=no
 --fields=+lK
diff --git a/Units/parser-markdown.r/packcc-parser-running-within-code-block-with-bom.d/args.ctags b/Units/parser-markdown.r/packcc-parser-running-within-code-block-with-bom.d/args.ctags
@@ -1 +1,3 @@
+--languages=+TOML
+
 --extras=+g
diff --git a/Units/parser-markdown.r/packcc-parser-running-within-code-block.d/args.ctags b/Units/parser-markdown.r/packcc-parser-running-within-code-block.d/args.ctags
@@ -1 +1,3 @@
+--languages=+TOML
+
 --extras=+g
diff --git a/Units/parser-sh.r/sh-heredoc-run-guest-parser-with-packcc-with-bom.d/args.ctags b/Units/parser-sh.r/sh-heredoc-run-guest-parser-with-packcc-with-bom.d/args.ctags
@@ -1,3 +1,5 @@
+--languages=+TOML
+
 --extras=+g
 --sort=no
 --fields=+lne
diff --git a/Units/parser-sh.r/sh-heredoc-run-guest-parser-with-packcc.d/args.ctags b/Units/parser-sh.r/sh-heredoc-run-guest-parser-with-packcc.d/args.ctags
@@ -1,3 +1,5 @@
+--languages=+TOML
+
 --extras=+g
 --sort=no
 --fields=+lne
diff --git a/Units/parser-toml.r/garbage-at-eof.b/README b/Units/parser-toml.r/garbage-at-eof.b/README
@@ -0,0 +1,2 @@
+Taken from https://github.com/BurntSushi/toml/raw/refs/heads/master/internal/toml-test/tests/invalid/encoding/bad-utf8-at-end.toml
+With this input, the TOML parser entered an infinite loop.
diff --git a/Units/parser-toml.r/garbage-at-eof.b/expected.tags b/Units/parser-toml.r/garbage-at-eof.b/expected.tags
@@ -0,0 +1 @@
+x	input.toml	/^x = """"""�/;"	K
diff --git a/Units/parser-toml.r/garbage-at-eof.b/features b/Units/parser-toml.r/garbage-at-eof.b/features
@@ -0,0 +1 @@
+packcc
diff --git a/Units/parser-toml.r/garbage-at-eof.b/input.toml b/Units/parser-toml.r/garbage-at-eof.b/input.toml
@@ -0,0 +1,5 @@
+# There is a 0xda at after the quotes, and no EOL at the end of the file.
+#
+# This is a bit of an edge case: This indicates there should be two bytes
+# (0b1101_1010) but there is no byte to follow because it's the end of the file.
+x = """"""�
diff --git a/Units/parser-toml.r/run-as-guest-with-bom.d/args.ctags b/Units/parser-toml.r/run-as-guest-with-bom.d/args.ctags
@@ -1,3 +1,5 @@
+--languages=+TOML
+
 --sort=no
 --fields=+ne
 --extras=+g
diff --git a/Units/parser-toml.r/run-as-guest.d/args.ctags b/Units/parser-toml.r/run-as-guest.d/args.ctags
@@ -1,3 +1,5 @@
+--languages=+TOML
+
 --sort=no
 --fields=+ne
 --extras=+g
diff --git a/Units/parser-toml.r/simple.d/args.ctags b/Units/parser-toml.r/simple.d/args.ctags
@@ -1,2 +1,4 @@
+--languages=+TOML
+
 --sort=no
 --fields=+K{nth}{end}{line}
diff --git a/docs/news/HEAD.rst b/docs/news/HEAD.rst
@@ -40,13 +40,18 @@ Parser related changes
 New parsers
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-* TOML *peg/packcc*
-* Cargo *TOML based subparser*
 * SELinuxIntefae *M4 based subparser*
 * SELinuxTypeEnforcement *optlib*
 * PythonEntryPoints *subparser*
 * Scdoc *optlib*
 
+.. note:: We added a TOML as a new parser in this version. However,
+		  after adding it, we learned its implementation didn't work
+		  entirely. So we deleted the TOML parser, and Cargo subparser
+		  runs on the TOML parser from this "New parsers" list.
+		  See `TOML: infinite loop <https://github.com/universal-ctags/ctags/issues/4096>`__
+		  about how it doesn't work.
+
 Changes about parser specific kinds, roles, fields, and extras
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
diff --git a/main/parse.c b/main/parse.c
@@ -1813,6 +1813,22 @@ extern void enableLanguage (const langType language, const bool state)
 {
 	Assert (0 <= language  &&  language < (int) LanguageCount);
 	LanguageTable [language].def->enabled = state;
+
+	static bool warned_toml;
+	static bool warned_cargo;
+
+	/* See #4096 */
+	if (!warned_toml && strcmp (LanguageTable [language].def->name, "TOML") == 0)
+	{
+		warned_toml = true;
+		error (WARNING, "The current implementation of the TOML parser is broken.");
+	}
+
+	if (!warned_cargo && strcmp (LanguageTable [language].def->name, "Cargo") == 0)
+	{
+		warned_cargo = true;
+		error (WARNING, "Enabling Cargo subparser may enable TOML parser.");
+	}
 }
 
 #ifdef DO_TRACING
diff --git a/main/parse.h b/main/parse.h
@@ -126,6 +126,7 @@ struct sParserDefinition {
 	tagXpathTableTable *tagXpathTableTable;
 	unsigned int tagXpathTableCount;
 	bool invisible;
+	bool enabled;	       /* currently enabled? */
 	fieldDefinition *fieldTable;
 	unsigned int fieldCount;
 	xtagDefinition *xtagTable;
@@ -149,7 +150,6 @@ struct sParserDefinition {
 
 	/* used internally */
 	langType id;		    /* id assigned to language */
-	unsigned int enabled:1;	       /* currently enabled? */
 	unsigned int traced:1;
 };
 
diff --git a/parsers/cargo.c b/parsers/cargo.c
@@ -136,5 +136,8 @@ extern parserDefinition* CargoParser (void)
 	def->patterns   = patterns;
 	def->parser     = findCargoTags;
 	def->useCork    = CORK_QUEUE;
+
+	def->enabled = false;		/* The base parser, TOML parser doesn't work well. */
+
 	return def;
 }
diff --git a/peg/toml_post.h b/peg/toml_post.h
@@ -243,7 +243,9 @@ extern parserDefinition* TomlParser (void)
 	def->extensions = extensions;
 	def->parser     = findTomlTags;
 	def->useCork    = true;
-	def->enabled    = true;
 	def->requestAutomaticFQTag = false;
+
+	def->enabled    = false;	/* This parser is broken. */
+
 	return def;
 }

Original file line number	Diff line number	Diff line change
`@@ -1 +1,3 @@`
	`1`	`+--languages=+TOML`
	`2`	`+`
`1`	`3`	`--extras=+g`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+Taken from https://github.com/BurntSushi/toml/raw/refs/heads/master/internal/toml-test/tests/invalid/encoding/bad-utf8-at-end.toml`
	`2`	`+With this input, the TOML parser entered an infinite loop.`
-Original file line number
+Diff line change
@@ @@ -0,0 +1,5 @@ @@
 +# There is a 0xda at after the quotes, and no EOL at the end of the file.
 +#
 +# This is a bit of an edge case: This indicates there should be two bytes
 +# (0b1101_1010) but there is no byte to follow because it's the end of the file.
 +x = """"""Ú