main: consider foreign languages when solving the field type specifying _field regex flag

The original code assumed the parser specified in a regex pattern
was the owner of the field specified with the _field regex flag in
the pattern.

For an example, in the option

    --regex-Foo=/.../{_field=afield:\1}...

, ctags assumed "afield" was a specific field of "Foo" parser.
The original code didn't consider an option like

    --regex-Foo=/.../{_language=Bar}{_field=afield:\1}...

In this case, ctags should consider "afield" as a specific
field of "Bar" parser.

NOTE: the language specified with _language flag must be
declared with _foreignLanguage= in --langdef-Foo.

Signed-off-by: Masatake YAMATO <yamato@redhat.com>

Author: masatake

Tmain/list-mline-regex-flags.d/stdout-expected.txt

@@ -10,7 +10,7 @@ p       pcre2                                         use pcre2 regex engine
 -       _advanceTo=N[start|end]                       a group in pattern from where the next scan starts [0end]
 -       _anonymous=PREFIX                             make an anonymous tag with PREFIX
 -       _extra=EXTRA                                  record the tag only when the extra is enabled
--       _field=FIELD:VALUE                            record the matched string(VALUE) to parser own FIELD of the tag
+-       _field=FIELD:VALUE                            record the matched string(VALUE) to the (foreign) language specific FIELD of the tag
 -       _guest=PARSERSPEC,N0[start|end],N1[start|end] run guest parser on the area
 -       _language=LANG                                make a foreign tag for LANG
 -       _role=ROLE                                    set the given ROLE to the roles field

Tmain/list-mtable-regex-flags.d/stdout-expected.txt

@@ -16,7 +16,7 @@ p       pcre2                                         use pcre2 regex engine
 -       _advanceTo=N[start|end]                       a group in pattern from where the next scan starts [0end]
 -       _anonymous=PREFIX                             make an anonymous tag with PREFIX
 -       _extra=EXTRA                                  record the tag only when the extra is enabled
--       _field=FIELD:VALUE                            record the matched string(VALUE) to parser own FIELD of the tag
+-       _field=FIELD:VALUE                            record the matched string(VALUE) to the (foreign) language specific FIELD of the tag
 -       _guest=PARSERSPEC,N0[start|end],N1[start|end] run guest parser on the area
 -       _language=LANG                                make a foreign tag for LANG
 -       _role=ROLE                                    set the given ROLE to the roles field

Tmain/list-regex-flags.d/stdout-expected.txt

@@ -11,7 +11,7 @@ x       exclusive                                     skip testing the other pat
 -       warning="MESSAGE"                             print the given MESSAGE at WARNING level
 -       _anonymous=PREFIX                             make an anonymous tag with PREFIX
 -       _extra=EXTRA                                  record the tag only when the extra is enabled
--       _field=FIELD:VALUE                            record the matched string(VALUE) to parser own FIELD of the tag
+-       _field=FIELD:VALUE                            record the matched string(VALUE) to the (foreign) language specific FIELD of the tag
 -       _guest=PARSERSPEC,N0[start|end],N1[start|end] run guest parser on the area
 -       _language=LANG                                make a foreign tag for LANG
 -       _role=ROLE                                    set the given ROLE to the roles field

Tmain/parser-own-fields-for-foreign-lang.d/input.unknownx

@@ -0,0 +1,5 @@
+public func foo(n, m);
+protected func bar(n);
+private func baz(n,...);
+X:tagme@iamowner
+Y:iamowner2=tagme2

Tmain/parser-own-fields-for-foreign-lang.d/knownz.ctags

@@ -0,0 +1,4 @@
+--langdef=knownz
+--kinddef-knownz=m,mark,makers
+
+--_fielddef-knownz=owner,the owner of the markers

Tmain/parser-own-fields-for-foreign-lang.d/run.sh

@@ -0,0 +1,15 @@
+# Copyright: 2024 Masatake YAMATO
+# License: GPL-2
+
+. ../utils.sh
+
+CTAGS=$1
+
+V=
+# V=valgrind
+
+${V} ${CTAGS} --options=NONE --options=./knownz.ctags --options=./unknownx.ctags \
+	 --fields=+l \
+	 --fields-unknownx=+'{protection}{signature}' \
+	 --fields-knownz=+'{owner}' \
+	 -o - input.unknownx

Tmain/parser-own-fields-for-foreign-lang.d/stderr-expected.txt

@@ -0,0 +1 @@
+ctags: Notice: No options will be read from files or environment

Tmain/parser-own-fields-for-foreign-lang.d/stdout-expected.txt

@@ -0,0 +1,5 @@
+bar	input.unknownx	/^protected func bar(n);$/;"	f	language:unknownx	protection:protected 	signature:(n)
+baz	input.unknownx	/^private func baz(n,...);$/;"	f	language:unknownx	protection:private 	signature:(n,...)
+foo	input.unknownx	/^public func foo(n, m);$/;"	f	language:unknownx	protection:public 	signature:(n, m)
+tagme	input.unknownx	/^X:tagme@iamowner$/;"	m	language:knownz	owner:iamowner
+tagme2	input.unknownx	/^Y:iamowner2=tagme2$/;"	m	language:knownz	owner:iamowner2

Tmain/parser-own-fields-for-foreign-lang.d/unknownx.ctags

@@ -0,0 +1,10 @@
+--langdef=unknownx{_foreignLanguage=knownz}
+--kinddef-unknownx=f,func,functions
+--map-unknownx=+.unknownx
+
+--_fielddef-unknownx=protection,protections
+--_fielddef-unknownx=signature,signatures
+
+--regex-unknownx=/^((public|protected|private) +)?func ([^\(]+)\((.*)\)/\3/f/{_field=protection:\1}{_field=signature:(\4)}
+--regex-unknownx=/^X:([a-z]+)@([a-z]+)/\1/m/{_language=knownz}{_field=owner:\2}
+--regex-unknownx=/^Y:([a-z0-9]+)=([a-z0-9]+)/\2/m/{_field=owner:\1}{_language=knownz}

docs/optlib.rst

@@ -463,6 +463,10 @@ about ``--fields-<LANG>`` option.
 
 `passwd` parser is a simple example that uses ``--fields-<LANG>`` option.
 
+By default, ctags assumes the field is a part of the language specified
+with `<LANG>` in ``--regex-<LANG>``. Together with ``{_language=<LANG>}``
+flag, you can switch the language of the field. See ":ref:`foreigntag`".
+The combination of these flags is new in version 6.2.0.
 
 .. _roles:
 
@@ -1961,6 +1965,12 @@ the output for input.docc:
 	compress	input.docc	/^- function: compress(const char *plain_text, enum algorithm alg) => char *$/;"	function	language:C	roles:documented
 	decompress	input.docc	/^- function: decompress(const char *compressed_byteseq) => char *$/;"	function	language:C	roles:documented
 
+
+.. TESTCASE: Tmain/parser-own-fields-for-foreign-lang.d
+
+``{_language=<LANG>}`` flag affects ``{_field=FIELDNAME:GROUP}`` flag; ctags looks up
+the field defintion in `<LANG>`.
+
 .. END: NOT REVIEWED YET
 
 .. _optlib2c:

main/lregex.c

@@ -1134,10 +1134,14 @@ static void common_flag_field_long (const char* const s, const char* const v, vo
 	}
 
 	fname = eStrndup (v, tmp - v);
-	ftype = getFieldTypeForNameAndLanguage (fname, cdata->owner);
+
+	langType lang = (ptrn->foreign_lang == LANG_IGNORE)
+		? cdata->owner
+		: ptrn->foreign_lang;
+	ftype = getFieldTypeForNameAndLanguage (fname, lang);
 	if (ftype == FIELD_UNKNOWN)
 	{
-		error (WARNING, "no such field \"%s\" in %s", fname, getLanguageName(cdata->owner));
+		error (WARNING, "no such field \"%s\" in %s", fname, getLanguageName(lang));
 		eFree (fname);
 		return;
 	}
@@ -1269,7 +1273,7 @@ static flagDefinition commonSpecFlagDef[] = {
 	{ '\0',  EXPERIMENTAL "extra", NULL, common_flag_extra_long ,
 	  "EXTRA", "record the tag only when the extra is enabled"},
 	{ '\0',  EXPERIMENTAL "field", NULL, common_flag_field_long ,
-	  "FIELD:VALUE", "record the matched string(VALUE) to parser own FIELD of the tag"},
+	  "FIELD:VALUE", "record the matched string(VALUE) to the (foreign) language specific FIELD of the tag"},
 	{ '\0',  EXPERIMENTAL "role", NULL, common_flag_role_long,
 	  "ROLE", "set the given ROLE to the roles field"},
 	{ '\0',  EXPERIMENTAL "anonymous", NULL, common_flag_anonymous_long,