From ac70c66127cf060b3cef5b9498c5def9d50c4544 Mon Sep 17 00:00:00 2001 From: Pierre-Thomas Meisels Date: Mon, 20 Jan 2025 14:13:15 +0100 Subject: [PATCH 01/32] feat: Add java and kotlin parser to get fqdn from source code --- .../tests/JavaClassShouldBeParsedTest.gdj | 24 +++ .../tests/KotlinClassShouldBeParsedTest.gdj | 24 +++ .../tests/JavaClassShouldBeParsedTest.java | 40 +++++ .../tests/KotlinClassShouldBeParsedTest.kt | 35 ++++ .../jvm_resource_format_loader.cpp | 4 + src/script/jvm_script.cpp | 165 +++++++++++++++++- src/script/jvm_script.h | 17 +- src/script/jvm_script_manager.cpp | 20 +-- src/script/jvm_script_manager.h | 10 +- src/script/language/java_script.h | 2 +- src/script/language/kotlin_script.h | 2 +- 11 files changed, 322 insertions(+), 21 deletions(-) create mode 100644 harness/tests/scripts/godot/tests/JavaClassShouldBeParsedTest.gdj create mode 100644 harness/tests/scripts/godot/tests/KotlinClassShouldBeParsedTest.gdj create mode 100644 harness/tests/src/main/java/godot/tests/JavaClassShouldBeParsedTest.java create mode 100644 harness/tests/src/main/kotlin/godot/tests/KotlinClassShouldBeParsedTest.kt diff --git a/harness/tests/scripts/godot/tests/JavaClassShouldBeParsedTest.gdj b/harness/tests/scripts/godot/tests/JavaClassShouldBeParsedTest.gdj new file mode 100644 index 0000000000..d45fa6c620 --- /dev/null +++ b/harness/tests/scripts/godot/tests/JavaClassShouldBeParsedTest.gdj @@ -0,0 +1,24 @@ +// THIS FILE IS GENERATED! DO NOT EDIT OR DELETE IT. EDIT OR DELETE THE ASSOCIATED SOURCE CODE FILE INSTEAD +// Note: You can however freely move this file inside your godot project if you want. Keep in mind however, that if you rename the originating source code file, this file will be deleted and regenerated as a new file instead of being updated! Other modifications to the source file however, will result in this file being updated. + +registeredName = JavaClassShouldBeParsedTest +fqName = godot.tests.JavaClassShouldBeParsedTest +relativeSourcePath = src/main/java/godot/tests/JavaClassShouldBeParsedTest.java +baseType = Node +supertypes = [ + godot.Node, + godot.Object, + godot.core.KtObject, + godot.common.interop.NativeWrapper, + godot.common.interop.NativePointer, + kotlin.Any +] +signals = [ + +] +properties = [ + +] +functions = [ + +] \ No newline at end of file diff --git a/harness/tests/scripts/godot/tests/KotlinClassShouldBeParsedTest.gdj b/harness/tests/scripts/godot/tests/KotlinClassShouldBeParsedTest.gdj new file mode 100644 index 0000000000..c1b8af3b7a --- /dev/null +++ b/harness/tests/scripts/godot/tests/KotlinClassShouldBeParsedTest.gdj @@ -0,0 +1,24 @@ +// THIS FILE IS GENERATED! DO NOT EDIT OR DELETE IT. EDIT OR DELETE THE ASSOCIATED SOURCE CODE FILE INSTEAD +// Note: You can however freely move this file inside your godot project if you want. Keep in mind however, that if you rename the originating source code file, this file will be deleted and regenerated as a new file instead of being updated! Other modifications to the source file however, will result in this file being updated. + +registeredName = KotlinClassShouldBeParsedTest +fqName = godot.tests.KotlinClassShouldBeParsedTest +relativeSourcePath = src/main/kotlin/godot/tests/KotlinClassShouldBeParsedTest.kt +baseType = Node +supertypes = [ + godot.Node, + godot.Object, + godot.core.KtObject, + godot.common.interop.NativeWrapper, + godot.common.interop.NativePointer, + kotlin.Any +] +signals = [ + +] +properties = [ + +] +functions = [ + +] \ No newline at end of file diff --git a/harness/tests/src/main/java/godot/tests/JavaClassShouldBeParsedTest.java b/harness/tests/src/main/java/godot/tests/JavaClassShouldBeParsedTest.java new file mode 100644 index 0000000000..61b88951c2 --- /dev/null +++ b/harness/tests/src/main/java/godot/tests/JavaClassShouldBeParsedTest.java @@ -0,0 +1,40 @@ +// package +/* * + * package + */ + +package /**/godot.tests//fdsfdsfs + ; + +//// dfsdfsfdsfd + +/** + * dfdsfs + */ + +/** + * fsdfs + */ + +import godot.Node; +import godot.annotation.RegisterClass; + +/*** + * dfsdfs + */ +@RegisterClass/*class*/()/*class*/ //class +/** + * class + * \/class + */ + + +// class +public class /*class*///class + //class + /** + * class + */ + JavaClassShouldBeParsedTest//lol + extends Node { +} diff --git a/harness/tests/src/main/kotlin/godot/tests/KotlinClassShouldBeParsedTest.kt b/harness/tests/src/main/kotlin/godot/tests/KotlinClassShouldBeParsedTest.kt new file mode 100644 index 0000000000..db6b531e42 --- /dev/null +++ b/harness/tests/src/main/kotlin/godot/tests/KotlinClassShouldBeParsedTest.kt @@ -0,0 +1,35 @@ +// package +/** + * package + */ + +package/**/ godot.tests//fdfsdsfdfsdgf + +import godot.Node +import godot.annotation.RegisterClass + +//fdfqsdfs + +class ShouldNotBeProcessed { + +} + +enum class HereForTrollingParser { + classTroll, + CLASS +} + +/** + * + */ + +/** + * + */ +@RegisterClass (className = ("KotlinClassShouldBeParsedTest")/**/ ) /**class**//*class*///class +/** + * class + */ +// class +class /*class*/ KotlinClassShouldBeParsedTest: Node() { +} \ No newline at end of file diff --git a/src/resource_format/jvm_resource_format_loader.cpp b/src/resource_format/jvm_resource_format_loader.cpp index 2becfafa4e..35cb2397d0 100644 --- a/src/resource_format/jvm_resource_format_loader.cpp +++ b/src/resource_format/jvm_resource_format_loader.cpp @@ -80,6 +80,10 @@ Ref JvmResourceFormatLoader::load(const String& p_path, const String& if (r_error) { *r_error = load_err; } jvm_script->set_source_code(source_code); + if (is_source) { + JVM_LOG_INFO(vformat("fqdn: %s", dynamic_cast(jvm_script.ptr())->get_functional_name())); + } + if (!script_is_new && is_source) { String path = jvm_script->get_path(); MessageQueue::get_singleton()->push_callable( diff --git a/src/script/jvm_script.cpp b/src/script/jvm_script.cpp index 35aadd4125..a3e7c0bb17 100644 --- a/src/script/jvm_script.cpp +++ b/src/script/jvm_script.cpp @@ -285,9 +285,172 @@ JvmScript::~JvmScript() { kotlin_class = nullptr; } -StringName PathScript::get_global_name() const { +StringName SourceScript::get_functional_name() const { + static String package_keyword { PACKAGE_KEYWORD }; + static int package_keyword_size { package_keyword.size() }; + + int initial_start_index = 0; + while (skip_comments(initial_start_index) || skip_spaces_and_newlines(initial_start_index)) {} + + int package_keyword_index { source.find(package_keyword, initial_start_index) }; + + String package_name; + if (package_keyword_index != -1) { + int package_start_index = package_keyword_index + package_keyword_size - 1; + + while (skip_comments(package_start_index) || skip_spaces_and_newlines(package_start_index)) {} + + char32_t next_character = source[package_start_index]; + int package_end_index = package_start_index; + while (package_end_index < source.size() && !is_package_end(next_character)) { + next_character = source[++package_end_index]; + } + + package_name = source.substr(package_start_index, package_end_index - package_start_index); + } + + static String register_class_annotation { REGISTER_CLASS_ANNOTATION }; + static int register_class_annotation_size { register_class_annotation.size() }; + int register_class_search_start = package_keyword_index == -1 ? 0 : package_keyword_index; + + while (skip_comments(register_class_search_start) || skip_spaces_and_newlines(register_class_search_start)) {} + + int register_class_index { source.find(register_class_annotation, register_class_search_start) }; + + if (register_class_index == -1) { + JVM_LOG_WARNING(vformat("Cannot find registered class in %s", get_path())); + return StringName(); + } + + int class_search_start_index = register_class_index + register_class_annotation_size - 1; + + while (skip_comments(class_search_start_index) || skip_spaces_and_newlines(class_search_start_index)) {} + + char32_t next_character = source[class_search_start_index]; + if (next_character == U'(') { + next_character = source[++class_search_start_index]; + + while (class_search_start_index < source.size() && next_character != U')') { + next_character = source[++class_search_start_index]; + } + + while (class_search_start_index < source.size() && next_character == U')') { + ++class_search_start_index; + while (skip_comments(class_search_start_index) || skip_spaces_and_newlines(class_search_start_index)) {} + next_character = source[class_search_start_index]; + } + } + + while (skip_comments(class_search_start_index) || skip_spaces_and_newlines(class_search_start_index)) {} + + static String class_keyword { CLASS_KEYWORD }; + static int class_keyword_size { class_keyword.size() }; + int class_keyword_index { source.find(class_keyword, class_search_start_index) }; + + if (class_keyword_index == -1) { + JVM_LOG_WARNING(vformat("Cannot find class declaration in %s", get_path())); + return StringName(); + } + + int class_start_index = class_keyword_index + class_keyword_size - 1; + + while (skip_comments(class_start_index) || skip_spaces_and_newlines(class_start_index)) {} + + next_character = source[class_start_index]; + int class_end_index = class_start_index; + while (class_end_index < source.size() && !is_class_name_end(next_character)) { + next_character = source[++class_end_index]; + } + + String class_name { source.substr(class_start_index, class_end_index - class_start_index) }; + + if (package_name.is_empty()) { + return class_name; + } + + return vformat("%s.%s", package_name, class_name); +} + + +StringName SourceScript::get_global_name() const { return {}; } +// +bool SourceScript::is_whitespace_or_linebreak(char32_t character) { + return is_whitespace(character) || is_linebreak(character); +} + +bool SourceScript::is_package_end(char32_t character) { + return is_whitespace_or_linebreak(character) || character == U';' || character == U'/'; +} + +bool SourceScript::is_class_name_end(char32_t character) { + return is_whitespace_or_linebreak(character) || + character == U':' || + character == U'{' || + character == U'<' || + character == U'[' || + character == U'(' || + character == U'/'; +} + +bool SourceScript::skip_spaces_and_newlines(int& start_index) const { + int initial_index = start_index; + char32_t next_character = source[start_index]; + + while (start_index < source.size() && is_whitespace_or_linebreak(next_character)) { + next_character = source[++start_index]; + } + + return start_index != initial_index; +} + +bool SourceScript::skip_comments(int& start_index) const { + char32_t next_character = source[start_index]; + + if (next_character != U'/') { + return false; + } + + bool isLineComment = false; + bool isMultilineComment = false; + if (start_index < source.size() - 1) { + next_character = source[++start_index]; + + isLineComment = next_character == U'/'; + isMultilineComment = next_character == U'*'; + } + + if (!isLineComment && !isMultilineComment) { + JVM_LOG_WARNING(vformat("Cannot parse %s, found unexpected '/' character", get_path())); + } + + if (isLineComment) { + while (start_index < source.size() && !is_linebreak(next_character)) { + next_character = source[++start_index]; + } + } + + if (isMultilineComment) { + while (start_index < source.size()) { + bool isCommentEnd = next_character == U'*'; + next_character = source[++start_index]; + + if (start_index == source.size()) { + JVM_LOG_WARNING(vformat("Cannot parse %s, found unclosed multiline comment", get_path())); + } + + isCommentEnd = isCommentEnd && next_character == U'/'; + + if (isCommentEnd) { + ++start_index; + break; + } + } + } + + return true; +} NamedScript::~NamedScript() { // The named script is the one that should delete the KtClass as its existence is guaranteed unlike its path based twin. diff --git a/src/script/jvm_script.h b/src/script/jvm_script.h index 95bf716bb6..09d5631cd7 100644 --- a/src/script/jvm_script.h +++ b/src/script/jvm_script.h @@ -81,11 +81,22 @@ class JvmScript : public Script { static void _bind_methods(); }; -class PathScript : public JvmScript { +class SourceScript : public JvmScript { public: - PathScript() = default; - ~PathScript() override = default; + SourceScript() = default; + ~SourceScript() override = default; + StringName get_functional_name() const; StringName get_global_name() const override; + +protected: + static constexpr const char* PACKAGE_KEYWORD = "package"; + static constexpr const char* CLASS_KEYWORD = "class"; + static constexpr const char* REGISTER_CLASS_ANNOTATION = "@RegisterClass"; + static bool is_whitespace_or_linebreak(char32_t character); + static bool is_package_end(char32_t character); + static bool is_class_name_end(char32_t character); + bool skip_spaces_and_newlines(int& start_index) const; + bool skip_comments(int& start_index) const; }; class NamedScript : public JvmScript { diff --git a/src/script/jvm_script_manager.cpp b/src/script/jvm_script_manager.cpp index c9c0a80bcf..cfb941fccf 100644 --- a/src/script/jvm_script_manager.cpp +++ b/src/script/jvm_script_manager.cpp @@ -22,7 +22,7 @@ void JvmScriptManager::create_and_update_scripts(Vector& classes) { name_to_filepath_map.clear(); filepath_to_name_map.clear(); - HashMap> path_script_cache = path_scripts_map; + HashMap> path_script_cache = path_scripts_map; path_scripts.clear(); path_scripts_map.clear(); #endif @@ -69,7 +69,7 @@ void JvmScriptManager::create_and_update_scripts(Vector& classes) { name_to_filepath_map[kotlin_class->registered_class_name] = script_path; filepath_to_name_map[script_path] = kotlin_class->registered_class_name; - Ref path_script; + Ref path_script; #ifdef TOOLS_ENABLED // Try to find if a matching PathScript exist if (name_to_filepath_cache.has(script_name)) { @@ -92,7 +92,7 @@ void JvmScriptManager::create_and_update_scripts(Vector& classes) { path_script->set_path(script_path, true); } else { #endif - path_script = Ref(ResourceLoader::load(script_path)); + path_script = Ref(ResourceLoader::load(script_path)); path_script->kotlin_class = kotlin_class; #ifdef TOOLS_ENABLED } @@ -123,8 +123,8 @@ void JvmScriptManager::create_and_update_scripts(Vector& classes) { } // We do the same with PathScripts - for (const KeyValue>& keyValue : path_script_cache) { - Ref path_script {keyValue.value}; + for (const KeyValue>& keyValue : path_script_cache) { + Ref path_script {keyValue.value}; String path {keyValue.key}; // No need to delete the KotlinClass, it has already been done with the NamedScript that shares it. path_script->kotlin_class = nullptr; @@ -150,7 +150,7 @@ Ref JvmScriptManager::get_named_script_from_index(int p_index) cons return named_scripts[p_index]; } -Ref JvmScriptManager::get_named_script_from_pathScript(Ref pathScript) const { +Ref JvmScriptManager::get_named_script_from_pathScript(Ref pathScript) const { String path = pathScript->get_path(); if (filepath_to_name_map.has(path)) { @@ -167,8 +167,8 @@ Ref JvmScriptManager::get_script_from_name(const StringName& name) return {}; } -Ref JvmScriptManager::get_script_from_path(const String& p_path) const { - if (HashMap>::ConstIterator element = path_scripts_map.find(p_path)) { +Ref JvmScriptManager::get_script_from_path(const String& p_path) const { + if (HashMap>::ConstIterator element = path_scripts_map.find(p_path)) { return element->value; } return {}; @@ -181,7 +181,7 @@ void JvmScriptManager::update_all_scripts(uint64_t update_time) { ptr->update_script_exports(); ptr->set_last_time_source_modified(update_time); } - for (const Ref& path_script : path_scripts) { + for (const Ref& path_script : path_scripts) { JvmScript* ptr = path_script.ptr(); ptr->update_script_exports(); ptr->set_last_time_source_modified(update_time); @@ -189,7 +189,7 @@ void JvmScriptManager::update_all_scripts(uint64_t update_time) { } void JvmScriptManager::invalidate_source(String path) { - Ref path_script = get_script_from_path(path); + Ref path_script = get_script_from_path(path); if (path_script.is_null()) { return; } uint64_t last_modified = path_script->get_last_modified_time(); diff --git a/src/script/jvm_script_manager.h b/src/script/jvm_script_manager.h index 12097305f5..0e2389a461 100644 --- a/src/script/jvm_script_manager.h +++ b/src/script/jvm_script_manager.h @@ -13,8 +13,8 @@ class JvmScriptManager: public Object { HashMap filepath_to_name_map; HashMap name_to_filepath_map; - Vector> path_scripts; - HashMap> path_scripts_map; + Vector> path_scripts; + HashMap> path_scripts_map; #ifdef TOOLS_ENABLED uint64_t last_reload = 0; @@ -37,8 +37,8 @@ class JvmScriptManager: public Object { Ref get_script_from_name(const StringName& name) const; Ref get_named_script_from_index(int p_index) const; - Ref get_named_script_from_pathScript(Ref pathScript) const; - Ref get_script_from_path(const String& p_path) const; + Ref get_named_script_from_pathScript(Ref pathScript) const; + Ref get_script_from_path(const String& p_path) const; template Ref