feat(textprocessing): unknown languages use eng stopwords

Author: guo-yong-zhi

src/textprocessing.jl

@@ -242,10 +242,12 @@ function processtext(counter::AbstractDict{<:AbstractString,<:Real};
 
     if stopwords == :auto
         language = detect_language(keys(counter), language)
+        lk = language
         if !haskey(STOPWORDS, language)
             @info "No built-in stopwords for $(language)!"
+            lk = "eng"
         end
-        stopwords = get(STOPWORDS, language, nothing)
+        stopwords = STOPWORDS[lk]
     end
     stopwords === nothing && (stopwords = Set{String}())
     stopwords isa AbstractSet || (stopwords = Set(stopwords))

test/test_textprocessing.jl

@@ -73,6 +73,7 @@
     @test processtext("word cloud is a cloud", language="en", stopwords_extra=["word"])[1] |> only == "cloud"
     # settokenizer! ...
     WordCloud.settokenizer!("mylang", t->split(t, "a"))
+    WordCloud.setstopwords!("mylang", [])
     @test Set(processtext("bananais", language="mylang")[1]) == Set(["b", "n", "is"])
     WordCloud.setlemmatizer!("mylang", uppercase)
     @test Set(processtext("bananais", language="mylang")[1]) == Set(["B", "N", "IS"])