Several improvements to token filter types (#4291) (#4332)

(cherry picked from commit f5dca08)

Co-authored-by: Josh Mock <joshua.mock@elastic.co>

Author: github-actions[bot]

specification/_types/analysis/StopWords.ts

@@ -17,10 +17,50 @@
  * under the License.
  */
 
+export enum StopWordLanguage {
+  _arabic_,
+  _armenian_,
+  _basque_,
+  _bengali_,
+  _brazilian_,
+  _bulgarian_,
+  _catalan_,
+  _cjk_,
+  _czech_,
+  _danish_,
+  _dutch_,
+  _english_,
+  _estonian_,
+  _finnish_,
+  _french_,
+  _galician_,
+  _german_,
+  _greek_,
+  _hindi_,
+  _hungarian_,
+  _indonesian_,
+  _irish_,
+  _italian_,
+  _latvian_,
+  _lithuanian_,
+  _norwegian_,
+  _persian_,
+  _portuguese_,
+  _romanian_,
+  _russian_,
+  _serbian_,
+  _sorani_,
+  _spanish_,
+  _swedish_,
+  _thai_,
+  _turkish_,
+  _none_
+}
+
 /**
  * Language value, such as _arabic_ or _thai_. Defaults to _english_.
  * Each language value corresponds to a predefined list of stop words in Lucene. See Stop words by language for supported language values and their stop words.
  * Also accepts an array of stop words.
  * @class_serializer: StopWordsFormatter
  */
-export type StopWords = string | string[]
+export type StopWords = StopWordLanguage | string[]

specification/_types/analysis/kuromoji-plugin.ts

@@ -19,6 +19,7 @@
 
 import { integer } from '@_types/Numeric'
 import { CharFilterBase } from './char_filters'
+import { StopWords } from './StopWords'
 import { TokenizerBase } from './tokenizers'
 import { TokenFilterBase } from './token_filters'
 
@@ -28,6 +29,11 @@ export class KuromojiAnalyzer {
   user_dictionary?: string
 }
 
+export class JaStopTokenFilter extends TokenFilterBase {
+  type: 'ja_stop'
+  stopwords?: StopWords
+}
+
 export class KuromojiIterationMarkCharFilter extends CharFilterBase {
   type: 'kuromoji_iteration_mark'
   normalize_kana: boolean

specification/_types/analysis/languages.ts

@@ -18,25 +18,30 @@
  */
 
 export enum SnowballLanguage {
+  Arabic,
   Armenian,
   Basque,
   Catalan,
   Danish,
   Dutch,
   English,
+  Estonian,
   Finnish,
   French,
   German,
   German2,
   Hungarian,
   Italian,
+  Irish,
   Kp,
+  Lithuanian,
   Lovins,
   Norwegian,
   Porter,
   Portuguese,
   Romanian,
   Russian,
+  Serbian,
   Spanish,
   Swedish,
   Turkish

specification/_types/analysis/nori-plugin.ts

@@ -18,6 +18,7 @@
  */
 
 import { TokenizerBase } from './tokenizers'
+import { TokenFilterBase } from './token_filters'
 
 export enum NoriDecompoundMode {
   discard,
@@ -32,3 +33,9 @@ export class NoriTokenizer extends TokenizerBase {
   user_dictionary?: string
   user_dictionary_rules?: string[]
 }
+
+export class NoriPartOfSpeechTokenFilter extends TokenFilterBase {
+  type: 'nori_part_of_speech'
+  /** An array of part-of-speech tags that should be removed. */
+  stoptags?: string[]
+}