Add unwrapping and inlining to expand-generics (#2504)

* Add unwrapping and inlining to expand-generics

* Make linter happy

* Add unwrapping and inlining to expand-generics (Rust edition)

* Update OpenAPI spec with inlined WithNullValue

---------

Co-authored-by: Laurent Saint-Félix <laurent.saintfelix@elastic.co>
(cherry picked from commit 2f1902b)

Author: 2 people

compiler-rs/clients_schema/src/builtins.rs

@@ -15,23 +15,30 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use once_cell::sync::Lazy;
-
 use crate::TypeName;
+use crate::type_name;
+
+macro_rules! declare_type_name {
+    ($id:ident,$namespace:expr,$name:expr) => {
+        pub const $id: TypeName = type_name!($namespace, $name);
+    };
+}
+
+declare_type_name!(STRING, "_builtins", "string");
+declare_type_name!(BOOLEAN, "_builtins", "boolean");
+declare_type_name!(OBJECT, "_builtins", "object");
+declare_type_name!(BINARY, "_builtins", "binary");
+declare_type_name!(VOID, "_builtins", "void");
+declare_type_name!(NUMBER, "_builtins", "number");
+declare_type_name!(BYTE, "_builtins", "byte");
+declare_type_name!(INTEGER, "_builtins", "integer");
+declare_type_name!(LONG, "_builtins", "long");
+declare_type_name!(FLOAT, "_builtins", "float");
+declare_type_name!(DOUBLE, "_builtins", "double");
+declare_type_name!(NULL, "_builtins", "null");
+declare_type_name!(DICTIONARY, "_builtins", "Dictionary");
+declare_type_name!(USER_DEFINED, "_builtins", "UserDefined");
 
-pub static STRING: Lazy<TypeName> = Lazy::new(|| TypeName::new("_builtins", "string"));
-pub static BOOLEAN: Lazy<TypeName> = Lazy::new(|| TypeName::new("_builtins", "boolean"));
-pub static OBJECT: Lazy<TypeName> = Lazy::new(|| TypeName::new("_builtins", "object"));
-pub static BINARY: Lazy<TypeName> = Lazy::new(|| TypeName::new("_builtins", "binary"));
-pub static VOID: Lazy<TypeName> = Lazy::new(|| TypeName::new("_builtins", "void"));
-pub static NUMBER: Lazy<TypeName> = Lazy::new(|| TypeName::new("_builtins", "number"));
-pub static BYTE: Lazy<TypeName> = Lazy::new(|| TypeName::new("_builtins", "byte"));
-pub static INTEGER: Lazy<TypeName> = Lazy::new(|| TypeName::new("_builtins", "integer"));
-pub static LONG: Lazy<TypeName> = Lazy::new(|| TypeName::new("_builtins", "long"));
-pub static FLOAT: Lazy<TypeName> = Lazy::new(|| TypeName::new("_builtins", "float"));
-pub static DOUBLE: Lazy<TypeName> = Lazy::new(|| TypeName::new("_builtins", "double"));
-pub static NULL: Lazy<TypeName> = Lazy::new(|| TypeName::new("_builtins", "null"));
-pub static DICTIONARY: Lazy<TypeName> = Lazy::new(|| TypeName::new("_builtins", "Dictionary"));
-pub static USER_DEFINED: Lazy<TypeName> = Lazy::new(|| TypeName::new("_builtins", "UserDefined"));
+declare_type_name!(ADDITIONAL_PROPERTIES, "_spec_utils", "AdditionalProperties");
 
-pub static ADDITIONAL_PROPERTIES: Lazy<TypeName> = Lazy::new(|| TypeName::new("_spec_utils", "AdditionalProperties"));
+declare_type_name!(WITH_NULL_VALUE, "_spec_utils", "WithNullValue");

compiler-rs/clients_schema/src/lib.rs

@@ -73,6 +73,17 @@ impl TypeName {
     }
 }
 
+/// Creates a constant `TypeName` from static strings
+#[macro_export]
+macro_rules! type_name {
+    ($namespace:expr,$name:expr) => {
+        TypeName {
+            namespace: arcstr::literal!($namespace),
+            name: arcstr::literal!($name),
+        }
+    };
+}
+
 impl Display for TypeName {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         write!(f, "{}:{}", self.namespace, self.name)
@@ -93,12 +104,6 @@ pub enum ValueOf {
     LiteralValue(LiteralValue),
 }
 
-impl ValueOf {
-    pub fn instance_of(name: TypeName) -> ValueOf {
-        ValueOf::InstanceOf(InstanceOf::new(name))
-    }
-}
-
 impl From<TypeName> for ValueOf {
     fn from(name: TypeName) -> Self {
         ValueOf::InstanceOf(InstanceOf::new(name))

compiler-rs/clients_schema/src/transform/expand_generics.rs

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
 
 use anyhow::bail;
 use indexmap::IndexMap;
@@ -26,22 +26,43 @@ use crate::*;
 struct Ctx {
     new_types: IndexMap<TypeName, TypeDefinition>,
     types_seen: std::collections::HashSet<TypeName>,
+    config: ExpandConfig,
 }
 
 /// Generic parameters of a type
 type GenericParams = Vec<TypeName>;
-/// Generic arguments for an instanciated generic type
+/// Generic arguments for an instantiated generic type
 type GenericArgs = Vec<ValueOf>;
 /// Mapping from generic arguments to values
 type GenericMapping = HashMap<TypeName, ValueOf>;
 
-/// Expand all generics by creating new concrete types for every instanciation of a generic type.
+#[derive(Clone, Debug)]
+pub struct ExpandConfig {
+    /// Generic types that will be inlined by replacing them with their definition, propagating generic arguments.
+    pub unwrap: HashSet<TypeName>,
+    // Generic types that will be unwrapped by replacing them with their (single) generic parameter.
+    pub inline: HashSet<TypeName>,
+}
+
+impl Default for ExpandConfig {
+    fn default() -> Self {
+        ExpandConfig {
+            unwrap: Default::default(),
+            inline: HashSet::from([builtins::WITH_NULL_VALUE])
+        }
+    }
+}
+
+/// Expand all generics by creating new concrete types for every instantiation of a generic type.
 ///
 /// The resulting model has no generics anymore. Top-level generic parameters (e.g. SearchRequest's TDocument) are
 /// replaced by user_defined_data.
-pub fn expand_generics(model: IndexedModel) -> anyhow::Result<IndexedModel> {
+pub fn expand(model: IndexedModel, config: ExpandConfig) -> anyhow::Result<IndexedModel> {
     let mut model = model;
-    let mut ctx = Ctx::default();
+    let mut ctx = Ctx {
+        config,
+        ..Ctx::default()
+    };
 
     for endpoint in &model.endpoints {
         for name in [&endpoint.request, &endpoint.response].into_iter().flatten() {
@@ -317,6 +338,14 @@ pub fn expand_generics(model: IndexedModel) -> anyhow::Result<IndexedModel> {
                     return Ok(p.clone());
                 }
 
+                // Inline or unwrap if required by the config
+                if ctx.config.inline.contains(&inst.typ) {
+                    return inline_generic_type(inst, mappings, model, ctx);
+                }
+                if ctx.config.unwrap.contains(&inst.typ) {
+                    return unwrap_generic_type(inst, mappings, model, ctx);
+                }
+
                 // Expand generic parameters, if any
                 let args = inst
                     .generics
@@ -346,6 +375,51 @@ pub fn expand_generics(model: IndexedModel) -> anyhow::Result<IndexedModel> {
         }
     }
 
+    /// Inlines a value of a generic type by replacing it with its definition, propagating
+    /// generic arguments.
+    fn inline_generic_type(
+        value: &InstanceOf,
+        _mappings: &GenericMapping,
+        model: &IndexedModel,
+        ctx: &mut Ctx,
+    ) -> anyhow::Result<ValueOf> {
+
+        // It has to be an alias (e.g. WithNullValue)
+        if let TypeDefinition::TypeAlias(inline_def) = model.get_type(&value.typ)? {
+            // Create mappings to resolve types in the inlined type's definition
+            let mut inline_mappings = GenericMapping::new();
+            for (source, dest) in inline_def.generics.iter().zip(value.generics.iter()) {
+                inline_mappings.insert(source.clone(), dest.clone());
+            }
+            // and expand the inlined type's alias definition
+            let result = expand_valueof(&inline_def.typ, &inline_mappings, model, ctx)?;
+            return Ok(result);
+        } else {
+            bail!("Expecting inlined type {} to be an alias", &value.typ);
+        }
+    }
+
+    /// Unwraps a value of a generic type by replacing it with its generic parameter
+    fn unwrap_generic_type(
+        value: &InstanceOf,
+        mappings: &GenericMapping,
+        model: &IndexedModel,
+        ctx: &mut Ctx,
+    ) -> anyhow::Result<ValueOf> {
+
+        // It has to be an alias (e.g. Stringified)
+        if let TypeDefinition::TypeAlias(_unwrap_def) = model.get_type(&value.typ)? {
+            // Expand the inlined type's generic argument (there must be exactly one)
+            if value.generics.len() != 1 {
+                bail!("Expecting unwrapped type {} to have exactly one generic parameter", &value.typ);
+            }
+            let result = expand_valueof(&value.generics[0], mappings, model, ctx)?;
+            return Ok(result);
+        } else {
+            bail!("Expecting unwrapped type {} to be an alias", &value.typ);
+        }
+    }
+
     //---------------------------------------------------------------------------------------------
     // Misc
     //---------------------------------------------------------------------------------------------
@@ -422,12 +496,12 @@ mod tests {
 
         let schema_json = std::fs::read_to_string("../../output/schema/schema.json")?;
         let model: IndexedModel = serde_json::from_str(&schema_json)?;
-        let model = expand_generics(model)?;
+        let model = expand(model, ExpandConfig::default())?;
 
         let json_no_generics = serde_json::to_string_pretty(&model)?;
 
         if canonical_json != json_no_generics {
-            std::fs::create_dir("test-output")?;
+            std::fs::create_dir_all("test-output")?;
             let mut out = std::fs::File::create("test-output/schema-no-generics-canonical.json")?;
             out.write_all(canonical_json.as_bytes())?;
 

compiler-rs/clients_schema/src/transform/mod.rs

@@ -17,14 +17,13 @@
 
 //! Utilities to transform API models and common transformations:
 //! * filtering according to availability
+//! * expand generic types so that the model doesn't contain generic types anymore
 
 mod availability;
 mod expand_generics;
 
 use std::collections::HashSet;
 
-use availability::Availability;
-
 use crate::{Availabilities, IndexedModel, TypeName};
 
 /// The working state of a type graph traversal algorithm. It keeps track of the types that
@@ -67,6 +66,7 @@ impl Iterator for Worksheet {
     }
 }
 
+pub use availability::Availability;
 /// Transform a model to only keep the endpoints and types that match a predicate on the `availability`
 /// properties.
 pub fn filter_availability(
@@ -76,6 +76,8 @@ pub fn filter_availability(
     Availability::filter(model, avail_filter)
 }
 
-pub fn expand_generics(model: IndexedModel) -> anyhow::Result<IndexedModel> {
-    expand_generics::expand_generics(model)
+
+pub use expand_generics::ExpandConfig;
+pub fn expand_generics(model: IndexedModel, config: ExpandConfig) -> anyhow::Result<IndexedModel> {
+    expand_generics::expand(model, config)
 }

compiler-rs/clients_schema_to_openapi/src/main.rs

@@ -87,8 +87,10 @@ impl Cli {
                     },
                 };
 
-                model = clients_schema::transform::expand_generics(model)?;
-                model = clients_schema::transform::filter_availability(model, filter)?;
+                use clients_schema::transform::*;
+
+                model = expand_generics(model, ExpandConfig::default())?;
+                model = filter_availability(model, filter)?;
             }
         }
 

compiler-rs/compiler-wasm-lib/pkg/compiler_wasm_lib_bg.wasm

@@ -18,6 +18,7 @@
 use anyhow::bail;
 use clients_schema::{Availabilities, Visibility};
 use wasm_bindgen::prelude::*;
+use clients_schema::transform::ExpandConfig;
 
 #[cfg(all(not(target_arch = "wasm32"), not(feature = "cargo-clippy")))]
 compile_error!("To build this crate use `make compiler-wasm-lib`");
@@ -43,7 +44,7 @@ fn convert0(json: &str, flavor: &str) -> anyhow::Result<String> {
     };
 
     let mut schema = clients_schema::IndexedModel::from_reader(json.as_bytes())?;
-    schema = clients_schema::transform::expand_generics(schema)?;
+    schema = clients_schema::transform::expand_generics(schema, ExpandConfig::default())?;
     if let Some(filter) = filter {
         schema = clients_schema::transform::filter_availability(schema, filter)?;
     }

compiler-rs/compiler-wasm-lib/src/lib.rs

@@ -360,7 +360,7 @@ fn generate_dictionary_def(
         typ: ValueOf::InstanceOf(InstanceOf {
             typ: builtins::DICTIONARY.clone(),
             generics: vec![
-                ValueOf::instance_of(builtins::STRING.clone()),
+                ValueOf::from(builtins::STRING.clone()),
                 match value {
                     AdditionalProperties::Any(_) => (&builtins::USER_DEFINED).into(),
 

compiler-rs/openapi_to_clients_schema/src/types.rs

@@ -35,16 +35,41 @@ import { sortTypeDefinitions } from '../model/utils'
 import { argv } from 'zx'
 import { join } from 'path'
 
+export class ExpansionConfig {
+  unwrappedTypes?: TypeName[] | string[]
+  inlinedTypes?: TypeName[] | string[]
+}
+
 /**
- * Expand all generics by creating new concrete types for every instanciation of a generic type.
+ * Expand all generics by creating new concrete types for every instantiation of a generic type.
  *
  * The resulting model has no generics anymore. Top-level generic parameters (e.g. SearchRequest's TDocument) are
  * replaced by user_defined_data.
  *
  * @param inputModel the input model
+ * @param unwrappedTypes types that should not be expanded but unwrapped as their generic parameter.
  * @return a new model with generics expanded
  */
-export function expandGenerics (inputModel: Model): Model {
+export function expandGenerics (inputModel: Model, config?: ExpansionConfig): Model {
+  const typesToUnwrap = new Set<string>()
+  const typesToInline: Set<string> = new Set<string>()
+
+  for (const name of config?.unwrappedTypes ?? []) {
+    if (typeof name === 'string') {
+      typesToUnwrap.add(name)
+    } else {
+      typesToUnwrap.add(nameKey(name))
+    }
+  }
+
+  for (const name of config?.inlinedTypes ?? []) {
+    if (typeof name === 'string') {
+      typesToInline.add(name)
+    } else {
+      typesToInline.add(nameKey(name))
+    }
+  }
+
   const typesSeen = new Set<string>()
 
   const types = new Array<TypeDefinition>()
@@ -338,6 +363,35 @@ export function expandGenerics (inputModel: Model): Model {
         }
 
       case 'instance_of': {
+        const valueOfType = nameKey(value.type)
+
+        // If this is a type that has to be unwrapped, return its generic parameter's type
+        if (typesToUnwrap.has(valueOfType)) {
+          // @ts-expect-error
+          const x = value.generics[0]
+          return expandValueOf(x, mappings)
+        }
+
+        // If this is a type that has to be inlined
+        if (typesToInline.has(valueOfType)) {
+          // It has to be an alias (e.g. Stringified or WithNullValue
+          const inlinedTypeDef = inputTypeByName.get(valueOfType)
+          if (inlinedTypeDef?.kind !== 'type_alias') {
+            throw Error(`Inlined type ${valueOfType} should be an alias definition`)
+          }
+
+          const inlineMappings = new Map<string, ValueOf>()
+          for (let i = 0; i < (inlinedTypeDef.generics?.length ?? 0); i++) {
+            // @ts-expect-error
+            const source = inlinedTypeDef.generics[i]
+            // @ts-expect-error
+            const dest = value.generics[i]
+            inlineMappings.set(nameKey(source), dest)
+          }
+
+          return expandValueOf(inlinedTypeDef.type, inlineMappings)
+        }
+
         // If this is a generic parameter, return its mapping
         const mapping = mappings.get(nameKey(value.type))
         if (mapping !== undefined) {
@@ -467,7 +521,10 @@ async function expandGenericsFromFile (inPath: string, outPath: string): Promise
   )
 
   const inputModel = JSON.parse(inputText)
-  const outputModel = expandGenerics(inputModel)
+  const outputModel = expandGenerics(inputModel, {
+    // unwrappedTypes: ["_spec_utils:Stringified"],
+    inlinedTypes: ['_spec_utils:WithNullValue']
+  })
 
   await writeFile(
     outPath,