1 /**
2  * Methods and classes used to generate classes and other data structures representing common
3  * schemas in an OpenAPI Specification.
4  *
5  * See_Also: https://spec.openapis.org/oas/latest.html#schema-object
6  */
7 module openapi_client.schemas;
8 
9 import vibe.data.json : Json, deserializeJson;
10 import vibe.core.log : logDebug;
11 
12 import std.container.rbtree : RedBlackTree;
13 import std.file : mkdir, mkdirRecurse, write;
14 import std.array : array, appender, split, Appender;
15 import std.algorithm : canFind, skipOver;
16 import std.path : buildNormalizedPath, dirName;
17 import std.string : tr;
18 import std.range : tail, takeOne;
19 import std.stdio : writeln;
20 
21 import openapi : OasDocument, OasSchema;
22 import openapi_client.util : toUpperCamelCase, toLowerCamelCase, wordWrapText;
23 
24 /**
25  * Descriptive information about a OpenAPI schema and the Dlang code that represents it.
26  *
27  * Information in this class is used during code generation via [jsonSchemaByRef].
28  */
29 class JsonSchema {
30   /**
31    * The name of the schema as per the OpenAPI Specification.
32    */
33   string schemaName;
34 
35   /**
36    * The name of the Dlang module that contains classes for the schema.
37    */
38   string moduleName;
39 
40   /**
41    * The name of the Dlang class representing this schema.
42    */
43   string className;
44 
45   /**
46    * The OpenAPI Specification data representing the schema, in case it needs re-processing.
47    */
48   OasSchema schema;
49 }
50 
51 JsonSchema[string] jsonSchemaByRef;
52 
53 /**
54  * In our code generation, the module is a file name which contains a class whose
55  * name is in CamelCase.
56  */
57 string getClassNameFromSchemaName(string schemaName) {
58   static immutable RedBlackTree!string RESERVED_CLASSES = new RedBlackTree!string([
59         "cpp_type_info_ptr",
60         "Error",
61         "Exception",
62         "Object",
63         "Throwable",
64         "TypeInfo",
65         "TypeInfo_Array",
66         "TypeInfo_AssociativeArray",
67         "TypeInfo_Class",
68         "TypeInfo_Const",
69         "TypeInfo_Delegate",
70         "TypeInfo_Enum",
71         "TypeInfo_Function",
72         "TypeInfo_Interface",
73         "TypeInfo_Invariant",
74         "TypeInfo_Pointer",
75         "TypeInfo_Shared",
76         "TypeInfo_StaticArray",
77         "TypeInfo_Struct",
78         "TypeInfo_Tuple",
79         "TypeInfo_Vector",
80         "TypeInfo_Wild"
81       ]);
82   string className = toUpperCamelCase(tr(schemaName, ".", "_"));
83   if (className in RESERVED_CLASSES)
84     return className ~ "_";
85   else
86     return className;
87 }
88 
89 /**
90  * Produce the full module name for a given schemaName and package.
91  *
92  * Params:
93  *   packageRoot = The D base-package for the modules, e.g. "stripe.openapi".
94  *   schemaName = The OpenAPI Spec schema name, e.g. "transfer_data".
95  */
96 string getModuleNameFromSchemaName(string packageRoot, string schemaName) {
97   if (packageRoot is null)
98     return schemaName;
99   else
100     return packageRoot ~ ".model." ~ schemaName;
101 }
102 
103 /**
104  * Returns the OpenAPI Specification schema name from a schema
105  * reference. E.g. "#/components/schemas/Thing" => "Thing".
106  */
107 string getSchemaNameFromRef(string ref_) {
108   string schemaName = ref_;
109   if (!skipOver(schemaName, "#/components/schemas/"))
110     throw new Exception("External references not supported! " ~ ref_);
111   return schemaName;
112 }
113 
114 /**
115  * Generates and writes to disk D-language files that correspond to the OpenAPI Document's
116  * components/schemas data. Depending on the software architecture ideas being used, such
117  * files can be known as "model" or "dto" files.
118  */
119 void writeSchemaFiles(OasDocument oasDocument, string targetDir, string packageRoot) {
120   foreach (string schemaName, OasSchema schema; oasDocument.components.schemas) {
121     JsonSchema jsonSchema = new JsonSchema();
122     jsonSchema.schemaName = schemaName;
123     jsonSchema.moduleName = getModuleNameFromSchemaName(packageRoot, schemaName);
124     jsonSchema.className = getClassNameFromSchemaName(schemaName);
125     jsonSchema.schema = schema;
126     string ref_ = "#/components/schemas/" ~ schemaName;
127     jsonSchemaByRef[ref_] = jsonSchema;
128     writeln("Added reference: ", ref_);
129 
130     generateModuleCode(targetDir, jsonSchema, schema, packageRoot);
131   }
132 }
133 
134 /**
135  * A collection of variable names that cannot be used to generate Dlang code.
136  */
137 static immutable RedBlackTree!string RESERVED_WORDS = new RedBlackTree!string([
138       "abstract",
139       "alias",
140       "align",
141       "asm",
142       "assert",
143       "auto",
144       "bool",
145       "break",
146       "byte",
147       "case",
148       "catch",
149       "cast",
150       "char",
151       "class",
152       "const",
153       "continue",
154       "dchar",
155       "debug",
156       "default",
157       "delegate",
158       "double",
159       "dstring",
160       "else",
161       "enum",
162       "export",
163       "extern",
164       "finally",
165       "float",
166       "for",
167       "foreach",
168       "foreach_reverse",
169       "function",
170       "if",
171       "in",
172       "invariant",
173       "immutable",
174       "import",
175       "int",
176       "lazy",
177       "long",
178       "mixin",
179       "module",
180       "new",
181       "nothrow",
182       "package",
183       "private",
184       "protected",
185       "public",
186       "pure",
187       "real",
188       "ref",
189       "scope",
190       "short",
191       "string",
192       "struct",
193       "switch",
194       "ulong",
195       "union",
196       "version",
197       "wchar",
198       "while",
199       "with",
200       "wstring",
201     ]);
202 
203 /**
204  * Writes a Dlang source file for a module that contains a class representing an OpenAPI
205  * Specification schema.
206  */
207 void generateModuleCode(string targetDir, JsonSchema jsonSchema, OasSchema oasSchema, string packageRoot) {
208   string fileName = buildNormalizedPath(targetDir, tr(jsonSchema.moduleName, ".", "/") ~ ".d");
209   logDebug("Generating module: %s", fileName);
210   auto buffer = appender!string();
211   with (buffer) {
212     put("// File automatically generated from OpenAPI spec.\n");
213     put("module " ~ jsonSchema.moduleName ~ ";\n\n");
214     // We generate the class in a separate buffer, because it's production may add dependencies.
215     auto classBuffer = appender!string();
216     generateClassCode(classBuffer, oasSchema, jsonSchema.className);
217 
218     put("import vibe.data.serialization : vibeName = name, vibeOptional = optional, vibeEmbedNullable = embedNullable;\n");
219     put("import vibe.data.json : Json;\n");
220     put("import builder : AddBuilder;\n");
221     put("\n");
222     put("import std.typecons : Nullable;\n\n");
223     // While generating the class code, we accumulated external references to import.
224     foreach (string schemaRef; getSchemaReferences(oasSchema)) {
225       logDebug("Adding import for schema reference: %s", schemaRef);
226       string schemaName = getSchemaNameFromRef(schemaRef);
227       logDebug("Adding import for schema name: %s", schemaName);
228       // Do not add an import for self-references.
229       if (schemaName == jsonSchema.schemaName)
230         continue;
231       put("import ");
232       put(getModuleNameFromSchemaName(packageRoot, schemaName));
233       put(" : ");
234       put(getClassNameFromSchemaName(schemaName));
235       put(";\n");
236     }
237     put("\n");
238 
239     // Finally add our class code to the module file.
240     put(classBuffer[]);
241   }
242   writeln("Writing file: ", fileName);
243   mkdirRecurse(dirName(fileName));
244   write(fileName, buffer[]);
245 }
246 
247 /**
248  * Writes to a buffer a Dlang class representing an OpenAPI Specification schema.
249  */
250 void generateClassCode(
251     Appender!string buffer, OasSchema schema, string className) {
252   string description = schema.description;
253   OasSchema[string] properties = schema.properties;
254   with (buffer) {
255     // Display the class description and declare it.
256     put("/**\n");
257     foreach (string line; wordWrapText(description, 95)) {
258       put(" * ");
259       put(line);
260       put("\n");
261     }
262     put(" */\n");
263     put("class " ~ className ~ " {\n");
264     // Define individual properties of the class.
265     foreach (string propertyName, OasSchema propertySchema; properties) {
266       try {
267         generateSchemaInnerClasses(buffer, propertySchema, "  ", propertyName.toUpperCamelCase());
268         generatePropertyCode(
269             buffer, propertyName, propertySchema,
270             "  ", canFind(schema.required, propertyName));
271       } catch (Exception e) {
272         writeln("Error writing className=", className);
273         throw e;
274       }
275     }
276     put("  mixin AddBuilder!(typeof(this));\n\n");
277     put("}\n");
278   }
279 }
280 
281 /**
282  * Produce code for a class declaring a named property based on an [OasSchema] for the property.
283  */
284 void generatePropertyCode(
285     Appender!string buffer, string propertyName, OasSchema propertySchema,
286     string prefix = "  ", bool required = true) {
287   // Determine the type of the property.
288   import std.conv : to;
289   logDebug("generatePropertyCode 0: propertyName=%s, required=%s", propertyName, required.to!string);
290   // TODO: Use the "schema.required" in the parent schema to determine which are nullable.
291   string propertyCodeType = getSchemaCodeType(propertySchema, propertyName.toUpperCamelCase());
292   if (propertyCodeType is null)
293     return;
294   if (!required)
295     propertyCodeType = "Nullable!(" ~ propertyCodeType ~ ")";
296 
297   if (propertySchema.description !is null) {
298     buffer.put(prefix);
299     buffer.put("/**\n");
300     foreach (string line; wordWrapText(propertySchema.description, 93)) {
301       buffer.put(prefix);
302       buffer.put(" * ");
303       buffer.put(line);
304       buffer.put("\n");
305     }
306     buffer.put(prefix);
307     buffer.put(" */\n");
308   }
309   try {
310     buffer.put(prefix ~ "@vibeName(\"" ~ propertyName ~ "\")\n");
311     buffer.put(prefix ~ "@vibeOptional\n");
312     if (!required)
313       buffer.put(prefix ~ "@vibeEmbedNullable\n");
314     buffer.put(prefix ~ propertyCodeType ~ " "
315         ~ getVariableName(propertyName) ~ ";\n\n");
316   } catch (Exception e) {
317     writeln("Error writing propertyName=", propertyName,
318         ", propertyDescription=", propertySchema.description);
319     throw e;
320   }
321 }
322 
323 /**
324  * Not every propertyName to be found in an OpenAPI Specification Document can be used to a
325  * variable name in code. E.g. the name "scope" is a reserved word in D, and must be replaced with
326  * "scope_".
327  */
328 static string getVariableName(string propertyName) {
329   string variableName = propertyName.toLowerCamelCase();
330   if (variableName in RESERVED_WORDS)
331     variableName ~= "_";
332   return variableName;
333 }
334 
335 /**
336  * Some OasSchema types refer to unnamed objects that have a fixed set of
337  * parameters. The best representation of this in D is a named class.
338  *
339  * Most types, like a simple "integer" or "string" will not generate any inner classes, but a few
340  * cases will, such as "object" with a specific set of valid "properties".
341  */
342 void generateSchemaInnerClasses(
343     Appender!string buffer, OasSchema schema, string prefix="  ", string defaultName = null,
344     RedBlackTree!string context = null) {
345   // To prevent the same class from being generated twice (which can happen when two properties are
346   // themselves objects and have an identical set of properties and names), keep track of class
347   // names that have been generated.
348   if (context is null)
349     context = new RedBlackTree!string();
350   // Otherwise, we create static inner classes that match any objects, arrays, or other things that
351   // are defined.
352   if (schema.type == "object" || schema.properties !is null) {
353     logDebug("Generating innerClass with defaultName=%s", prefix, defaultName);
354     if (schema.properties is null) {
355       // If additionalProperties is an object, it's a schema for the data type, but an arbitrary set
356       // of attributes may exist.
357       if (schema.additionalProperties.type == Json.Type.Undefined
358           || schema.additionalProperties.type == Json.Type.Bool) {
359         // Do not generate a class, this will be either Json or nothing at all.
360       }
361       else if (schema.additionalProperties.type == Json.Type.Object) {
362         OasSchema propertySchema = deserializeJson!OasSchema(schema.additionalProperties);
363         generateSchemaInnerClasses(buffer, propertySchema, prefix, defaultName, context);
364       }
365     }
366     else {
367       // We will have to make a class/struct out of this type from its name.
368       string className = getClassNameFromSchemaName(
369           (schema.title !is null) ? schema.title.toUpperCamelCase() : defaultName);
370       if (className is null)
371         throw new Exception("Creating an Inner Class for property requires a title or default name!");
372       if (className in context) {
373         writeln("Avoiding generating duplicate inner class '", className, "'.");
374         return;
375       }
376       if (schema.additionalProperties.type == Json.Type.Undefined ||
377           (schema.additionalProperties.type == Json.Type.Bool
378               && schema.additionalProperties.get!bool == true)) {
379         writeln("Warning: ", className, " may have additional properties!");
380       }
381       buffer.put(prefix);
382       buffer.put("static class " ~ className ~ " {\n");
383       // Before we start a new context, let the previous one know about the class being defined.
384       context.insert(className);
385       // Start a new context, because the inner class creates a new naming scope.
386       context = new RedBlackTree!string();
387       foreach (string propertyName, OasSchema propertySchema; schema.properties) {
388         logDebug("Generating propertyName: %s", propertyName);
389         generateSchemaInnerClasses(
390             buffer, propertySchema, prefix ~ "  ", propertyName.toUpperCamelCase(), context);
391         generatePropertyCode(
392             buffer, propertyName, propertySchema,
393             prefix ~ "  ", canFind(schema.required, propertyName));
394       }
395       buffer.put(prefix ~ "  mixin AddBuilder!(typeof(this));\n\n");
396       buffer.put(prefix ~ "}\n\n");
397     }
398   }
399   // The type might be an array and it's schema could be hidden beneath.
400   else if (schema.type == "array" || schema.items !is null) {
401     if (schema.items is null) {
402       throw new Exception("Schema missing 'items' definition for array item with defaultName="
403           ~ (defaultName is null ? "" : defaultName));
404     }
405     generateSchemaInnerClasses(buffer, schema.items, prefix, defaultName, context);
406   }
407   // Sometimes data has no explicit properties, but we can infer them from validation data.
408   else if (schema.anyOf !is null && schema.anyOf.length == 1) {
409     generateSchemaInnerClasses(buffer, schema.anyOf[0], prefix, null, context);
410   }
411 }
412 
413 /**
414  * Converts a given [OasSchema] type into the equivalent type in source code.
415  *
416  * Params:
417  *   defaultName = If a structured type can be created as an inner class, the default name to use
418  *     for that class.
419  */
420 string getSchemaCodeType(OasSchema schema, string defaultName = null, bool required = true) {
421   // Default to string if no schema exists.
422   if (schema is null) {
423     return "string";
424   }
425   // This could be a reference to an existing type.
426   if (schema.ref_ !is null) {
427     string schemaName = getSchemaNameFromRef(schema.ref_);
428     // Resolving this class name depends on having an import statement.
429     return getClassNameFromSchemaName(schemaName);
430   }
431   // First check if we have a primitive type.
432   // See: https://swagger.io/docs/specification/data-models/data-types/
433   else if (schema.type !is null || schema.items || schema.properties) {
434     if (schema.type == "integer") {
435       if (schema.format == "int32")
436         return "int";
437       else if (schema.format == "int64")
438         return "long";
439       else if (schema.format == "unix-time")
440         return "long";
441       return "int";
442     } else if (schema.type == "number") {
443       if (schema.format == "float")
444         return "float";
445       else if (schema.format == "double")
446         return "double";
447       return "float";
448     } else if (schema.type == "boolean") {
449       return "bool";
450     } else if (schema.type == "string") {
451       return "string";
452     } else if (schema.type == "array" || schema.items !is null) {
453       string arrayCodeType = getSchemaCodeType(schema.items, defaultName);
454       return arrayCodeType !is null ? arrayCodeType ~ "[]" : null;
455     } else if (schema.type == "object" || schema.properties !is null) {
456       // If we are missing both properties and additionalProperties, we assume a generic string[string] object.
457       if (schema.properties is null) {
458         // If additionalProperties is an object, it's a schema for the data type, but any number of
459         // fields may exist.
460         if (schema.additionalProperties.type == Json.Type.Object) {
461           OasSchema propertySchema = deserializeJson!OasSchema(schema.additionalProperties);
462           string propertyCodeType = getSchemaCodeType(propertySchema);
463           return propertyCodeType !is null ? propertyCodeType ~ "[string]" : null;
464         }
465         // If additional properties exist, but we have no type information, it can be anything.
466         else if (schema.additionalProperties.type == Json.Type.Undefined
467             || (schema.additionalProperties.type == Json.Type.Bool
468                 && schema.additionalProperties.get!bool == true)) {
469           return "Json";
470         }
471         // If there are no properties, and no additional properties, then it's not a type at all.
472         else {
473           return null;
474         }
475       }
476       // If properties are present we can safely assume a class will be created.
477       else {
478         // We will have to make a class/struct out of this type from its name.
479         if (schema.title !is null)
480           return getClassNameFromSchemaName(schema.title.toUpperCamelCase());
481         else if (defaultName !is null)
482           return getClassNameFromSchemaName(defaultName);
483         throw new Exception("Creating a named object type requires a title or defaultName!");
484       }
485     }
486   }
487   // Perhaps we can infer the type from the "anyOf" validation.
488   else if (schema.anyOf !is null && schema.anyOf.length == 1) {
489     return getSchemaCodeType(schema.anyOf[0]);
490   }
491   // If all else fails, put the programmer in the driver's seat.
492   return "Json";
493 }
494 
495 /**
496  * When using a schema, it may reference other external schemas which have to be imported into any
497  * module that uses them.
498  */
499 string[] getSchemaReferences(OasSchema schema) {
500   RedBlackTree!string refs = new RedBlackTree!string();
501   getSchemaReferences(schema, refs);
502   return refs[].array;
503 }
504 
505 /// ditto
506 private void getSchemaReferences(OasSchema schema, ref RedBlackTree!string refs) {
507   if (schema.ref_ !is null) {
508     refs.insert(schema.ref_);
509   } else if (schema.type == "array" || schema.items !is null) {
510     getSchemaReferences(schema.items, refs);
511   } else if (schema.type == "object" || schema.properties !is null) {
512     if (schema.properties !is null) {
513       foreach (string propertyName, OasSchema propertySchema; schema.properties) {
514         getSchemaReferences(propertySchema, refs);
515       }
516     }
517     if (schema.additionalProperties.type == Json.Type.Object) {
518       getSchemaReferences(deserializeJson!OasSchema(schema.additionalProperties), refs);
519     }
520   } else if (schema.anyOf !is null) {
521     foreach (OasSchema anyOfSchema; schema.anyOf) {
522       getSchemaReferences(anyOfSchema, refs);
523     }
524   } else if (schema.allOf !is null) {
525     foreach (OasSchema allOfSchema; schema.allOf) {
526       getSchemaReferences(allOfSchema, refs);
527     }
528   }
529 }