Adding a "preamble" section to allow arbitrary python to be inserted at the top of the file

ialarmedalien · ialarmedalien · commit 1a70568e98e4 · 2022-08-26T11:12:28.000-07:00
diff --git a/README.md b/README.md
@@ -1,4 +1,4 @@
-[![json2python-models](/etc/logo.png)](https://github.com/bogdandm/json2python-models)
+dr[![json2python-models](/etc/logo.png)](https://github.com/bogdandm/json2python-models)
 
 [![PyPI version](https://img.shields.io/pypi/v/json2python-models.svg?color=green)](https://badge.fury.io/py/json2python-models)
 [![Build](https://github.com/bogdandm/json2python-models/actions/workflows/test_and_release.yml/badge.svg)](https://github.com/bogdandm/json2python-models/actions/workflows/test_and_release.yml)
@@ -7,9 +7,9 @@
 
 ![Example](/etc/convert.png)
 
-json2python-models is a [Python](https://www.python.org/) tool that can generate Python models classes 
-([pydantic](https://github.com/samuelcolvin/pydantic), dataclasses, [attrs](https://github.com/python-attrs/attrs)) 
-from JSON dataset. 
+json2python-models is a [Python](https://www.python.org/) tool that can generate Python models classes
+([pydantic](https://github.com/samuelcolvin/pydantic), dataclasses, [attrs](https://github.com/python-attrs/attrs))
+from JSON dataset.
 
 ## Features
 
@@ -26,18 +26,23 @@ from JSON dataset.
 
 ## Table of Contents
 
-* [Features](#features)
-* [Table of Contents](#table-of-contents)
-* [Example](#example)
-* [Installation](#installation)
-* [Usage](#usage)
-    * [CLI](#cli)
-    * [Low level API]()
-* [Tests](#tests)
-    * [Test examples](#test-examples)
-* [Built With](#built-with)
-* [Contributing](#contributing)
-* [License](#license)
+- [Features](#features)
+- [Table of Contents](#table-of-contents)
+- [Examples](#examples)
+  - [Part of Path of Exile public items API](#part-of-path-of-exile-public-items-api)
+  - [F1 Season Results](#f1-season-results)
+  - [Swagger](#swagger)
+  - [Github-actions config files](#github-actions-config-files)
+  - [Example with preamble](#example-with-preamble)
+- [Installation](#installation)
+- [Usage](#usage)
+  - [CLI](#cli)
+  - [Low level API](#low-level-api)
+- [Tests](#tests)
+  - [Test examples](#test-examples)
+- [Built With](#built-with)
+- [Contributing](#contributing)
+- [License](#license)
 
 ## Examples
 
@@ -155,9 +160,9 @@ class Constructor(BaseModel):
 
 `swagger.json` from any online API (I tested file generated by drf-yasg and another one for Spotify API)
 
-It requires a lit bit of tweaking:
+It requires a bit of tweaking:
 * Some fields store routes/models specs as dicts
-* There is a lot of optinal fields so we reduce merging threshold 
+* There are a lot of optinal fields so we reduce merging threshold
 * Disable string literals
 
 ```
@@ -405,9 +410,45 @@ class Run(BaseModel):
 
 </p></details>
 
+### Example with preamble
+
+<details><summary>----- Show -----</summary>
+<p>
+A simple example to demonstrate adding extra code before the class list.
+
+```sh
+json2models -f pydantic --preamble "# set up defaults
+USERNAME = 'user'
+SERVER_IP = '127.0.0.1'
+" -m Swagger testing_tools/swagger.json
+```
+
+```py
+r"""
+generated by json2python-models v0.2.5 at Tue Aug 23 08:55:09 2022
+command: json2models -f pydantic --preamble # set up defaults
+USERNAME = 'user'
+SERVER_IP = '127.0.0.1'
+ -m Swagger testing_tools/swagger.json -o output.py
+"""
+from pydantic import BaseModel, Field
+from typing import Any, List, Literal, Optional, Union
+
+
+# set up defaults
+USERNAME = 'user'
+SERVER_IP = '127.0.0.1'
+
+
+
+class Swagger(BaseModel):
+    # etc.
+```
+</p></details>
+
 ## Installation
 
-| **Be ware**: this project supports only `python3.7` and higher. |
+| **Beware**: this project supports only `python3.7` and higher. |
 | --- |
 
 To install it, use `pip`:
@@ -426,7 +467,7 @@ python setup.py install
 
 ### CLI
 
-For regular usage CLI tool is the best option. After you install this package you could use it as `json2models <arguments>` 
+For regular usage CLI tool is the best option. After you install this package you can use it as `json2models <arguments>`
 or `python -m json_to_models <arguments>`. I.e.:
 ```
 json2models -m Car car_*.json -f attrs > car.py
@@ -464,61 +505,71 @@ Arguments:
     * **Format**: `-f {base, pydantic, attrs, dataclasses, custom}`
     * **Example**: `-f pydantic`
     * **Default**: `-f base`
-    
+
 * `-s`, `--structure` - Models composition style.
-    * **Format**: `-s {flat, nested}` 
+    * **Format**: `-s {flat, nested}`
     * **Example**: `-s nested`
     * **Default**: `-s flat`
-    
+
+* `--preamble` - Additional material to be
+    * **Format**: `--preamble "<formatted python code string to be added after module imports>"`
+    * **Example**:
+  ```sh
+  --preamble "# set up defaults
+  USERNAME = 'user'
+  SERVER = '127.0.0.1'"
+  ```
+    * **Optional**
+
 * `--datetime` - Enable datetime/date/time strings parsing.
     * **Default**: disabled
     * **Warning**: This can lead to 6-7 times slowdown on large datasets. Be sure that you really need this option.
-    
+
 * `--disable-unicode-conversion`, `--no-unidecode` - Disable unicode conversion in field labels and class names
     * **Default**: enabled
-    
+
 * `--strings-converters` - Enable generation of string types converters (i.e. `IsoDatetimeString` or `BooleanString`).
     * **Default**: disabled
-    
+
 * `--max-strings-literals` -  Generate `Literal['foo', 'bar']` when field have less than NUMBER string constants as values.
-    * **Format**: `--max-strings-literals <NUMBER>` 
+    * **Format**: `--max-strings-literals <NUMBER>`
     * **Default**: 10 (generator classes could override it)
     * **Example**: `--max-strings-literals 5` - only 5 literals will be saved and used to code generation
     * **Note**: There could not be more than **15** literals per field (for performance reasons)
     * **Note**: `attrs` code generator do not use Literals and just generate `str` fields instead
 
-* `--merge` - Merge policy settings. Possible values are: 
+* `--merge` - Merge policy settings. Possible values are:
     * **Format**: `--merge MERGE_POLICY [MERGE_POLICY ...]`
     * **Possible values** (MERGE_POLICY):
-        * `percent[_<percent>]` - two models had a certain percentage of matched field names. 
-            Custom value could be i.e. `percent_95`. 
-        * `number[_<number>]` - two models had a certain number of matched field names. 
+        * `percent[_<percent>]` - two models had a certain percentage of matched field names.
+            Custom value could be i.e. `percent_95`.
+        * `number[_<number>]` - two models had a certain number of matched field names.
         * `exact` - two models should have exact same field names to merge.
     * **Example**: `--merge percent_95 number_20` - merge if 95% of fields are matched or 20 of fields are matched
     * **Default**: `--merge percent_70 number_10`
-    
+
 * `--dict-keys-regex`, `--dkr` - List of regular expressions (Python syntax).
-    If all keys of some dict are match one of the pattern then 
+    If all keys of some dict are match one of the pattern then
     this dict will be marked as dict field but not nested model.
     * **Format**: `--dkr RegEx [RegEx ...]`
     * **Example**: `--dkr node_\d+ \d+_\d+_\d+`
-    * **Note**: `^` and `$` (string borders) tokens will be added automatically but you 
+    * **Note**: `^` and `$` (string borders) tokens will be added automatically but you
         have to escape other special characters manually.
     * **Optional**
-    
+
 * `--dict-keys-fields`, `--dkf` - List of model fields names that will be marked as dict fields
     * **Format**: `--dkf FIELD_NAME [FIELD_NAME ...]`
     * **Example**: `--dkf "dict_data" "mapping"`
     * **Optional**
-    
+
 * `--code-generator` - Absolute import path to `GenericModelCodeGenerator` subclass.
     * **Format**: `--code-generator CODE_GENERATOR`
     * **Example**: `-f mypackage.mymodule.DjangoModelsGenerator`
     * **Note**: Is ignored without `-f custom` but is required with it.
-    
-* `--code-generator-kwargs` - List of GenericModelCodeGenerator subclass arguments (for `__init__` method, 
-    see docs of specific subclass). 
-    Each argument should be in following format: `argument_name=value` or `"argument_name=value with space"`. 
+
+* `--code-generator-kwargs` - List of GenericModelCodeGenerator subclass arguments (for `__init__` method,
+    see docs of specific subclass).
+    Each argument should be in following format: `argument_name=value` or `"argument_name=value with space"`.
     Boolean values should be passed in JS style: `true` or `false`
     * **Format**: `--code-generator-kwargs [NAME=VALUE [NAME=VALUE ...]]`
     * **Example**:  `--code-generator-kwargs kwarg1=true kwarg2=10 "kwarg3=It is string with spaces"`
diff --git a/json_to_models/cli.py b/json_to_models/cli.py
@@ -102,11 +102,12 @@ def parse_args(self, args: List[str] = None):
         code_generator_kwargs_raw: List[str] = namespace.code_generator_kwargs
         dict_keys_regex: List[str] = namespace.dict_keys_regex
         dict_keys_fields: List[str] = namespace.dict_keys_fields
+        preamble: str = namespace.preamble
 
-        self.validate(models, models_lists, merge_policy, framework, code_generator)
+        self.validate(models_lists, merge_policy, framework, code_generator)
         self.setup_models_data(models, models_lists, parser)
         self.set_args(merge_policy, structure, framework, code_generator, code_generator_kwargs_raw,
-                      dict_keys_regex, dict_keys_fields, disable_unicode_conversion)
+                      dict_keys_regex, dict_keys_fields, disable_unicode_conversion, preamble)
 
     def run(self):
         if self.enable_datetime:
@@ -122,8 +123,11 @@ def run(self):
         registry.merge_models(generator)
         registry.generate_names()
         structure = self.structure_fn(registry.models_map)
-        output = self.version_string + \
-                 generate_code(structure, self.model_generator, class_generator_kwargs=self.model_generator_kwargs)
+        output = self.version_string + generate_code(
+            structure,
+            self.model_generator,
+            class_generator_kwargs=self.model_generator_kwargs,
+            preamble=self.preamble)
         if self.output_file:
             with open(self.output_file, "w", encoding="utf-8") as f:
                 f.write(output)
@@ -140,11 +144,10 @@ def version_string(self):
             '"""\n'
         )
 
-    def validate(self, models, models_list, merge_policy, framework, code_generator):
+    def validate(self, models_list, merge_policy, framework, code_generator):
         """
         Validate parsed args
 
-        :param models:  List of pairs (model name, list of filesystem path)
         :param models_list: List of pairs (model name, list of lookup expr and filesystem path)
         :param merge_policy: List of merge policies. Each merge policy is either string or string and policy arguments
         :param framework: Framework name (predefined code generator)
@@ -196,7 +199,8 @@ def set_args(
             code_generator_kwargs_raw: List[str],
             dict_keys_regex: List[str],
             dict_keys_fields: List[str],
-            disable_unicode_conversion: bool
+            disable_unicode_conversion: bool,
+            preamble: str,
     ):
         """
         Convert CLI args to python representation and set them to appropriate object attributes
@@ -236,7 +240,7 @@ def set_args(
 
         self.dict_keys_regex = [re.compile(rf"^{r}$") for r in dict_keys_regex] if dict_keys_regex else ()
         self.dict_keys_fields = dict_keys_fields or ()
-
+        self.preamble = preamble or None
         self.initialized = True
 
     @classmethod
@@ -366,6 +370,11 @@ def _create_argparser(cls) -> argparse.ArgumentParser:
                  "Boolean values should be passed in JS style: true | false"
                  "\n\n"
         )
+        parser.add_argument(
+            "--preamble",
+            type=str,
+            help="Code to insert into the generated file after the imports and before the list of classes\n\n"
+        )
 
         return parser
 
diff --git a/json_to_models/models/base.py b/json_to_models/models/base.py
@@ -54,19 +54,19 @@ class GenericModelCodeGenerator:
     @{{ decorator }}
     {% endfor -%}
     class {{ name }}{% if bases %}({{ bases }}){% endif %}:
-    
+
     {%- for code in nested %}
     {{ code }}
     {% endfor -%}
-    
+
     {%- if fields -%}
     {%- for field in fields %}
         {{ field }}
     {%- endfor %}
     {%- else %}
         pass
     {%- endif -%}
-    {%- if extra %}    
+    {%- if extra %}
     {{ extra }}
     {%- endif -%}
     """)
@@ -210,7 +210,7 @@ def _generate_code(
         lvl=0
 ) -> Tuple[ImportPathList, List[str]]:
     """
-    Walk thought models structure and covert them into code
+    Walk through the model structures and convert them into code
 
     :param structure: Result of compose_models or similar function
     :param class_generator: GenericModelCodeGenerator subclass
@@ -241,23 +241,28 @@ def _generate_code(
 
 
 def generate_code(structure: ModelsStructureType, class_generator: Type[GenericModelCodeGenerator],
-                  class_generator_kwargs: dict = None, objects_delimiter: str = OBJECTS_DELIMITER) -> str:
+                  class_generator_kwargs: dict = None,
+                  objects_delimiter: str = OBJECTS_DELIMITER,
+                  preamble: str = None) -> str:
     """
     Generate ready-to-use code
 
     :param structure: Result of compose_models or similar function
     :param class_generator: GenericModelCodeGenerator subclass
     :param class_generator_kwargs: kwags for GenericModelCodeGenerator init
     :param objects_delimiter: Delimiter between root level classes
+    :param preamble: code to insert after the imports and before the classes
     :return: Generated code
     """
     root, mapping = structure
     with AbsoluteModelRef.inject(mapping):
         imports, classes = _generate_code(root, class_generator, class_generator_kwargs or {})
+        imports_str = ""
     if imports:
         imports_str = compile_imports(imports) + objects_delimiter
-    else:
-        imports_str = ""
+    if preamble:
+        imports_str += preamble + objects_delimiter
+
     return imports_str + objects_delimiter.join(classes) + "\n"
 
 
diff --git a/test/test_cli/test_script.py b/test/test_cli/test_script.py
