Merge pull request #39 from bogdandm/yaml-ini-parsers

Add yaml and ini parsers

Author: bogdandm

README.md

@@ -286,6 +286,125 @@ class Definition_Schema:
 </p>
 </details>
 
+### Github-actions config files
+
+<details><summary>----- Show -----</summary>
+<p>
+
+Github-actions model based on files from [starter-workflows](https://github.com/actions/starter-workflows/tree/main/ci)
+
+```
+json2models -m Actions "./starter-workflows/ci/*.yml" -s flat -f pydantic -i yaml --dkf env with jobs
+```
+
+```python
+r"""
+generated by json2python-models v0.2.3 at Tue Jul 13 19:52:43 2021
+command: /opt/projects/json2python-models/venv/bin/json2models -m Actions ./starter-workflows/ci/*.yml -s flat -f pydantic -i yaml --dkf env with jobs
+"""
+from pydantic import BaseModel, Field
+from typing import Dict, List, Optional, Union
+from typing_extensions import Literal
+
+
+class Actions(BaseModel):
+    on: Union['On', List[Literal["push"]]]
+    jobs: Dict[str, 'Job']
+    name: Optional[str] = None
+    env: Optional[Dict[str, Union[int, str]]] = {}
+
+
+class On(BaseModel):
+    push: Optional['Push'] = None
+    pull_request: Optional['PullRequest'] = None
+    release: Optional['Release'] = None
+    schedule: Optional[List['Schedule']] = []
+    workflow_dispatch: Optional[None] = None
+
+
+class Push(BaseModel):
+    branches: List[Literal["$default-branch"]]
+    tags: Optional[List[Literal["v*.*.*"]]] = []
+
+
+class PullRequest(BaseModel):
+    branches: List[Literal["$default-branch"]]
+
+
+class Release(BaseModel):
+    types: List[Literal["created", "published"]]
+
+
+class Schedule(BaseModel):
+    cron: Literal["$cron-daily"]
+
+
+class Job(BaseModel):
+    runson: Literal["${{ matrix.os }}", "macOS-latest", "macos-latest", "ubuntu-18.04", "ubuntu-latest", "windows-latest"] = Field(..., alias="runs-on")
+    steps: List['Step']
+    name: Optional[str] = None
+    environment: Optional[Literal["production"]] = None
+    outputs: Optional['Output'] = None
+    container: Optional['Container'] = None
+    needs: Optional[Literal["build"]] = None
+    permissions: Optional['Permission'] = None
+    strategy: Optional['Strategy'] = None
+    defaults: Optional['Default'] = None
+    env: Optional[Dict[str, str]] = {}
+
+
+class Step(BaseModel):
+    uses: Optional[str] = None
+    name: Optional[str] = None
+    with_: Optional[Dict[str, Union[bool, float, str]]] = Field({}, alias="with")
+    run: Optional[str] = None
+    env: Optional[Dict[str, str]] = {}
+    workingdirectory: Optional[str] = Field(None, alias="working-directory")
+    id_: Optional[Literal["build-image", "composer-cache", "deploy-and-expose", "image-build", "login-ecr", "meta", "push-to-registry", "task-def"]] = Field(None, alias="id")
+    if_: Optional[str] = Field(None, alias="if")
+    shell: Optional[Literal["Rscript {0}"]] = None
+
+
+class Output(BaseModel):
+    route: str = Field(..., alias="ROUTE")
+    selector: str = Field(..., alias="SELECTOR")
+
+
+class Container(BaseModel):
+    image: Literal["crystallang/crystal", "erlang:22.0.7"]
+
+
+class Permission(BaseModel):
+    contents: Literal["read"]
+    packages: Literal["write"]
+
+
+class Strategy(BaseModel):
+    matrix: Optional['Matrix'] = None
+    maxparallel: Optional[int] = Field(None, alias="max-parallel")
+    failfast: Optional[bool] = Field(None, alias="fail-fast")
+
+
+class Matrix(BaseModel):
+    rversion: Optional[List[float]] = Field([], alias="r-version")
+    pythonversion: Optional[List[float]] = Field([], alias="python-version")
+    deno: Optional[List[Literal["canary", "v1.x"]]] = []
+    os: Optional[List[Literal["macOS-latest", "ubuntu-latest", "windows-latest"]]] = []
+    rubyversion: Optional[List[float]] = Field([], alias="ruby-version")
+    nodeversion: Optional[List[Literal["12.x", "14.x", "16.x"]]] = Field([], alias="node-version")
+    configuration: Optional[List[Literal["Debug", "Release"]]] = []
+
+
+class Default(BaseModel):
+    run: 'Run'
+
+
+class Run(BaseModel):
+    shell: Literal["bash"]
+```
+
+</p></details>
+
 ## Installation
 
 | **Be ware**: this project supports only `python3.7` and higher. |
@@ -315,24 +434,33 @@ json2models -m Car car_*.json -f attrs > car.py
 
 Arguments:
 * `-h`, `--help` - Show help message and exit
-    
-* `-m`, `--model` - Model name and its JSON data as path or unix-like path pattern. `*`,  `**` or `?` patterns symbols are supported.
+
+* `-m`, `--model` - Model name and its JSON data as path or unix-like path pattern. `*`,  `**` or `?` patterns symbols
+  are supported.
     * **Format**: `-m <Model name> [<JSON files> ...]`
     * **Example**: `-m Car audi.json reno.json` or `-m Car audi.json -m Car reno.json` (results will be the same)
-    
-* `-l`, `--list` - Like `-m` but given json file should contain list of model data (dataset). 
-    If this file contains dict with nested list than you can pass `<JSON key>` to lookup. 
-    Deep lookups are supported by dot-separated path. If no lookup needed pass `-` as `<JSON key>`.
+
+* `-l`, `--list` - Like `-m` but given json file should contain list of model data (dataset). If this file contains dict
+  with nested list than you can pass `<JSON key>` to lookup. Deep lookups are supported by dot-separated path. If no
+  lookup needed pass `-` as `<JSON key>`.
     * **Format**: `-l <Model name> <JSON key> <JSON file>`
     * **Example**: `-l Car - cars.json -l Person fetch_results.items.persons result.json`
-    * **Note**: Models names under this arguments should be unique.
-    
+    * **Note**: Models names under these arguments should be unique.
+
+* `-i`, `--input-format` - Input file format (parser). Default is JSON parser. Yaml parser requires PyYaml or
+  ruamel.yaml to be installed. Ini parser uses
+  builtin [configparser](https://docs.python.org/3/library/configparser.html). To implement new one - add new method
+  to `cli.FileLoaders` (and create pull request :) )
+    * **Format**: `-i {json, yaml, ini}`
+    * **Example**: `-i yaml`
+    * **Default**: `-i json`
+
 * `-o`, `--output` - Output file
     * **Format**: `-o <FILE>`
     * **Example**: `-o car_model.py`
-    
-* `-f`, `--framework` - Model framework for which python code is generated. 
-    `base` (default) mean no framework so code will be generated without any decorators and additional meta-data.
+
+* `-f`, `--framework` - Model framework for which python code is generated.
+  `base` (default) mean no framework so code will be generated without any decorators and additional meta-data.
     * **Format**: `-f {base, pydantic, attrs, dataclasses, custom}`
     * **Example**: `-f pydantic`
     * **Default**: `-f base`

json_to_models/cli.py

@@ -1,4 +1,5 @@
 import argparse
+import configparser
 import importlib
 import itertools
 import json
@@ -10,6 +11,14 @@
 from pathlib import Path
 from typing import Any, Callable, Dict, Generator, Iterable, List, Tuple, Type, Union
 
+try:
+    import ruamel.yaml as yaml
+except ImportError:
+    try:
+        import yaml
+    except ImportError:
+        yaml = None
+
 from . import __version__ as VERSION
 from .dynamic_typing import ModelMeta, register_datetime_classes
 from .generator import MetadataGenerator
@@ -80,6 +89,7 @@ def parse_args(self, args: List[str] = None):
             (model_name, (lookup, Path(path)))
             for model_name, lookup, path in namespace.list or ()
         ]
+        parser = getattr(FileLoaders, namespace.input_format)
         self.output_file = namespace.output
         self.enable_datetime = namespace.datetime
         disable_unicode_conversion = namespace.disable_unicode_conversion
@@ -94,7 +104,7 @@ def parse_args(self, args: List[str] = None):
         dict_keys_fields: List[str] = namespace.dict_keys_fields
 
         self.validate(models, models_lists, merge_policy, framework, code_generator)
-        self.setup_models_data(models, models_lists)
+        self.setup_models_data(models, models_lists, parser)
         self.set_args(merge_policy, structure, framework, code_generator, code_generator_kwargs_raw,
                       dict_keys_regex, dict_keys_fields, disable_unicode_conversion)
 
@@ -157,16 +167,20 @@ def validate(self, models, models_list, merge_policy, framework, code_generator)
         elif framework != 'custom' and code_generator is not None:
             raise ValueError("--code-generator argument has no effect without '--framework custom' argument")
 
-    def setup_models_data(self, models: Iterable[Tuple[str, Iterable[Path]]],
-                          models_lists: Iterable[Tuple[str, Tuple[str, Path]]]):
+    def setup_models_data(
+            self,
+            models: Iterable[Tuple[str, Iterable[Path]]],
+            models_lists: Iterable[Tuple[str, Tuple[str, Path]]],
+            parser: 'FileLoaders.T'
+    ):
         """
         Initialize lazy loaders for models data
         """
         models_dict: Dict[str, List[Iterable[dict]]] = defaultdict(list)
         for model_name, paths in models:
-            models_dict[model_name].append(map(safe_json_load, paths))
+            models_dict[model_name].append(parser(path) for path in paths)
         for model_name, (lookup, path) in models_lists:
-            models_dict[model_name].append(iter_json_file(path, lookup))
+            models_dict[model_name].append(iter_json_file(parser(path), lookup))
 
         self.models_data = {
             model_name: itertools.chain(*list_of_gen)
@@ -252,6 +266,12 @@ def _create_argparser(cls) -> argparse.ArgumentParser:
                  "I.e. for file that contains dict {\"a\": {\"b\": [model_data, ...]}} you should\n"
                  "pass 'a.b' as <JSON key>.\n\n"
         )
+        parser.add_argument(
+            "-i", "--input-format",
+            default="json",
+            choices=['json', 'yaml', 'ini'],
+            help="Input files parser ('PyYaml' is required to parse yaml files)\n\n"
+        )
         parser.add_argument(
             "-o", "--output",
             metavar="FILE", default="",
@@ -385,7 +405,31 @@ def path_split(path: str) -> List[str]:
     return folders
 
 
-def dict_lookup(d: dict, lookup: str) -> Union[dict, list]:
+class FileLoaders:
+    T = Callable[[Path], Union[dict, list]]
+
+    @staticmethod
+    def json(path: Path) -> Union[dict, list]:
+        with path.open() as fp:
+            return json.load(fp)
+
+    @staticmethod
+    def yaml(path: Path) -> Union[dict, list]:
+        if yaml is None:
+            print('Yaml parser is not installed. To parse yaml files PyYaml (or ruamel.yaml) is required.')
+            raise ImportError('yaml')
+        with path.open() as fp:
+            return yaml.safe_load(fp)
+
+    @staticmethod
+    def ini(path: Path) -> dict:
+        config = configparser.ConfigParser()
+        with path.open() as fp:
+            config.read_file(fp)
+        return {s: dict(config.items(s)) for s in config.sections()}
+
+
+def dict_lookup(d: Union[dict, list], lookup: str) -> Union[dict, list]:
     """
     Extract nested dictionary value from key path.
     If lookup is "-" returns dict as is.
@@ -403,7 +447,7 @@ def dict_lookup(d: dict, lookup: str) -> Union[dict, list]:
     return d
 
 
-def iter_json_file(path: Path, lookup: str) -> Generator[Union[dict, list], Any, None]:
+def iter_json_file(data: Union[dict, list], lookup: str) -> Generator[Union[dict, list], Any, None]:
     """
     Loads given 'path' file, perform lookup and return generator over json list.
     Does not open file until iteration is started.
@@ -412,21 +456,11 @@ def iter_json_file(path: Path, lookup: str) -> Generator[Union[dict, list], Any,
     :param lookup: Dot separated lookup path
     :return:
     """
-    with path.open() as f:
-        l = json.load(f)
-    l = dict_lookup(l, lookup)
+    l = dict_lookup(data, lookup)
     assert isinstance(l, list), f"Dict lookup return {type(l)} but list is expected, check your lookup path"
     yield from l
 
 
-def safe_json_load(path: Path) -> Union[dict, list]:
-    """
-    Open file, load json and close it.
-    """
-    with path.open(encoding="utf-8") as f:
-        return json.load(f)
-
-
 def _process_path(path: str) -> Iterable[Path]:
     """
     Convert path pattern into path iterable.

json_to_models/dynamic_typing/complex.py

@@ -265,7 +265,10 @@ def to_typing_code(self, types_style: Dict[Union['BaseType', Type['BaseType']],
         if options.get(self.TypeStyle.use_literals):
             limit = options.get(self.TypeStyle.max_literals)
             if limit is None or len(self.literals) < limit:
-                parts = ', '.join(f'"{s}"' for s in sorted(self.literals))
+                parts = ', '.join(
+                    '"{}"'.format(s.replace('\\', '\\\\').replace('"', '\\"'))
+                    for s in sorted(self.literals)
+                )
                 return [(Literal.__module__, 'Literal')], f"Literal[{parts}]"
 
         return [], 'str'

json_to_models/generator.py

@@ -56,6 +56,11 @@ def _convert(self, data: dict):
         """
         fields = dict()
         for key, value in data.items():
+            if not isinstance(key, str):
+                raise TypeError(f'You probably using some not JSON-compatible parser and have some {type(key)} as dict key. '
+                                f'This is not supported.\n'
+                                f'Context: {data}\n'
+                                f'(If you parsing yaml try to replace PyYaml with ruamel.yaml)')
             convert_dict = key not in self.dict_keys_fields
             fields[key] = self._detect_type(value, convert_dict)
         return fields

json_to_models/models/base.py

@@ -23,7 +23,7 @@
 
 keywords_set = set(keyword.kwlist)
 builtins_set = set(__builtins__.keys())
-other_common_names_set = {'datetime', 'time', 'date', 'defaultdict'}
+other_common_names_set = {'datetime', 'time', 'date', 'defaultdict', 'schema'}
 blacklist_words = frozenset(keywords_set | builtins_set | other_common_names_set)
 ones = ['', 'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine']
 

setup.py

@@ -50,6 +50,6 @@ def run_tests(self):
     },
     install_requires=required,
     cmdclass={"test": PyTest},
-    tests_require=["pytest>=4.4.0", "pytest-xdist", "requests", "attrs", "pydantic>=1.3"],
+    tests_require=["pytest>=4.4.0", "pytest-xdist", "requests", "attrs", "pydantic>=1.3", "ruamel.yaml"],
     data_files=[('', ['requirements.txt', 'pytest.ini', '.coveragerc', 'LICENSE', 'README.md', 'CHANGELOG.md'])]
 )

test/test_cli/data/file.ini

@@ -0,0 +1,9 @@
+[owner]
+name = John Doe
+organization = Acme Widgets Inc.
+
+[database]
+; use IP address in case network name resolution is not working
+server = 192.0.2.62
+port = 143
+file = "payroll.dat"