Merge pull request #24 from bogdandm/disable-unicode-conversion-flag

Disable unicode conversion flag

Author: bogdandm

README.md

@@ -162,13 +162,17 @@ Arguments:
     * **Example**: `-l Car - cars.json -l Person fetch_results.items.persons result.json`
     * **Note**: Models names under this arguments should be unique.
 
+* `-o`, `--output` - Output file
+    * **Format**: `-o <FILE>`
+    * **Example**: `-o car_model.py`
+    
 * `-f`, `--framework` - Model framework for which python code is generated. 
     `base` (default) mean no framework so code will be generated without any decorators and additional meta-data.
     * **Format**: `-f {base,attrs,dataclasses,custom}`
     * **Example**: `-f attrs`
     * **Default**: `-f base`
 
-* `-s , --structure` - Models composition style.
+* `-s`, `--structure` - Models composition style.
     * **Format**: `-s {nested, flat}` 
     * **Example**: `-s flat`
     * **Default**: `-s nested`
@@ -177,6 +181,9 @@ Arguments:
     * **Default**: disabled
     * **Warning**: This can lead to 6-7 times slowdown on large datasets. Be sure that you really need this option.
 
+* `--disable-unicode-conversion`, `--no-unidecode` - Disable unicode conversion in field labels and class names
+    * **Default**: enabled
+    
 * `--strings-converters` - Enable generation of string types converters (i.e. `IsoDatetimeString` or `BooleanString`).
     * **Default**: disabled
 
@@ -196,7 +203,7 @@ Arguments:
     * **Format**: `--dkr RegEx [RegEx ...]`
     * **Example**: `--dkr node_\d+ \d+_\d+_\d+`
     * **Note**: `^` and `$` (string borders) tokens will be added automatically but you 
-        have escape to other special characters manually.
+        have to escape other special characters manually.
     * **Optional**
 
 * `--dict-keys-fields`, `--dkf` - List of model fields names that will be marked as dict fields

TODO.md

@@ -1,4 +1,6 @@
-- (!) README.md
+- README
+    - [ ] Restrictions
+    - [ ] Low lvl API wiki or sphinx docs
 - Docstrings
 - Features
     - Models layer
@@ -35,10 +37,11 @@
             - [ ] Complex python types annotations
                 - [ ] Decorator to specify field metatype
                 - [ ] Specify metatype in attr/dataclass argument (if dataclasses has such)
-            - [X] String based types (Warning: 6 times slow down)
+            - String based types (Warning: 6 times slow down)
                 - [X] ISO date
                 - [X] ISO time
                 - [X] ISO datetime
+                - [ ] Web addresses (www, http, https, etc.)
             - [X] Don't create metadata (J2M_ORIGINAL_FIELD) if original_field == generated_field
             - [X] Decode unicode in keys
     - [X] Cli tool

json_to_models/cli.py

@@ -4,23 +4,24 @@
 import json
 import os.path
 import re
+import sys
 from collections import defaultdict
 from datetime import datetime
 from pathlib import Path
 from typing import Any, Callable, Dict, Generator, Iterable, List, Tuple, Type, Union
 
-import json_to_models
-from json_to_models.dynamic_typing import ModelMeta, register_datetime_classes
-from json_to_models.generator import MetadataGenerator
-from json_to_models.models import ModelsStructureType
-from json_to_models.models.attr import AttrsModelCodeGenerator
-from json_to_models.models.base import GenericModelCodeGenerator, generate_code
-from json_to_models.models.dataclasses import DataclassModelCodeGenerator
-from json_to_models.models.structure import compose_models, compose_models_flat
-from json_to_models.registry import (
+from . import __version__ as VERSION
+from .dynamic_typing import ModelMeta, register_datetime_classes
+from .generator import MetadataGenerator
+from .models import ModelsStructureType
+from .models.attr import AttrsModelCodeGenerator
+from .models.base import GenericModelCodeGenerator, generate_code
+from .models.dataclasses import DataclassModelCodeGenerator
+from .models.structure import compose_models, compose_models_flat
+from .registry import (
     ModelCmp, ModelFieldsEquals, ModelFieldsNumberMatch, ModelFieldsPercentMatch, ModelRegistry
 )
-from json_to_models.utils import convert_args
+from .utils import convert_args
 
 STRUCTURE_FN_TYPE = Callable[[Dict[str, ModelMeta]], ModelsStructureType]
 bool_js_style = lambda s: {"true": True, "false": False}.get(s, None)
@@ -75,7 +76,9 @@ def parse_args(self, args: List[str] = None):
             (model_name, (lookup, Path(path)))
             for model_name, lookup, path in namespace.list or ()
         ]
+        self.output_file = namespace.output
         self.enable_datetime = namespace.datetime
+        disable_unicode_conversion = namespace.disable_unicode_conversion
         self.strings_converters = namespace.strings_converters
         merge_policy = [m.split("_") if "_" in m else m for m in namespace.merge]
         structure = namespace.structure
@@ -88,7 +91,7 @@ def parse_args(self, args: List[str] = None):
         self.validate(models, models_lists, merge_policy, framework, code_generator)
         self.setup_models_data(models, models_lists)
         self.set_args(merge_policy, structure, framework, code_generator, code_generator_kwargs_raw,
-                      dict_keys_regex, dict_keys_fields)
+                      dict_keys_regex, dict_keys_fields, disable_unicode_conversion)
 
     def run(self):
         if self.enable_datetime:
@@ -104,7 +107,23 @@ def run(self):
         registry.merge_models(generator)
         registry.generate_names()
         structure = self.structure_fn(registry.models_map)
-        return generate_code(structure, self.model_generator, class_generator_kwargs=self.model_generator_kwargs)
+        output = self.version_string + \
+                 generate_code(structure, self.model_generator, class_generator_kwargs=self.model_generator_kwargs)
+        if self.output_file:
+            with open(self.output_file, "w", encoding="utf-8") as f:
+                f.write(output)
+            return f"Output is written to {self.output_file}"
+        else:
+            return output
+
+    @property
+    def version_string(self):
+        return (
+            'r"""\n'
+            f'generated by json2python-models v{VERSION} at {datetime.now().ctime()}\n'
+            f'command: {" ".join(sys.argv)}\n'
+            '"""\n'
+        )
 
     def validate(self, models, models_list, merge_policy, framework, code_generator):
         """
@@ -149,9 +168,17 @@ def setup_models_data(self, models: Iterable[Tuple[str, Iterable[Path]]],
             for model_name, list_of_gen in models_dict.items()
         }
 
-    def set_args(self, merge_policy: List[Union[List[str], str]],
-                 structure: str, framework: str, code_generator: str, code_generator_kwargs_raw: List[str],
-                 dict_keys_regex: List[str], dict_keys_fields: List[str]):
+    def set_args(
+            self,
+            merge_policy: List[Union[List[str], str]],
+            structure: str,
+            framework: str,
+            code_generator: str,
+            code_generator_kwargs_raw: List[str],
+            dict_keys_regex: List[str],
+            dict_keys_fields: List[str],
+            disable_unicode_conversion: bool
+    ):
         """
         Convert CLI args to python representation and set them to appropriate object attributes
         """
@@ -175,6 +202,7 @@ def set_args(self, merge_policy: List[Union[List[str], str]],
             self.model_generator = getattr(m, cls)
 
         self.model_generator_kwargs = {} if not self.strings_converters else {'post_init_converters': True}
+        self.model_generator_kwargs['convert_unicode'] = not disable_unicode_conversion
         if code_generator_kwargs_raw:
             for item in code_generator_kwargs_raw:
                 if item[0] == '"':
@@ -216,6 +244,11 @@ def _create_argparser(cls) -> argparse.ArgumentParser:
                  "I.e. for file that contains dict {\"a\": {\"b\": [model_data, ...]}} you should\n"
                  "pass 'a.b' as <JSON key>.\n\n"
         )
+        parser.add_argument(
+            "-o", "--output",
+            metavar="FILE", default="",
+            help="Path to output file\n\n"
+        )
         parser.add_argument(
             "-f", "--framework",
             default="base",
@@ -243,22 +276,29 @@ def _create_argparser(cls) -> argparse.ArgumentParser:
             action="store_true",
             help="Enable generation of string types converters (i.e. IsoDatetimeString or BooleanString).\n\n"
         )
+        parser.add_argument(
+            "--disable-unicode-conversion", "--no-unidecode",
+            action="store_true",
+            help="Disabling unicode conversion in fields and class names.\n\n"
+        )
 
         default_percent = f"{ModelFieldsPercentMatch.DEFAULT * 100:.0f}"
         default_number = f"{ModelFieldsNumberMatch.DEFAULT:.0f}"
         parser.add_argument(
             "--merge",
             default=["percent", "number"],
             nargs="+",
-            help=f"Merge policy settings. Default is 'percent_{default_percent} number_{default_number}' (percent of field match\n"
-                 "or number of fields match).\n"
-                 "Possible values are:\n"
-                 "'percent[_<percent>]' - two models had a certain percentage of matched field names.\n"
-                 f"                        Default percent is {default_percent}%%. "
-                 "Custom value could be i.e. 'percent_95'.\n"
-                 "'number[_<number>]'   - two models had a certain number of matched field names.\n"
-                 f"                        Default number of fields is {default_number}.\n"
-                 "'exact'               - two models should have exact same field names to merge.\n\n"
+            help=(
+                f"Merge policy settings. Default is 'percent_{default_percent} number_{default_number}' (percent of field match\n"
+                "or number of fields match).\n"
+                "Possible values are:\n"
+                "'percent[_<percent>]' - two models had a certain percentage of matched field names.\n"
+                f"                        Default percent is {default_percent}%%. "
+                "Custom value could be i.e. 'percent_95'.\n"
+                "'number[_<number>]'   - two models had a certain number of matched field names.\n"
+                f"                        Default number of fields is {default_number}.\n"
+                "'exact'               - two models should have exact same field names to merge.\n\n"
+            )
         )
         parser.add_argument(
             "--dict-keys-regex", "--dkr",
@@ -293,8 +333,7 @@ def _create_argparser(cls) -> argparse.ArgumentParser:
         return parser
 
 
-def main(version_string=None):
-    import sys
+def main():
     import os
 
     if os.getenv("TRAVIS", None) or os.getenv("FORCE_COVERAGE", None):
@@ -305,14 +344,7 @@ def main(version_string=None):
 
     cli = Cli()
     cli.parse_args()
-    if not version_string:
-        version_string = (
-            'r"""\n'
-            f'generated by json2python-models v{json_to_models.__version__} at {datetime.now().ctime()}\n'
-            f'command: {" ".join(sys.argv)}\n'
-            '"""\n'
-        )
-    print(version_string + cli.run())
+    print(cli.run())
 
 
 def path_split(path: str) -> List[str]:
@@ -374,7 +406,7 @@ def safe_json_load(path: Path) -> Union[dict, list]:
     """
     Open file, load json and close it.
     """
-    with path.open() as f:
+    with path.open(encoding="utf-8") as f:
         return json.load(f)
 
 

json_to_models/dynamic_typing/base.py

@@ -1,5 +1,5 @@
 from inspect import isclass
-from typing import Iterable, List, Tuple, Union
+from typing import Any, Generator, Iterable, List, Tuple, Union
 
 ImportPathList = List[Tuple[str, Union[Iterable[str], str, None]]]
 
@@ -50,6 +50,14 @@ def _to_hash_string(self) -> str:
         """
         raise NotImplementedError()
 
+    def iter_child(self) -> Generator['MetaData', Any, None]:
+        yield self
+        for child in self:
+            if isinstance(child, BaseType):
+                yield from child.iter_child()
+            else:
+                yield child
+
 
 class UnknownType(BaseType):
     __slots__ = []

json_to_models/generator.py

@@ -8,6 +8,7 @@
 
 _static_types = {float, bool, int}
 
+
 class MetadataGenerator:
     CONVERTER_TYPE = Optional[Callable[[str], Any]]
 

json_to_models/models/attr.py

@@ -15,15 +15,16 @@ class AttrsModelCodeGenerator(GenericModelCodeGenerator):
     ATTRS = template(f"attr.s{{% if kwargs %}}({KWAGRS_TEMPLATE}){{% endif %}}")
     ATTRIB = template(f"attr.ib({KWAGRS_TEMPLATE})")
 
-    def __init__(self, model: ModelMeta, meta=False, post_init_converters=False, attrs_kwargs: dict = None):
+    def __init__(self, model: ModelMeta, meta=False, post_init_converters=False, attrs_kwargs: dict = None,
+                 convert_unicode=True):
         """
         :param model: ModelMeta instance
         :param meta: Enable generation of metadata as attrib argument
         :param post_init_converters: Enable generation of type converters in __post_init__ methods
         :param attrs_kwargs: kwargs for @attr.s() decorators
         :param kwargs:
         """
-        super().__init__(model, post_init_converters)
+        super().__init__(model, post_init_converters, convert_unicode)
         self.no_meta = not meta
         self.attrs_kwargs = attrs_kwargs or {}
 

json_to_models/models/base.py

@@ -12,7 +12,7 @@
 from .utils import indent
 from ..dynamic_typing import (AbsoluteModelRef, ImportPathList, MetaData,
                               ModelMeta, compile_imports, metadata_to_typing)
-from ..utils import cached_classmethod
+from ..utils import cached_method
 
 METADATA_FIELD_NAME = "J2M_ORIGINAL_FIELD"
 KWAGRS_TEMPLATE = "{% for key, value in kwargs.items() %}" \
@@ -71,20 +71,19 @@ class {{ name }}:
                                      % KWAGRS_TEMPLATE)
     FIELD: Template = template("{{name}}: {{type}}{% if body %} = {{ body }}{% endif %}")
 
-    def __init__(self, model: ModelMeta, post_init_converters=False):
+    def __init__(self, model: ModelMeta, post_init_converters=False, convert_unicode=True):
         self.model = model
         self.post_init_converters = post_init_converters
+        self.convert_unicode = convert_unicode
+        self.model.name = self.convert_class_name(self.model.name)
 
-    @cached_classmethod
-    def convert_field_name(cls, name):
-        if name in keywords_set:
-            name += "_"
-        name = unidecode(name)
-        name = re.sub(r"\W", "", name)
-        if not ('a' <= name[0].lower() <= 'z'):
-            if '0' <= name[0] <= '9':
-                name = ones[int(name[0])] + "_" + name[1:]
-        return inflection.underscore(name)
+    @cached_method
+    def convert_class_name(self, name):
+        return prepare_label(name, convert_unicode=self.convert_unicode)
+
+    @cached_method
+    def convert_field_name(self, name):
+        return inflection.underscore(prepare_label(name, convert_unicode=self.convert_unicode))
 
     def generate(self, nested_classes: List[str] = None, extra: str = "") -> Tuple[ImportPathList, str]:
         """
@@ -144,7 +143,7 @@ def fields(self) -> Tuple[ImportPathList, List[str]]:
 
         :return: imports, list of fields as string
         """
-        required, optional = sort_fields(self.model)
+        required, optional = sort_fields(self.model, unicode_fix=not self.convert_unicode)
         imports: ImportPathList = []
         strings: List[str] = []
         for is_optional, fields in enumerate((required, optional)):
@@ -241,3 +240,15 @@ def sort_kwargs(kwargs: dict, ordering: Iterable[Iterable[str]]) -> dict:
                     current[item] = value
     sorted_dict = {**sorted_dict_1, **kwargs, **sorted_dict_2}
     return sorted_dict
+
+
+def prepare_label(s: str, convert_unicode: bool) -> str:
+    if s in keywords_set:
+        s += "_"
+    if convert_unicode:
+        s = unidecode(s)
+    s = re.sub(r"\W", "", s)
+    if not ('a' <= s[0].lower() <= 'z'):
+        if '0' <= s[0] <= '9':
+            s = ones[int(s[0])] + "_" + s[1:]
+    return s

json_to_models/models/dataclasses.py

@@ -15,15 +15,16 @@ class DataclassModelCodeGenerator(GenericModelCodeGenerator):
     DC_DECORATOR = template(f"dataclass{{% if kwargs %}}({KWAGRS_TEMPLATE}){{% endif %}}")
     DC_FIELD = template(f"field({KWAGRS_TEMPLATE})")
 
-    def __init__(self, model: ModelMeta, meta=False, post_init_converters=False, dataclass_kwargs: dict = None):
+    def __init__(self, model: ModelMeta, meta=False, post_init_converters=False, dataclass_kwargs: dict = None,
+                 convert_unicode=True):
         """
         :param model: ModelMeta instance
         :param meta: Enable generation of metadata as attrib argument
         :param post_init_converters: Enable generation of type converters in __post_init__ methods
         :param dataclass_kwargs: kwargs for @dataclass() decorators
         :param kwargs:
         """
-        super().__init__(model, post_init_converters)
+        super().__init__(model, post_init_converters, convert_unicode)
         self.no_meta = not meta
         self.dataclass_kwargs = dataclass_kwargs or {}