Add interfaces and env dict/list parsing (#25) #minor

* add more interface and env dict/list parsing

* fix bad renaming

* fix tests

* change publishing system with auto semver

* fix poetry dev version parsing

Author: zifeo

.flake8

@@ -1,3 +1,3 @@
 [flake8]
-ignore = E501
+ignore = E501, E203
 import-order-style = google

.github/workflows/ci.yaml

@@ -1,6 +1,8 @@
 name: CI
 
-on: [push, pull_request]
+on:
+  - push
+  - pull_request
 
 jobs:
 
@@ -30,22 +32,37 @@ jobs:
 
   publish:
     needs: test
-    if: startsWith(github.ref, 'refs/tags/v')
+    if: github.ref == 'refs/heads/master' && github.event_name == 'push'
     runs-on: ubuntu-latest
 
     steps:
     - uses: actions/checkout@v2
-    - name: Set up Python ${{ matrix.python-version }}
+      with:
+        fetch-depth: '0'
+    - name: Set up Python
       uses: actions/setup-python@v2
       with:
-        python-version: 3.9
+        python-version: 3.x
     - name: Install poetry
       uses: abatilo/actions-poetry@v2.0.0
+    - name: Bump version and push tag
+      id: version
+      uses: anothrNick/github-tag-action@1.36.0
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        WITH_V: true
     - name: Publish
       env:
         POETRY_PYPI_TOKEN_PYPI: ${{ secrets.PYPI_TOKEN }}
+        NEW_TAG: ${{ steps.version.outputs.new_tag }}
       run: |
         python -m pip install --upgrade pip
         poetry install
+        poetry version $NEW_TAG
         poetry build
         poetry publish
+    - name: Create a GitHub release
+      uses: ncipollo/release-action@v1
+      with:
+        tag: ${{ steps.version.outputs.new_tag }}
+        name: Release ${{ steps.version.outputs.new_tag }}

.pre-commit-config.yaml

@@ -1,18 +1,15 @@
 repos:
 - repo: https://github.com/sqlalchemyorg/zimports/
-  rev: 0.3.0
+  rev: v0.4.1
   hooks:
   - id: zimports
 - repo: https://github.com/psf/black
-  rev: 21.5b0
+  rev: 21.9b0
   hooks:
   - id: black
 - repo: https://gitlab.com/pycqa/flake8
-  rev: 3.9.1
+  rev: 3.9.2
   hooks:
   - id: flake8
-- repo: https://github.com/zifeo/pre-commit
-  rev: v0.1.4
-  hooks:
-  - id: poetry-tag-checker
+
 

README.md

@@ -36,6 +36,7 @@ str_name = test
 str_name = ${?HOSTNAME}
 dash-to-underscore = true
 float_num = 2.2
+# this is a comment
 list_data = [
     a
     b
@@ -120,6 +121,11 @@ conf = dataconf.loads(str_conf, Base)
 ```python
 import dataconf
 
+conf = dataconf.string('{ name: Test }', Config)
+conf = dataconf.env('PREFIX_', Config)
+conf = dataconf.url('https://github.com/zifeo/dataconf/blob/master/.pre-commit-config.yaml', Config)
+conf = dataconf.file('confs/test.{hocon,json,yaml,properties}', Config)
+
 conf = dataconf.loads('confs/test.hocon', Config)
 conf = dataconf.loads('confs/test.json', Config)
 conf = dataconf.loads('confs/test.yaml', Config)
@@ -134,6 +140,53 @@ dataconf.dumps('confs/test.properties', out='properties')
 Follows same api as python JSON package (e.g. `load`, `loads`, `dump`, `dumps`). 
 For full HOCON capabilities see [here](https://github.com/chimpler/pyhocon/#example-of-hocon-file).
 
+## Env dict/list parsing
+
+```
+PREFIX_VAR=a
+PREFIX_VAR_NAME=b
+PREFIX_TEST__NAME=c
+PREFIX_LS_0=d
+PREFIX_LS_1=e
+PREFIX_LSLS_0_0=f
+PREFIX_LSOB_0__NAME=g
+PREFIX_NESTED="{ name: Test }"
+PREFIX_SUB="{ value: ${PREFIX_VAR} }"
+```
+
+is equivalent to
+
+```
+{
+    var = a
+    var_name = b
+    test {
+        name = c
+    }
+    ls = [
+        d
+        e
+    ]
+    lsls = [
+        [
+            f
+        ]
+    ]
+    lsob = [
+        {
+            name = g
+        }
+    ]
+    nested {
+        name: Test
+    }
+    sub {
+        # will have value "a" at parsing, useful for aliases
+        value = ${PREFIX_VAR}
+    }
+}
+```
+
 ## CI
 
 ```shell

dataconf/__init__.py

@@ -1,7 +1,21 @@
-from dataconf.utils import dump
-from dataconf.utils import dumps
-from dataconf.utils import load
-from dataconf.utils import loads
+from dataconf.main import dump
+from dataconf.main import dumps
+from dataconf.main import env
+from dataconf.main import file
+from dataconf.main import load
+from dataconf.main import loads
+from dataconf.main import string
+from dataconf.main import url
 from dataconf.version import __version__
 
-__all__ = ["load", "loads", "dump", "dumps", "__version__"]
+__all__ = [
+    "load",
+    "loads",
+    "dump",
+    "dumps",
+    "env",
+    "url",
+    "file",
+    "string",
+    "__version__",
+]

dataconf/exceptions.py

@@ -12,3 +12,7 @@ class MalformedConfigException(Exception):
 
 class UnexpectedKeysException(Exception):
     pass
+
+
+class EnvListOrderException(Exception):
+    pass

dataconf/main.py

@@ -0,0 +1,65 @@
+import os
+
+from dataconf import utils
+from dataconf.exceptions import MalformedConfigException
+from pyhocon import ConfigFactory
+from pyhocon import HOCONConverter
+import pyparsing
+
+
+def __parse_config_tree(conf, clazz):
+    try:
+        return utils.__parse(conf, clazz, "")
+    except pyparsing.ParseSyntaxException as e:
+        raise MalformedConfigException(
+            f'parsing failure line {e.lineno} character {e.col}, got "{e.line}"'
+        )
+
+
+def env(prefix: str, clazz):
+    conf = ConfigFactory.from_dict(utils.__dict_list_parsing(prefix, os.environ))
+    return __parse_config_tree(conf, clazz)
+
+
+def string(s: str, clazz):
+    conf = ConfigFactory.parse_string(s)
+    return __parse_config_tree(conf, clazz)
+
+
+def url(uri: str, clazz):
+    conf = ConfigFactory.parse_URL(uri)
+    return __parse_config_tree(conf, clazz)
+
+
+def file(path: str, clazz):
+    conf = ConfigFactory.parse_file(path)
+    return __parse_config_tree(conf, clazz)
+
+
+def load(path: str, clazz):
+    return file(path, clazz)
+
+
+def loads(s: str, clazz):
+    return string(s, clazz)
+
+
+def dump(file: str, instance: object, out: str):
+    with open(file, "w") as f:
+        f.write(dumps(instance, out=out))
+
+
+def dumps(instance: object, out: str):
+    conf = utils.__generate(instance, "")
+
+    if out:
+        if out.lower() == "hocon":
+            return HOCONConverter.to_hocon(conf)
+        if out.lower() == "yaml":
+            return HOCONConverter.to_yaml(conf)
+        if out.lower() == "json":
+            return HOCONConverter.to_json(conf)
+        if out.lower() == "properties":
+            return HOCONConverter.to_properties(conf)
+
+    return conf

dataconf/utils.py

@@ -6,17 +6,18 @@
 from typing import get_origin
 from typing import Union
 
+from dataconf.exceptions import EnvListOrderException
 from dataconf.exceptions import MalformedConfigException
 from dataconf.exceptions import MissingTypeException
 from dataconf.exceptions import TypeConfigException
 from dataconf.exceptions import UnexpectedKeysException
 from dateutil.relativedelta import relativedelta
 from pyhocon import ConfigFactory
-from pyhocon import HOCONConverter
 from pyhocon.config_tree import ConfigList
 from pyhocon.config_tree import ConfigTree
 import pyparsing
 
+
 NoneType = type(None)
 
 
@@ -181,42 +182,74 @@ def __generate(value: object, path):
     return value
 
 
-def load(file: str, clazz):
-    try:
-        conf = ConfigFactory.parse_file(file)
-        return __parse(conf, clazz, "")
-    except pyparsing.ParseSyntaxException as e:
-        raise MalformedConfigException(
-            f'parsing failure line {e.lineno} character {e.col}, got "{e.line}"'
-        )
+def __dict_list_parsing(prefix: str, obj):
+    ret = {}
 
+    def set_lens(p, focus, v):
 
-def loads(string: str, clazz):
-    try:
-        conf = ConfigFactory.parse_string(string)
-        return __parse(conf, clazz, "")
-    except pyparsing.ParseSyntaxException as e:
-        raise MalformedConfigException(
-            f'parsing failure line {e.lineno} character {e.col}, got "{e.line}"'
-        )
+        # value
+        if len(p) == 1:
+            # []x
+            if isinstance(focus, list):
+                if p[0] != len(focus):
+                    raise EnvListOrderException
+                focus.append(v)
+            # {}x
+            else:
+                focus[p[0]] = v
+            return
+
+        # dict
+        if p[1] == "":
+
+            if p[0] not in focus:
+                # []{x}
+                if isinstance(focus, list):
+                    if p[0] != len(focus):
+                        raise EnvListOrderException
+                    focus.append({})
+                # {}{x}
+                else:
+                    focus[p[0]] = {}
 
+            return set_lens(p[2:], focus[p[0]], v)
 
-def dump(file: str, instance: object, out: str):
-    with open(file, "w") as f:
-        f.write(dumps(instance, out=out))
+        # list
+        if isinstance(p[1], int):
 
+            if p[0] not in focus:
+                # [][x]
+                if isinstance(focus, list):
+                    if p[1] != len(focus):
+                        raise EnvListOrderException
+                    focus.append([])
+                # {}[x]
+                else:
+                    focus[p[0]] = []
 
-def dumps(instance: object, out: str):
-    conf = __generate(instance, "")
+            return set_lens(p[1:], focus[p[0]], v)
+
+        # compose path
+        return set_lens([f"{p[0]}_{p[1]}"] + p[2:], focus, v)
+
+    def int_or_string(v):
+        try:
+            return int(v)
+        except ValueError:
+            return v
+
+    if not prefix.endswith("_"):
+        prefix = f"{prefix}_"
+
+    for k, v in sorted(obj.items(), key=lambda x: x[0]):
+        if k.startswith(prefix):
+
+            path = [int_or_string(e) for e in k[len(prefix) :].lower().split("_")]
+            try:
+                value = ConfigFactory.parse_string(v)
+            except pyparsing.ParseSyntaxException:
+                value = v
 
-    if out:
-        if out.lower() == "hocon":
-            return HOCONConverter.to_hocon(conf)
-        if out.lower() == "yaml":
-            return HOCONConverter.to_yaml(conf)
-        if out.lower() == "json":
-            return HOCONConverter.to_json(conf)
-        if out.lower() == "properties":
-            return HOCONConverter.to_properties(conf)
+            set_lens(path, ret, value)
 
-    return conf
+    return ret

dataconf/version.py

@@ -1,3 +1,3 @@
 import importlib.metadata
 
-__version__ = importlib.metadata.version("dataconf")
+__version__ = importlib.metadata.version("dataconf").lstrip("v")

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dataconf"
-version = "0.2.1"
+version = "0+dev"
 description = "Lightweight configuration with automatic dataclasses parsing (HOCON/JSON/YAML/PROPERTIES)"
 authors = []
 license = "Apache2"