Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support serializing timedelta and relativedelta to string (hocon, json etc.) #263

Merged
merged 6 commits into from
Jul 12, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 59 additions & 1 deletion pyhocon/converter.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import json
import re
import sys
from datetime import timedelta

from pyhocon import ConfigFactory
from pyhocon.config_tree import ConfigQuotedString
Expand All @@ -16,6 +17,11 @@
basestring = str
unicode = str

try:
from dateutil.relativedelta import relativedelta
except Exception:
relativedelta = None


class HOCONConverter(object):
@classmethod
Expand Down Expand Up @@ -55,6 +61,8 @@ def to_json(cls, config, compact=False, indent=2, level=0):
)
lines += ',\n'.join(bet_lines)
lines += '\n{indent}]'.format(indent=''.rjust(level * indent, ' '))
elif cls._is_timedelta_like(config):
lines += cls._timedelta_to_str(config)
elif isinstance(config, basestring):
lines = json.dumps(config, ensure_ascii=False)
elif config is None or isinstance(config, NoneValue):
Expand Down Expand Up @@ -130,6 +138,8 @@ def to_hocon(cls, config, compact=False, indent=2, level=0):
lines = '"""{value}"""'.format(value=config.value) # multilines
else:
lines = '"{value}"'.format(value=cls.__escape_string(config.value))
elif cls._is_timedelta_like(config):
lines += cls._timedelta_to_hocon(config)
elif config is None or isinstance(config, NoneValue):
lines = 'null'
elif config is True:
Expand Down Expand Up @@ -171,6 +181,8 @@ def to_yaml(cls, config, compact=False, indent=2, level=0):
bet_lines.append('{indent}- {value}'.format(indent=''.rjust(level * indent, ' '),
value=cls.to_yaml(item, compact, indent, level + 1)))
lines += '\n'.join(bet_lines)
elif cls._is_timedelta_like(config):
lines += cls._timedelta_to_str(config)
elif isinstance(config, basestring):
# if it contains a \n then it's multiline
lines = config.split('\n')
Expand All @@ -189,13 +201,14 @@ def to_yaml(cls, config, compact=False, indent=2, level=0):
return lines

@classmethod
def to_properties(cls, config, compact=False, indent=2, key_stack=[]):
def to_properties(cls, config, compact=False, indent=2, key_stack=None):
"""Convert HOCON input into a .properties output

:return: .properties string representation
:type return: basestring
:return:
"""
key_stack = key_stack or []

def escape_value(value):
return value.replace('=', '\\=').replace('!', '\\!').replace('#', '\\#').replace('\n', '\\\n')
Expand All @@ -210,6 +223,8 @@ def escape_value(value):
for index, item in enumerate(config):
if item is not None:
lines.append(cls.to_properties(item, compact, indent, stripped_key_stack + [str(index)]))
elif cls._is_timedelta_like(config):
lines.append('.'.join(stripped_key_stack) + ' = ' + cls._timedelta_to_str(config))
elif isinstance(config, basestring):
lines.append('.'.join(stripped_key_stack) + ' = ' + escape_value(config))
elif config is True:
Expand Down Expand Up @@ -276,3 +291,46 @@ def __escape_match(cls, match):
@classmethod
def __escape_string(cls, string):
return re.sub(r'[\x00-\x1F"\\]', cls.__escape_match, string)

@classmethod
def _is_timedelta_like(cls, config):
return isinstance(config, timedelta) or relativedelta is not None and isinstance(config, relativedelta)

@classmethod
def _timedelta_to_str(cls, config):
if isinstance(config, relativedelta):
time_delta = cls._relative_delta_to_timedelta(config)
else:
time_delta = config
return str(int(time_delta.total_seconds() * 1000))

@classmethod
def _timedelta_to_hocon(cls, config):
"""
:type config: timedelta
"""
if relativedelta is not None and isinstance(config, relativedelta):
if config.hours > 0:
return str(config.hours) + ' hours'
Copy link

@dolfinus dolfinus Jul 13, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Pyhocon supports multiple input formats for timedelta, like:

1s
1 s
1 second
1 seconds
1 sec

But output format is fixed,and thus some applications expecting format other than this one will fail while parsing output config.

For example, Spark config values could contain time units like 150ms, but converter will save it as 150000 microseconds (there is just no case for milliseconds).

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@dolfinus
output is specifically for hocon format. if spark config is different then it is not hocon and requires it's own output. don't know if it's in the scope of this library.

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't see here any specification about default unit format - all the options are described as supported by a parser. There is no preferred unit format, so why does implementation have one without any capability to change it whenever user wants?

Spark config is just the simple .properties file which is successfully read by any HOCON library implementation. But the output file saved by the current package cannot be changed at all - no class or instance options for setting up some preferred format.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@dolfinus
"so why does implementation have one without any capability to change it whenever user wants?" - because that is the implementation I submitted.

"Spark config is just the simple .properties" - so it's not hocon.

"file which is successfully read by any HOCON library implementation" - I do believe you are free to contribute code which enhances this library to your additional requirements. That's what I did.

ps.
From my perspective, you are coming off a bit demanding, considering this is open-source and based on free contributions from people who care enough to do so. less talk, more code :)

elif config.minutes > 0:
return str(config.minutes) + ' minutes'

if config.days > 0:
return str(config.days) + ' days'
elif config.seconds > 0:
return str(config.seconds) + ' seconds'
elif config.microseconds > 0:
return str(config.microseconds) + ' microseconds'
else:
return '0 seconds'

@classmethod
def _relative_delta_to_timedelta(cls, relative_delta):
"""
:type relative_delta: relativedelta
"""
return timedelta(days=relative_delta.days,
hours=relative_delta.hours,
minutes=relative_delta.minutes,
seconds=relative_delta.seconds,
microseconds=relative_delta.microseconds)
22 changes: 22 additions & 0 deletions tests/test_converter.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
# -*- encoding: utf-8 -*-
from datetime import timedelta

from pyhocon import ConfigTree
from pyhocon.converter import HOCONConverter
Expand Down Expand Up @@ -106,3 +107,24 @@ def test_format_multiline_string(self):
assert 'a = """\nc"""' == to_hocon({'a': '\nc'})
assert 'a = """b\n"""' == to_hocon({'a': 'b\n'})
assert 'a = """\n\n"""' == to_hocon({'a': '\n\n'})

def test_format_time_delta(self):
for time_delta, expected_result in ((timedelta(days=0), 'td = 0 seconds'),
(timedelta(days=5), 'td = 5 days'),
(timedelta(seconds=51), 'td = 51 seconds'),
(timedelta(microseconds=786), 'td = 786 microseconds')):
assert expected_result == to_hocon({'td': time_delta})

def test_format_relativedelta(self):
try:
from dateutil.relativedelta import relativedelta
except Exception:
return

for time_delta, expected_result in ((relativedelta(seconds=0), 'td = 0 seconds'),
(relativedelta(hours=0), 'td = 0 seconds'),
(relativedelta(days=5), 'td = 5 days'),
(relativedelta(weeks=3), 'td = 21 days'),
(relativedelta(hours=2), 'td = 2 hours'),
(relativedelta(minutes=43), 'td = 43 minutes'),):
assert expected_result == to_hocon({'td': time_delta})
8 changes: 7 additions & 1 deletion tests/test_tool.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ class TestHOCONConverter(object):
h = null
i = {}
"a.b" = 2
td_days = 4 days
"""

CONFIG = ConfigFactory.parse_string(CONFIG_STRING)
Expand All @@ -41,7 +42,8 @@ class TestHOCONConverter(object):
"g": [],
"h": null,
"i": {},
"a.b": 2
"a.b": 2,
"td_days": 345600000
}
"""

Expand All @@ -63,6 +65,7 @@ class TestHOCONConverter(object):
h = null
i {}
"a.b" = 2
td_days = 4 days
"""

EXPECTED_COMPACT_HOCON = \
Expand All @@ -81,6 +84,7 @@ class TestHOCONConverter(object):
h = null
i {}
"a.b" = 2
td_days = 4 days
"""

EXPECTED_YAML = \
Expand All @@ -102,6 +106,7 @@ class TestHOCONConverter(object):
h: null
i:
a.b: 2
td_days: 345600000
"""

EXPECTED_PROPERTIES = \
Expand All @@ -117,6 +122,7 @@ class TestHOCONConverter(object):
f1 = true
f2 = false
a.b = 2
td_days = 345600000
"""

def test_to_json(self):
Expand Down