Wish list and ideas for xdol

[thorwhalen]

To collect ideas and code that we may want to add to xdol

[thorwhalen]

Register yaml ValueCodec when yaml available

import yaml

def yaml_load(stream, Loader=yaml.SafeLoader):
    """Load a YAML stream."""
    return yaml.load(stream, Loader)

def yaml_dump(data, Dumper=None, **kwds):
    """Dump data to a YAML stream."""
    return yaml.dump(data, Dumper=Dumper, **kwds)

[thorwhalen]

Appendables and AppendingSettables

In dol.appendable we deal with the functionality of making objects a that have an .append method (and .extend) that have the effect of doing a s[k] = v on some underlying store, where the k and v were derived from the item of a.append(item) via an item2kv parameter function.

So there, we have a MutableMapping backend, and we want to have an Appendable (i.e. "has an append") interface.

Here, the problem we want to solve here is (partial dual) ability to have a s[k] = v as the interface of an Appendable.
For example, say we have a generator of (k, v) pairs that we want to store, and the process that stores it assumes a Settable (i.e. has a __setitem__, i.e. can do s[k] = v) interface, but our backend is actually an Appendable.

For example, we're going through some files (indexed by k) and computing some stats v about them, and storing these(k, v) pairs by doing a s[k] = v... That's all fine and good when the statistics are significant enough to warrant sticking these statistics in individual files or mongo doc, but what if the statistic was just a number of bytes? Would make a lot more sense to collect the (name, size) pairs as csv strings, appending this to an open file for example.

Would be easier if the interface was s.append((k,v)) right?
But then what?
You add some code around the s[k] = v to condition the way you save stuff according to the situation?
No. You're more mature than that and know about dependency injection and the open-closed principle.
It's better you believe, or maybe you don't even have the choice, to adapt to the interface that is there and make an s so that s[k] = v does what you want it to do.

(Note that I said partial dual because here all we're assuming is that s is Settable (can do s[k] = v), not that it's a Mapping, or even Iterable. We're constraining ourselves to the sole concern of storing the (k, v) pair.)

Here are some ideas around that.

First let's setup a few types to annotate our code with, and convey the concepts and language around them.

from typing import Protocol, Iterable

class Settable(Protocol):
    """Objects where you can do self[k] = v"""
    def __setitem__(self, k, v):
        """That is, self[k] = v"""

class Appendable(Protocol):
    """Objects that have an append method"""
    def append(self, obj):
        """Usually has the effect of adding obj to self. Usually at "the end" of it"""

# In case we want to use it later:
class Extendable(Protocol):
    def extend(self, objs: Iterable): 
        """Usually has the effect of adding an iteralble of objs to self.
        Usually at "the end" of it."""

Now we'll make a few appendables with different backends to them.
That is, we want to have objects that have an append method, but that append to a string, or to a file, or to a file but through an open file pointer...

# Append to a string------------------------------------
class AppendableString(str):
    _repr = str

    def __init__(self, initial_string=""):
        self._chars = list(initial_string)

    def append(self, string):
        self._chars.extend(list(string))

    def __str__(self):
        return ''.join(self._chars)
    
    def __eq__(self, __value: object) -> bool:
        return str(self) == str(__value)
    
    def __repr__(self) -> str:
        return self._repr(self)
    
    def ch_repr(self, repr_func):
        """To change the repr function 
        (for example, if strings are too big, you way want to contain their display)
        """
        self._repr = repr_func
        return self


# Usage
a = AppendableString()
assert isinstance(a, str)
a.append('app')
a.append('end')
assert str(a) == 'append'
a == 'append'

# Append to a file ------------------------------------

from dataclasses import dataclass

@dataclass
class AppendableFile:
    filepath: str

    def append(self, obj):
        with open(self.filepath, 'a') as f:
            f.write(obj)
        return self
    
# Usage
from tempfile import NamedTemporaryFile
filepath = NamedTemporaryFile().name
a = AppendableFile(filepath)
a = a.append('app')
a = a.append('end')
from pathlib import Path
assert Path(filepath).read_text() == 'append'

# Append to an open file ------------------------------------

import contextlib

@dataclass
class AppendableFilePointer:
    """Objects where obj.append(v) appends v to the obj.filepath open file. 
    
    AppendableFilePointer is a context manager, so you can use it in a with statement,
    which will have the effect of opening and closing the file for you. 
    Normal usage is:

    >>> from tempfile import NamedTemporaryFile
    >>> filepath = NamedTemporaryFile().name
    >>> with AppendableFilePointer(filepath) as afp:
    ...     afp.append('app')
    ...     afp.append('end')
    >>> from pathlib import Path
    >>> assert Path(filepath).read_text() == 'append'
    """
    filepath: str

    def __enter__(self):
        self.f = open(self.filepath, 'a')
        return self
    
    def __exit__(self, exc_type, exc_val, exc_tb):
        self.f.close()
        return False
    
    def append(self, obj):
        self.f.write(obj)
        return self
    
# Usage
from tempfile import NamedTemporaryFile
filepath = NamedTemporaryFile().name
with AppendableFilePointer(filepath) as afp:
    afp.append('app')
    afp.append('end')
from pathlib import Path
assert Path(filepath).read_text() == 'append'


def return_kv_tuple(k, v):
    return k, v

def repr_of_appendale_attr(obj):
    return repr(obj.appendable)

def repr_of_obj(obj):
    return obj.super().__repr__()

from typing import Callable

@dataclass
class AppendingSettable:
    """A MutableMapping s where s[k] = v has the effect of s.append((k, v)
    
    The appendable is an object that has an append method.
    The encoder is a function that encodes the (k, v) pair into whatever format the 
    appendable expects.

    """
    appendable: Appendable
    kv_encoder: Callable = return_kv_tuple
    repr_func: Callable = repr_of_appendale_attr

    def __setitem__(self, k, v):
        encoded_kv = self.kv_encoder(k, v)
        self.appendable.append(encoded_kv)

    def __repr__(self) -> str:
        return self.repr_func(self)
    
    def __eq__(self, __value: object) -> bool:
        if isinstance(__value, AppendingSettable):
            return self.appendable == __value.appendable
        else:
            return self.appendable == __value

    
# Usage
s = AppendingSettable(appendable=[])
s['a'] = 1
s['b'] = 2
s.appendable == [('a', 1), ('b', 2)]

Here are now a few examples using AppendingSettable but with different appendables and kv_encoders.

def csv_kv_encoder(k, v):
    return f"{k},{v}\n"

s = AppendingSettable(appendable=list(), kv_encoder=csv_kv_encoder)
s['a'] = 1
s['b'] = 2
assert s == ['a,1\n', 'b,2\n']


s = AppendingSettable(appendable=AppendableString(), kv_encoder=csv_kv_encoder)
s['a'] = 1
s['b'] = 2
assert s == 'a,1\nb,2\n'

import yaml

def yaml_kv_encoder(k, v):
    return yaml.dump({k: v})

s = AppendingSettable(AppendableString(), yaml_kv_encoder)
s['a'] = 1
s['b'] = 2
assert s == '''a: 1
b: 2
'''

[thorwhalen]

Using an ObjectProxy (dol has one, but let's use wrapt, which I trust more, and is maintained by others...)

This gives us cleaner code, that also is a lot more "transparent" as far as the appendable that is wrapped.
With this one, we keep all the methods of the wrapped appendable!

from wrapt import ObjectProxy

def return_kv_tuple(k, v):
    return k, v

class AppendingSettableProxy(ObjectProxy):
    """A MutableMapping s where s[k] = v has the effect of s.append((k, v)
    
    The appendable is an object that has an append method.
    The encoder is a function that encodes the (k, v) pair into whatever format the 
    appendable expects.

    """
    kv_encoder = return_kv_tuple

    def __setitem__(self, k, v):
        encoded_kv = self.kv_encoder(k, v)
        self.__wrapped__.append(encoded_kv)

    @classmethod
    def with_kv_encoder(cls, name, kv_encoder, *, kv_encoder_wrap=staticmethod):
        return type(name, (cls,), {'kv_encoder': kv_encoder_wrap(kv_encoder)})
    

# Usage
s = AppendingSettable([])
s['a'] = 1
s['b'] = 2
s == [('a', 1), ('b', 2)]


def csv_kv_encoder(k, v):
    return f"{k},{v}\n"

AppendingSettableCsv = AppendingSettableProxy.with_kv_encoder('AppendingSettableCsv', csv_kv_encoder)

s = AppendingSettableCsv(list())
s['a'] = 1
s['b'] = 2
assert s == ['a,1\n', 'b,2\n']

s = AppendingSettableCsv(AppendableString())
s['a'] = 1
s['b'] = 2
assert s == 'a,1\nb,2\n'

The only problem here is that it's not picklable (standar serialization of python).
Our previous versions were.
Here, if you try to pickle s, you'll get this error:

NotImplementedError: object proxy must define __reduce_ex__()

So I added a __reduce_ex__, but it doesn't work yet (it works for dumps, but not for loads):

class SerializableAppendingSettableProxy(AppendingSettableProxy):
    """A MutableMapping s where s[k] = v has the effect of s.append((k, v)
    
    The appendable is an object that has an append method.
    The encoder is a function that encodes the (k, v) pair into whatever format the 
    appendable expects.

    """
    def __reduce_ex__(self, protocol):
        cls = type(self)
        kv_encoder = self.kv_encoder
        if isinstance(kv_encoder, staticmethod):
            kv_encoder = kv_encoder.__func__
        return (cls.with_kv_encoder, 
                (cls.__name__, kv_encoder), 
                self.__wrapped__)  # Return the wrapped object directly as the state

    def __setstate__(self, state):
        self.__wrapped__ = state

SerializableAppendingSettableCsv = SerializableAppendingSettableProxy.with_kv_encoder('AppendingSettableCsv', csv_kv_encoder)

s = SerializableAppendingSettableCsv(AppendableString())
s['a'] = 1
s['b'] = 2
assert s == 'a,1\nb,2\n'


import pickle

encoded = pickle.dumps(s)
ss = pickle.loads(encoded)

assert ss == s