Skip to content

Core

KoshFileLoader

Bases: KoshLoader

Kosh loader to load content from files

Source code in kosh/loaders/core.py 
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
class KoshFileLoader(KoshLoader):
    """Kosh loader to load content from files"""
    types = {"file": []}

    def __init__(self, obj, **kwargs):
        super(KoshFileLoader, self).__init__(obj, **kwargs)

    def open(self, mode='r'):
        """open/load the matching Kosh Sina File

        :param mode: mode to open the file in, defaults to 'r'
        :type mode: str, optional
        :return: Kosh File object
        """
        return KoshGenericObjectFromFile(self.uri, mode, self.obj)

    def extract(self, feature, format):
        """extract return a feature from the loaded object.

        :param feature: variable to read from file
        :type feature: str
        :param format: desired output format
        :type format: str
        :return: data
        """
        with open(self.uri) as f:
            return f.read()

    def list_features(self, *args, **kargs):
        """list_features list features in file,

        :return: list of features available in file
        :rtype: list
        """
        return []

    def describe_feature(self, feature):
        """describe a feature

        :param feature: feature (variable) to read, defaults to None
        :type feature: str, optional if loader does not require this
        :return: dictionary describing the feature
        :rtype: dict
        """
        if feature not in self.list_features():
            raise ValueError("feature {feature} is not available".format(feature=feature))
        return {}

describe_feature(feature)

describe a feature

Parameters:

Name Type Description Default
feature str, optional if loader does not require this

feature (variable) to read, defaults to None

 required

Returns:

Type Description
dict

dictionary describing the feature
Source code in kosh/loaders/core.py 
335
336
337
338
339
340
341
342
343
344
345
def describe_feature(self, feature):
    """describe a feature

    :param feature: feature (variable) to read, defaults to None
    :type feature: str, optional if loader does not require this
    :return: dictionary describing the feature
    :rtype: dict
    """
    if feature not in self.list_features():
        raise ValueError("feature {feature} is not available".format(feature=feature))
    return {}

extract(feature, format)

extract return a feature from the loaded object.

Parameters:

Name Type Description Default
feature str

variable to read from file

 required
format str

desired output format

 required

Returns:

Type Description

data
Source code in kosh/loaders/core.py 
315
316
317
318
319
320
321
322
323
324
325
def extract(self, feature, format):
    """extract return a feature from the loaded object.

    :param feature: variable to read from file
    :type feature: str
    :param format: desired output format
    :type format: str
    :return: data
    """
    with open(self.uri) as f:
        return f.read()

list_features(*args, **kargs)

list_features list features in file,

Returns:

Type Description
list

list of features available in file
Source code in kosh/loaders/core.py 
327
328
329
330
331
332
333
def list_features(self, *args, **kargs):
    """list_features list features in file,

    :return: list of features available in file
    :rtype: list
    """
    return []

open(mode='r')

open/load the matching Kosh Sina File

Parameters:

Name Type Description Default
mode str, optional

mode to open the file in, defaults to 'r'

 'r'

Returns:

Type Description

Kosh File object
Source code in kosh/loaders/core.py 
306
307
308
309
310
311
312
313
def open(self, mode='r'):
    """open/load the matching Kosh Sina File

    :param mode: mode to open the file in, defaults to 'r'
    :type mode: str, optional
    :return: Kosh File object
    """
    return KoshGenericObjectFromFile(self.uri, mode, self.obj)

KoshGenericObjectFromFile

Bases: KoshSinaObject

Kosh object pointing to a file

Source code in kosh/loaders/core.py 
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
class KoshGenericObjectFromFile(KoshSinaObject):
    """Kosh object pointing to a file"""

    def __init__(self, uri, mode, obj, *args, **kwds):
        super(KoshGenericObjectFromFile, self).__init__(obj.id, obj.__store__,
                                                        obj.__type__, obj.__record_handler__, protected=["file_obj"])
        self.file_obj = open(uri, mode, *args, **kwds)

    def __enter__(self):
        return self.file_obj

    def __exit__(self, *args):
        self.file_obj.close()

    def get(self, *args, **kargs):
        """Reads the file all arguments are ignored"""
        return self.file_obj.read()

get(*args, **kargs)

Reads the file all arguments are ignored

Source code in kosh/loaders/core.py 
28
29
30
def get(self, *args, **kargs):
    """Reads the file all arguments are ignored"""
    return self.file_obj.read()

KoshLoader

Bases: KoshExecutionGraph

Parameters:

Name Type Description Default
types dict

types is a dictionary on known type that can be loaded as key and export format as value, defaults to {"dataset": []}

 required
Source code in kosh/loaders/core.py 
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
class KoshLoader(KoshExecutionGraph):
    """
    :param types: types is a dictionary on known type that can be loaded
    as key and export format as value, defaults to {"dataset": []}
    :type types: dict
    """
    types = {"dataset": []}

    def __init__(self, obj, mime_type=None, uri=None, requestorId=None):
        """KoshLoader generic Kosh loader
        :param obj: object the loader will try to load from
        :type obj: object
        :param mime_type: If you want to force the mime_type to use
        :type mime_type: str
        :param uri: If you want/need to force the uri to use
        :type uri: str
        :param requestorId: The id of the dataset requesting data
        :type requestorId: str
        """
        self.requestorId = requestorId
        self.signature = hashlib.sha256(repr(self.__class__).encode())
        self.signature = self.update_signature(obj.id)
        if mime_type is None:
            self._mime_type = obj.mime_type
        else:
            self._mime_type = mime_type
        if uri is None:
            try:
                self.uri = obj.uri
            except AttributeError:  # Not uri on this
                self.uri = None
        else:
            self.uri = uri
        rec = obj.__store__.get_record(obj.id)
        if (rec["type"] not in obj.__store__._kosh_reserved_record_types and mime_type is None)\
                or rec["type"] in ["__kosh_storeinfo__", obj.__store__._ensembles_type]:
            self._mime_type = "dataset"
        if self._mime_type not in self.types:
            open_anything = False
            for t in self.types:
                if t == "dataset":  # special skipping
                    continue
                if len(self.types[t]) == 0:
                    open_anything = True
            if not open_anything:
                raise RuntimeError(
                    "will not be able to load object of type {mime_type}".format(mime_type=self._mime_type))
        self.obj = obj
        self.__listed_features = None

    def get_requestor(self):
        """Returns the Kosh object requesting the data"""
        try:
            return self.obj.__store__.open(self.requestorId)
        except Exception:  # some kosh object do not have an open function
            return self.obj.__store__._load(self.requestorId)

    def known_types(self):
        """Lists types of Kosh objects this loader can handle

        :return: list of Kosh type this loader can handle
        :rtype: list
        """
        return list(self.types.keys())

    def known_load_formats(self, atype):
        """Lists all the formats this loader knows how to export to for a given type

        :param atype: type we wish to to the formats for
        :type format: str
        :return: list of format this type can be exported to by the loader
        :rtype: list
        """
        return self.types.get(atype, [])

    def open(self, mode="r"):
        """Open function
        :param mode: mode to open the object in
        :type mode: str
        :return: opened object
        :rtype: object"""
        return self

    def update_signature(self, *args, **kargs):
        """Updated the signature based to a set of args and kargs
        :param *args: as many arguments as you want
        :type *args: list
        :param **kargs: key=value style arguments
        :type **kargs: dict
        :return: updated signature
        :rtype: str
        """
        signature = self.signature.copy()
        for arg in args:
            signature.update(repr(arg).encode())
        for kw in kargs:
            signature.update(repr(kw).encode())
            signature.update(repr(kargs[kw]).encode())
        return signature

    def get_execution_graph(self, feature, transformers=[]):
        """Generates the execution graph to extract a feature and possibly transform it.

        :param feature: desired feature
        :type feature: str
        :param format: desired output format
        :type format: str
        :param transformers: A list of transformers to use after the data is loaded
        :type transformers: kosh.transformer.KoshTranformer
        :return: execution graph to get to the possibly transformed feature
        :rtype: networkx.OrderDiGraph
        """
        # Let's get the execution path
        G = get_graph(self._mime_type, self, transformers)
        return G

    def get(self, feature, format=None, transformers=[],
            use_cache=True, cache_file_only=False, cache_dir=None,
            **kargs):
        """Extracts a feature and possibly transforms it
        :param feature: desired feature
        :type feature: str
        :param format: desired output format
        :type format: str
        :param transformers: A list of transformers to use after the data is loaded
        :type transformers: kosh.transformer.KoshTranformer
        :param use_cache: Try to use cached data if available
        :type use_cache: bool
        :param cache_file_only: If True, simply return name of cache_file
        :type cache_file_only: bool
        :param cache_dir: where do we cache the result?
        :type cache_dir: str
        **kargs will be stored on loader object
        format and feature are stored on the object for extraction by extraction functions
        This function calls first the loader's preprocess function
        This is followed by an actual data extraction via the 'extract' function
        Finally 'postprocess' is called on the extracted data

        Reserved keyword:
        preprocess: function use to preprocess (default to self.preprocess)
        postprocess: function use to postprocess (default to self.postprocess)
        """

        if cache_dir is None:
            cache_dir = kosh_cache_dir
        self.cache_dir = cache_dir
        self.use_cache = use_cache
        self.cache_file_only = cache_file_only
        self.feature = feature
        self._user_passed_parameters = (None, kargs)
        G = self.get_execution_graph(feature, transformers=transformers)
        return KoshExecutionGraph(G).traverse(format=format, **kargs)

    def extract_(self, format):
        if format is None:
            format = self.types[self._mime_type][0]
        if len(self.types) != 0 and format not in self.types[self._mime_type]:
            raise ValueError("Loader cannot output type {self._mime_type} to {format} format".format(
                self=self, format=format))
        self.format = format
        _, kargs = self._user_passed_parameters
        signature = self.update_signature(self.feature, format, **kargs).hexdigest()
        if self.cache_file_only is True:
            # Ok user just wants to know where cache should be (plus/minus extesions)
            return signature
        # Let's generate the signatures for each step of the path.
        # And try to load it
        cache_success = False
        if self.use_cache:
            try:
                data = self.load(signature)
                cache_success = True
            except Exception:
                pass
            if cache_success:
                return data

        kargs.get("preprocess", self.preprocess)()
        data = self.extract()
        data = kargs.get("postprocess", self.postprocess)(data)
        return data

    def save(self, cache_file, content):
        """Pickles some data to a cache file
        :param cache_file: name of cache file, will be joined with self.cache_dir
        :type cache_file: str
        :param content: content to save to cache
        :type content: object
        """
        with open(os.path.join(self.cache_dir, cache_file), "wb") as f:
            pickle.dump(content, f)

    def load(self, cache_file):
        """Loads content from cache
        :param cache_file: name of cache file, will be joined with self.cache_dir
        :type cache_file: str
        :return: unpickled data
        :rtpye: object
        """
        with open(os.path.join(self.cache_dir, cache_file), "rb") as f:
            data = pickle.load(f)
        return data

    def _list_features(self, *args, **kargs):
        """Wrapper on top of list_features to snatch from cache rather than calling every time"""
        use_cache = kargs.pop("use_cache", True)
        if self.__listed_features is None or not use_cache:
            try:
                self.__listed_features = self.list_features(*args, **kargs)
            except Exception:
                # Broken loader at the moment
                self.__listed_features = []
        out = self.__listed_features
        # Reset
        if not use_cache:
            self.__listed_features = None
        return out

    def list_features(self, *args, **kargs):
        """list_features Given the obj it's loading return a list of features (variables)
        it can extract

        :return: list of available features from this loader
        :rtype: list
        """
        return []

    def describe_feature(self, feature):
        """describe_feature describe the feature as a dictionary

        :param feature: feature to describe
        :type feature: str
        :return: dictionary with attributes describing the feature
        :rtype: dict
        """
        raise NotImplementedError("describe_feature method not implemented")

    def preprocess(self):
        """preprocess sets things up for the extract function

        This should be preceeded by a call to 'get' which stored its args
        in self._user_passed_parameters
        """
        return

    def extract(self):
        """extract this function does the heavy lifting of the extraction
        it needs to be implemented by each loader.

        We recommend returning pointer to the data as much as possible

        :raises NotImplementedError:
        """
        raise NotImplementedError

    def postprocess(self, data):
        """postprocess Given the extracted data apply some post processing to it

        :param data: result of the extract function
        :type data: any
        :return: post processed
        :rtype: any
        """
        return data

__init__(obj, mime_type=None, uri=None, requestorId=None)

KoshLoader generic Kosh loader

Parameters:

Name Type Description Default
obj object

object the loader will try to load from

 required
mime_type str

If you want to force the mime_type to use

 None
uri str

If you want/need to force the uri to use

 None
requestorId str

The id of the dataset requesting data

 None
Source code in kosh/loaders/core.py 
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
def __init__(self, obj, mime_type=None, uri=None, requestorId=None):
    """KoshLoader generic Kosh loader
    :param obj: object the loader will try to load from
    :type obj: object
    :param mime_type: If you want to force the mime_type to use
    :type mime_type: str
    :param uri: If you want/need to force the uri to use
    :type uri: str
    :param requestorId: The id of the dataset requesting data
    :type requestorId: str
    """
    self.requestorId = requestorId
    self.signature = hashlib.sha256(repr(self.__class__).encode())
    self.signature = self.update_signature(obj.id)
    if mime_type is None:
        self._mime_type = obj.mime_type
    else:
        self._mime_type = mime_type
    if uri is None:
        try:
            self.uri = obj.uri
        except AttributeError:  # Not uri on this
            self.uri = None
    else:
        self.uri = uri
    rec = obj.__store__.get_record(obj.id)
    if (rec["type"] not in obj.__store__._kosh_reserved_record_types and mime_type is None)\
            or rec["type"] in ["__kosh_storeinfo__", obj.__store__._ensembles_type]:
        self._mime_type = "dataset"
    if self._mime_type not in self.types:
        open_anything = False
        for t in self.types:
            if t == "dataset":  # special skipping
                continue
            if len(self.types[t]) == 0:
                open_anything = True
        if not open_anything:
            raise RuntimeError(
                "will not be able to load object of type {mime_type}".format(mime_type=self._mime_type))
    self.obj = obj
    self.__listed_features = None

describe_feature(feature)

describe_feature describe the feature as a dictionary

Parameters:

Name Type Description Default
feature str

feature to describe

 required

Returns:

Type Description
dict

dictionary with attributes describing the feature
Source code in kosh/loaders/core.py 
260
261
262
263
264
265
266
267
268
def describe_feature(self, feature):
    """describe_feature describe the feature as a dictionary

    :param feature: feature to describe
    :type feature: str
    :return: dictionary with attributes describing the feature
    :rtype: dict
    """
    raise NotImplementedError("describe_feature method not implemented")

extract()

extract this function does the heavy lifting of the extraction it needs to be implemented by each loader.

We recommend returning pointer to the data as much as possible

Raises:

Type Description
NotImplementedError
Source code in kosh/loaders/core.py 
278
279
280
281
282
283
284
285
286
def extract(self):
    """extract this function does the heavy lifting of the extraction
    it needs to be implemented by each loader.

    We recommend returning pointer to the data as much as possible

    :raises NotImplementedError:
    """
    raise NotImplementedError

get(feature, format=None, transformers=[], use_cache=True, cache_file_only=False, cache_dir=None, **kargs)

Extracts a feature and possibly transforms it

Parameters:

Name Type Description Default
feature str

desired feature

 required
format str

desired output format

 None
transformers kosh.transformer.KoshTranformer

A list of transformers to use after the data is loaded

 []
use_cache bool

Try to use cached data if available

 True
cache_file_only bool

If True, simply return name of cache_file

 False
cache_dir str **kargs will be stored on loader object format and feature are stored on the object for extraction by extraction functions This function calls first the loader's preprocess function This is followed by an actual data extraction via the 'extract' function Finally 'postprocess' is called on the extracted data Reserved keyword: preprocess: function use to preprocess (default to self.preprocess) postprocess: function use to postprocess (default to self.postprocess)

where do we cache the result?

 None
Source code in kosh/loaders/core.py 
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
def get(self, feature, format=None, transformers=[],
        use_cache=True, cache_file_only=False, cache_dir=None,
        **kargs):
    """Extracts a feature and possibly transforms it
    :param feature: desired feature
    :type feature: str
    :param format: desired output format
    :type format: str
    :param transformers: A list of transformers to use after the data is loaded
    :type transformers: kosh.transformer.KoshTranformer
    :param use_cache: Try to use cached data if available
    :type use_cache: bool
    :param cache_file_only: If True, simply return name of cache_file
    :type cache_file_only: bool
    :param cache_dir: where do we cache the result?
    :type cache_dir: str
    **kargs will be stored on loader object
    format and feature are stored on the object for extraction by extraction functions
    This function calls first the loader's preprocess function
    This is followed by an actual data extraction via the 'extract' function
    Finally 'postprocess' is called on the extracted data

    Reserved keyword:
    preprocess: function use to preprocess (default to self.preprocess)
    postprocess: function use to postprocess (default to self.postprocess)
    """

    if cache_dir is None:
        cache_dir = kosh_cache_dir
    self.cache_dir = cache_dir
    self.use_cache = use_cache
    self.cache_file_only = cache_file_only
    self.feature = feature
    self._user_passed_parameters = (None, kargs)
    G = self.get_execution_graph(feature, transformers=transformers)
    return KoshExecutionGraph(G).traverse(format=format, **kargs)

get_execution_graph(feature, transformers=[])

Generates the execution graph to extract a feature and possibly transform it.

Parameters:

Name Type Description Default
feature str

desired feature

 required
format str

desired output format

 required
transformers kosh.transformer.KoshTranformer

A list of transformers to use after the data is loaded

 []

Returns:

Type Description
networkx.OrderDiGraph

execution graph to get to the possibly transformed feature
Source code in kosh/loaders/core.py 
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
def get_execution_graph(self, feature, transformers=[]):
    """Generates the execution graph to extract a feature and possibly transform it.

    :param feature: desired feature
    :type feature: str
    :param format: desired output format
    :type format: str
    :param transformers: A list of transformers to use after the data is loaded
    :type transformers: kosh.transformer.KoshTranformer
    :return: execution graph to get to the possibly transformed feature
    :rtype: networkx.OrderDiGraph
    """
    # Let's get the execution path
    G = get_graph(self._mime_type, self, transformers)
    return G

get_requestor()

Returns the Kosh object requesting the data

Source code in kosh/loaders/core.py 
83
84
85
86
87
88
def get_requestor(self):
    """Returns the Kosh object requesting the data"""
    try:
        return self.obj.__store__.open(self.requestorId)
    except Exception:  # some kosh object do not have an open function
        return self.obj.__store__._load(self.requestorId)

known_load_formats(atype)

Lists all the formats this loader knows how to export to for a given type

Parameters:

Name Type Description Default
atype

type we wish to to the formats for

 required

Returns:

Type Description
list

list of format this type can be exported to by the loader
Source code in kosh/loaders/core.py 
 98
 99
100
101
102
103
104
105
106
def known_load_formats(self, atype):
    """Lists all the formats this loader knows how to export to for a given type

    :param atype: type we wish to to the formats for
    :type format: str
    :return: list of format this type can be exported to by the loader
    :rtype: list
    """
    return self.types.get(atype, [])

known_types()

Lists types of Kosh objects this loader can handle

Returns:

Type Description
list

list of Kosh type this loader can handle
Source code in kosh/loaders/core.py 
90
91
92
93
94
95
96
def known_types(self):
    """Lists types of Kosh objects this loader can handle

    :return: list of Kosh type this loader can handle
    :rtype: list
    """
    return list(self.types.keys())

list_features(*args, **kargs)

list_features Given the obj it's loading return a list of features (variables) it can extract

Returns:

Type Description
list

list of available features from this loader
Source code in kosh/loaders/core.py 
251
252
253
254
255
256
257
258
def list_features(self, *args, **kargs):
    """list_features Given the obj it's loading return a list of features (variables)
    it can extract

    :return: list of available features from this loader
    :rtype: list
    """
    return []

load(cache_file)

Loads content from cache :rtpye: object

Parameters:

Name Type Description Default
cache_file str

name of cache file, will be joined with self.cache_dir

 required

Returns:

Type Description

unpickled data
Source code in kosh/loaders/core.py 
225
226
227
228
229
230
231
232
233
234
def load(self, cache_file):
    """Loads content from cache
    :param cache_file: name of cache file, will be joined with self.cache_dir
    :type cache_file: str
    :return: unpickled data
    :rtpye: object
    """
    with open(os.path.join(self.cache_dir, cache_file), "rb") as f:
        data = pickle.load(f)
    return data

open(mode='r')

Open function

Parameters:

Name Type Description Default
mode str

mode to open the object in

 'r'

Returns:

Type Description
object

opened object
Source code in kosh/loaders/core.py 
108
109
110
111
112
113
114
def open(self, mode="r"):
    """Open function
    :param mode: mode to open the object in
    :type mode: str
    :return: opened object
    :rtype: object"""
    return self

postprocess(data)

postprocess Given the extracted data apply some post processing to it

Parameters:

Name Type Description Default
data any

result of the extract function

 required

Returns:

Type Description
any

post processed
Source code in kosh/loaders/core.py 
288
289
290
291
292
293
294
295
296
def postprocess(self, data):
    """postprocess Given the extracted data apply some post processing to it

    :param data: result of the extract function
    :type data: any
    :return: post processed
    :rtype: any
    """
    return data

preprocess()

preprocess sets things up for the extract function

This should be preceeded by a call to 'get' which stored its args in self._user_passed_parameters

Source code in kosh/loaders/core.py 
270
271
272
273
274
275
276
def preprocess(self):
    """preprocess sets things up for the extract function

    This should be preceeded by a call to 'get' which stored its args
    in self._user_passed_parameters
    """
    return

save(cache_file, content)

Pickles some data to a cache file

Parameters:

Name Type Description Default
cache_file str

name of cache file, will be joined with self.cache_dir

 required
content object

content to save to cache

 required
Source code in kosh/loaders/core.py 
215
216
217
218
219
220
221
222
223
def save(self, cache_file, content):
    """Pickles some data to a cache file
    :param cache_file: name of cache file, will be joined with self.cache_dir
    :type cache_file: str
    :param content: content to save to cache
    :type content: object
    """
    with open(os.path.join(self.cache_dir, cache_file), "wb") as f:
        pickle.dump(content, f)

update_signature(*args, **kargs)

Updated the signature based to a set of args and kargs

Parameters:

Name Type Description Default
*args list

as many arguments as you want

 ()
**kargs dict

key=value style arguments

 {}

Returns:

Type Description
str

updated signature
Source code in kosh/loaders/core.py 
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
def update_signature(self, *args, **kargs):
    """Updated the signature based to a set of args and kargs
    :param *args: as many arguments as you want
    :type *args: list
    :param **kargs: key=value style arguments
    :type **kargs: dict
    :return: updated signature
    :rtype: str
    """
    signature = self.signature.copy()
    for arg in args:
        signature.update(repr(arg).encode())
    for kw in kargs:
        signature.update(repr(kw).encode())
        signature.update(repr(kargs[kw]).encode())
    return signature

KoshSinaLoader

Bases: KoshLoader

Sina base class for loaders

Source code in kosh/loaders/core.py 
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
class KoshSinaLoader(KoshLoader):
    """Sina base class for loaders"""
    types = {"dataset": ["numpy", ]}

    def __init__(self, obj, **kargs):
        """KoshSinaLoader generic sina-based loader
        """
        super(KoshSinaLoader, self).__init__(obj, **kargs)

    def open(self, *args, **kargs):
        """open the object
        """
        record = self.obj.__store__.get_record(self.obj.id)
        if record["type"] not in self.obj.__store__._kosh_reserved_record_types:
            return KoshDataset(
                self.obj.id, store=self.obj.__store__, record=record)
        if record["type"] == self.obj.__store__._sources_type:
            return KoshSinaFile(
                self.obj.id, store=self.obj.__store__, record=record)
        elif record["type"] == self.obj.__store__._ensembles_type:
            return KoshEnsemble(
                self.obj.id, store=self.obj.__store__, record=record)
        else:
            return KoshSinaObject(self.obj.id, self.obj.__store__, record["type"], protected=[
            ], record_handler=self.obj.__store__.__record_handler__, record=record)

    def list_features(self):
        record = self.obj.__store__.get_record(self.obj.id)
        # Using set in case a variable is both in independent and dependent
        # Dependent would win when getting the data
        curves = ()
        for curve in record["curve_sets"]:
            curves += ((curve, None), )
            for curve_type in ["independent", "dependent"]:
                for name in record["curve_sets"][curve][curve_type]:
                    curves += ((curve, name), )
        joined = sorted([x if y is None else x+"/"+y for x, y in curves])
        if joined == sorted(set(joined)):
            return joined
        else:
            return sorted(curves, key=lambda x: (x[0], "" if x[1] is None else x[1]))

    def extract(self, *args, **kargs):
        features = self.feature
        if not isinstance(features, list):
            features = [self.feature, ]
        record = self.obj.__store__.get_record(self.obj.id)
        out = []
        for feature in features:
            if isinstance(feature, six.string_types):
                possibilities = find_curveset_and_curve_name(feature, record)
                if len(possibilities) > 1:
                    raise ValueError("Could not uniquely resolve {} if to could belong to any of: {}".format(
                        feature, possibilities))
                curve_set, curve_name = possibilities[0]
            else:
                curve_set, curve_name = feature
            # Here we are assuming the curve root name cannot have "/" in it
            curve_set = record["curve_sets"][curve_set]
            if curve_name is not None:
                if curve_name in curve_set["independent"]:
                    curve = curve_set["independent"][curve_name]["value"]
                elif curve_name in curve_set["dependent"]:
                    curve = curve_set["dependent"][curve_name]["value"]
                else:
                    raise ValueError("Cannot find curve {} in curve_set {}".format(curve_name, curve_set))
                out.append(numpy.array(curve))
            else:
                # we want all curves
                all = []
                # Matching order (indep/dep) that we used in list_features
                for curve_type in ["independent", "dependent"]:
                    # Same order as list_features()
                    for curve_name in sorted(curve_set[curve_type].keys()):
                        curve = curve_set[curve_type][curve_name]["value"]
                        all.append(numpy.array(curve))
                out.append(all)
        if not isinstance(self.feature, list):
            return out[0]
        else:
            return out

__init__(obj, **kargs)

KoshSinaLoader generic sina-based loader

Source code in kosh/loaders/core.py 
352
353
354
355
def __init__(self, obj, **kargs):
    """KoshSinaLoader generic sina-based loader
    """
    super(KoshSinaLoader, self).__init__(obj, **kargs)

open(*args, **kargs)

open the object

Source code in kosh/loaders/core.py 
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
def open(self, *args, **kargs):
    """open the object
    """
    record = self.obj.__store__.get_record(self.obj.id)
    if record["type"] not in self.obj.__store__._kosh_reserved_record_types:
        return KoshDataset(
            self.obj.id, store=self.obj.__store__, record=record)
    if record["type"] == self.obj.__store__._sources_type:
        return KoshSinaFile(
            self.obj.id, store=self.obj.__store__, record=record)
    elif record["type"] == self.obj.__store__._ensembles_type:
        return KoshEnsemble(
            self.obj.id, store=self.obj.__store__, record=record)
    else:
        return KoshSinaObject(self.obj.id, self.obj.__store__, record["type"], protected=[
        ], record_handler=self.obj.__store__.__record_handler__, record=record)