This file is indexed.

/usr/lib/python2.7/dist-packages/stetl/filters/templatingfilter.py is in python-stetl 1.1+ds-2.

This file is owned by root:root, with mode 0o644.

The actual contents of the file can be viewed below.

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Transformation of any input using Python Templating as
# meant in: https://wiki.python.org/moin/Templating.
# A TemplatingFilter typically is configured with a template file.
# The input is typically the Template context, the variables to be substituted.
# The output is a string passed to the next Filter or Output.
#
# Author:Just van den Broecke

from stetl.util import Util, ogr, osr
from stetl.component import Config
from stetl.filter import Filter
from stetl.packet import FORMAT
from string import Template
import os

log = Util.get_log("templatingfilter")


class TemplatingFilter(Filter):
    """
    Abstract base class for specific template-based filters.
    See https://wiki.python.org/moin/Templating
    Subclasses implement a specific template language like Python string.Template, Mako, Genshi, Jinja2,

    consumes=FORMAT.any, produces=FORMAT.string
    """

    # Start attribute config meta
    # Applying Decorator pattern with the Config class to provide
    # read-only config values from the configured properties.

    @Config(ptype=str, default=None, required=False)
    def template_file(self):
        """
        Path to template file. One of template_file or template_string needs to be configured.
        """
        pass

    @Config(ptype=str, default=None, required=False)
    def template_string(self):
        """
        Template string. One of template_file or template_string needs to be configured.
        """
        pass

    # End attribute config meta

    # Constructor
    def __init__(self, configdict, section, consumes=FORMAT.any, produces=FORMAT.string):
        Filter.__init__(self, configdict, section, consumes, produces)

    def init(self):
        log.info('Init: templating')
        if self.template_file is None and self.template_string is None:
            # If no file present nor template_string error:
            err_s = 'One of template_file or template_string needs to be configured'
            log.error(err_s)
            raise ValueError('One of template_file or template_string needs to be configured')

        self.create_template()

    def exit(self):
        log.info('Exit: templating')
        self.destroy_template()

    def create_template(self):
        '''
        To be overridden in subclasses.
        '''
        pass

    def destroy_template(self):
        pass

    def invoke(self, packet):
        if packet.data is None:
            return packet
        return self.render_template(packet)

    def render_template(self, packet):
        pass


class StringTemplatingFilter(TemplatingFilter):
    """
    Implements Templating using Python's internal string.Template.
    A template string or file should be configured. The input record
    contains the actual values to be substituted in the template string as a record (key/value pairs).
    Output is a regular string.

    consumes=FORMAT.record or FORMAT.record_array, produces=FORMAT.string
    """

    def __init__(self, configdict, section):
        TemplatingFilter.__init__(self, configdict, section, consumes=[FORMAT.record, FORMAT.record_array])

    def create_template(self):
        # Init once
        if self.template_file is not None:
            # Get template string from file content, otherwise assume template_string is populated
            log.info('Init: reading template file %s ..." % self.template_file')
            template_fp = open(self.template_file, 'r')
            self.template_string = template_fp.read()
            template_fp.close()
            log.info("template file read OK: %s" % self.template_file)

        # Create a standard Python string.Template
        self.template = Template(self.template_string)

    def render_template(self, packet):
        if type(packet.data) is list:
            packet.data = [self.template.substitute(item) for item in packet.data]
        else:
            packet.data = self.template.substitute(packet.data)

        return packet


class Jinja2TemplatingFilter(TemplatingFilter):
    """
    Implements Templating using Jinja2. Jinja2 http://jinja.pocoo.org,
    is a modern and designer-friendly templating language for Python
    modelled after Django’s templates. A 'struct' format as input provides
    a tree-like structure that could originate from a JSON file or REST service.
    This input struct provides all the variables to be inserted into the template.
    The template itself can be configured in this component as a Jinja2 string or -file.
    An optional 'template_search_paths' provides a list of directories from which templates
    can be fethced. Default is the current working directory. Via the optional 'globals_path'
    a JSON structure can be inserted into the Template environment. The variables in this
    globals struture are typically "boilerplate" constants like: id-prefixes, point of contacts etc.

    consumes=FORMAT.struct, produces=FORMAT.string
    """

    # Start attribute config meta
    # Applying Decorator pattern with the Config class to provide
    # read-only config values from the configured properties.

    @Config(ptype=str, default=None, required=False)
    def template_search_paths(self):
        """
        List of directories where to search for templates, default is current working directory only.
        """
        pass

    @Config(ptype=str, default=None, required=False)
    def template_globals_path(self):
        """
        One or more JSON files or URLs with global variables that can be used anywhere in template.
        Multiple files will be merged into one globals dictionary
        """
        pass

    # End attribute config meta

    json_package = None
    ogr_package = ogr
    osr_package = osr

    def __init__(self, configdict, section):
        TemplatingFilter.__init__(self, configdict, section, consumes=[FORMAT.struct, FORMAT.geojson_collection])

    def create_template(self):
        try:
            from jinja2 import Environment, FileSystemLoader
        except Exception as e:
            log.error(
                'Cannot import modules from Jinja2, err= %s; You probably need to install Jinja2 first, see http://jinja.pocoo.org',
                str(e))
            raise e

        import json

        Jinja2TemplatingFilter.json_package = json

        # Check for a file with global variables configured in json format
        # TODO get globals in other formats like XML and possibly from a web service
        template_globals = None
        if self.template_globals_path is not None:
            globals_path_list = self.template_globals_path.strip().split(',')

            for file_path in globals_path_list:

                try:
                    log.info('Read JSON file with globals from: %s', file_path)
                    # Globals can come from local file or remote URL
                    if file_path.startswith('http'):
                        import urllib2

                        fp = urllib2.urlopen(file_path)
                        globals_struct = json.loads(fp.read())
                    else:
                        with open(file_path) as data_file:
                            globals_struct = json.load(data_file)

                    # First file: starts a globals dict, additional globals are merged into that dict
                    if template_globals is None:
                        template_globals = globals_struct
                    else:
                        template_globals.update(globals_struct)

                except Exception, e:
                    log.error('Cannot read JSON file, err= %s', str(e))
                    raise e

        # Load and Init Template once
        loader = FileSystemLoader(self.template_search_paths or [os.getcwd()])
        self.jinja2_env = Environment(loader=loader, extensions=['jinja2.ext.do'], lstrip_blocks=True, trim_blocks=True)

        # Register additional Filters on the template environment by updating the filters dict:
        self.add_env_filters(self.jinja2_env)

        if self.template_file is not None:
            # Get template string from file content and pass optional globals into context
            self.template = self.jinja2_env.get_template(self.template_file, globals=template_globals)
            log.info("template file read and template created OK: %s" % self.template_file)
        elif self.template_string is None:
            # If no file present, template_string should have been configured
            self.template = self.jinja2_env.from_string(self.template_string, globals=template_globals)

    def render_template(self, packet):
        packet.data = self.template.render(packet.data)
        return packet

    def add_env_filters(self, jinja2_env):
        '''Register additional Filters on the template environment by updating the filters dict:
        Somehow min and max of list are not present so add them as well.
        '''
        jinja2_env.filters['maximum'] = max
        jinja2_env.filters['minimum'] = min
        jinja2_env.filters['geojson2gml'] = Jinja2TemplatingFilter.geojson2gml_filter

    @staticmethod
    def import_ogr():
        if Jinja2TemplatingFilter.ogr_package is None:
            log.error(
                'Cannot import Python ogr package; You probably need to install GDAL/OGR Python bindings, see https://pypi.python.org/pypi/GDAL')
            raise ImportError

    @staticmethod
    def create_spatial_ref(crs):
        # "crs": {
        #    "type": "EPSG",
        #    "properties": {
        #        "code": "4326"
        #    }
        spatial_ref = Jinja2TemplatingFilter.osr_package.SpatialReference()
        if type(crs) is dict and crs['type'] == 'EPSG':
            # from the GeoJSON source data, though in future deprecated
            spatial_ref.ImportFromEPSG(int(crs['properties']['code']))
        elif type(crs) is str:
            # Like "EPSG:4326"
            spatial_ref.SetFromUserInput(crs)
        elif type(crs) is int:
            # Like 4326
            spatial_ref.ImportFromEPSG(crs)
        return spatial_ref

    @staticmethod
    def geojson2gml_filter(value, source_crs=4326, target_crs=None, gml_id=None, gml_format='GML2', gml_longsrs='NO'):
        """
        Jinja2 custom Filter: generates any GML geometry from a GeoJSON geometry.
        By specifying a target_crs we can even reproject from the source CRS.
        The gml_format=GML2|GML3 determines the general GML form: e.g. pos/posList
        or coordinates. gml_longsrs=YES|NO determines the srsName format like EPSG:4326 or
        urn:ogc:def:crs:EPSG::4326 (long).
        """
        # Import Python OGR lib once
        Jinja2TemplatingFilter.import_ogr()

        try:
            # Create an OGR geometry from a GeoJSON string
            geojson_str = Jinja2TemplatingFilter.json_package.dumps(value)
            geom = Jinja2TemplatingFilter.ogr_package.CreateGeometryFromJson(geojson_str)

            # Create and assign CRS from source CRS
            source_spatial_ref = Jinja2TemplatingFilter.create_spatial_ref(source_crs)
            geom.AssignSpatialReference(source_spatial_ref)

            # Optional: reproject Geometry from source CRS to target CRS
            if target_crs is not None:
                target_spatial_ref = Jinja2TemplatingFilter.create_spatial_ref(target_crs)
                transform = Jinja2TemplatingFilter.osr_package.CoordinateTransformation(source_spatial_ref,
                                                                                        target_spatial_ref)
                geom.Transform(transform)

            # Generate the GML Geometry elements as string, GMLID is optional
            options = ['FORMAT=%s' % gml_format, 'GML3_LONGSRS=%s' % gml_longsrs]
            if gml_id is not None:
                options.append('GMLID=%s' % gml_id)

            gml_str = geom.ExportToGML(options=options)
        except Exception, e:
            gml_str = 'Failure in CreateGeometryFromJson or ExportToGML, err= %s; check your data and Stetl log' % str(
                e)
            log.error(gml_str)

        return gml_str