/usr/lib/python2.7/dist-packages/stetl/main.py is in python-stetl 1.1+ds-2.
This file is owned by root:root, with mode 0o644.
The actual contents of the file can be viewed below.
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 | #!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Main Stetl program.
#
# Author: Just van den Broecke
#
from etl import ETL
from factory import factory
from util import Util
from version import __version__
import argparse # apt-get install python-argparse
import inspect
import os
log = Util.get_log('main')
def parse_args():
log.info("Stetl version = %s" % __version__)
argparser = argparse.ArgumentParser(description='Invoke Stetl')
argparser.add_argument('-c ', '--config', type=str, help='ETL config file in .ini format', dest='config_file',
required=False)
argparser.add_argument('-s ', '--section', type=str, help='Section in the config file to execute, default is [etl]',
dest='config_section', required=False)
argparser.add_argument('-a ', '--args', type=str,
help='Arguments or .properties file to be substituted for {argN}s in config file, as -a "arg1=foo arg2=bar" or -a args.properties',
dest='config_args', required=False)
argparser.add_argument('-d ', '--doc', type=str,
help='Get component documentation like its configuration parameters, e.g. stetl doc stetl.inputs.fileinput.FileInput',
dest='doc_args', required=False)
args = argparser.parse_args()
if args.config_args:
if os.path.isfile(args.config_args):
log.info('Found args file at: %s' % args.config_args)
args.config_args = Util.propsfile_to_dict(args.config_args)
else:
# Convert string to dict: http://stackoverflow.com/a/1248990
args.config_args = Util.string_to_dict(args.config_args)
return args
# DEPRECATED, now using @Config which also documents with Sphinx
def print_config_attrs(clazz):
skip = ['Filter', 'Input', 'Output', 'Component']
for base in clazz.__bases__:
if base.__name__ not in skip:
print_config_attrs(base)
"""Print documentation for Attr object"""
module_name, _, class_name = clazz.__name__.rpartition('.')
# print 'From class: %s' % class_name
attr_count = 0
for member in clazz.__dict__.keys():
if member.startswith('cfg_'):
config_obj = clazz.__dict__[member]
print ('----------------------------------------------')
print ('NAME: %s' % member.replace('cfg_', ''))
print ('MANDATORY: %s' % config_obj.mandatory)
print ('TYPE: %s' % str(config_obj.type))
print ('\n%s' % config_obj.doc)
print ('\nDEFAULT: %s' % str(config_obj.default))
attr_count += 1
if attr_count == 0:
print ('No config attributes or class not yet documented')
def print_classes(package):
# is_module = inspect.ismodule(class_name)
import inputs
import pkgutil
package = inputs
for importer, modname, ispkg in pkgutil.walk_packages(path=package.__path__,
prefix=package.__name__ + '.',
onerror=lambda x: None):
print('stetl.' + modname)
for name, data in inspect.getmembers(modname, inspect.isclass):
if name == '__builtins__':
continue
print name, data
# DEPRECATED, now using @Config which also documents with Sphinx
def print_doc(class_name):
"""Print documentation for class in particular config options"""
# print_classes(class_name)
# return
try:
# class object from module.class name
class_obj = factory.class_forname(class_name)
print ('DOCUMENTATION\n')
print ('CLASS: %s' % class_name)
print (class_obj.__doc__)
# print ('\nConfiguration attributes: \n')
# print_config_attrs(class_obj)
except Exception, e:
log.error("cannot print info class named '%s' e=%s - you made a typo?" % (class_name, str(e)))
raise e
def main():
"""The `main` function, to be called from commandline, like `python src/main.py -c etl.cfg`.
Args:
-c --config <config_file> the Stetl config file.
-s --section <section_name> the section in the Stetl config (ini) file to execute (default is [etl]).
-a --args <arglist> substitutable args for symbolic, {arg}, values in Stetl config file, in format "arg1=foo arg2=bar" etc.
-d --doc <class> Get component documentation like its configuration parameters, e.g. stetl --doc stetl.inputs.fileinput.FileInput
-h --help get help info
"""
args = parse_args()
if args.config_file:
# Do the ETL
etl = ETL(vars(args), args.config_args)
etl.run()
elif args.doc_args:
print_doc(args.doc_args)
else:
print('Unknown option, try stetl -h for help')
if __name__ == "__main__":
main()
|