#!/opt/dosis2/bin/python3-venv

import argparse
import grp
import logging.config
import os
import pwd
import re
import shutil
import subprocess
import sys
import tarfile
import xml.etree.ElementTree as ET
from copy import deepcopy
from datetime import datetime
from multiprocessing import Lock
from multiprocessing.pool import ThreadPool, Pool
from tempfile import mkdtemp

import paramiko
import psycopg2
import scp
import xmltodict
from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT, ISOLATION_LEVEL_READ_COMMITTED
from psycopg2.extras import DictConnection

DEBUG = False
SHOW_DEBUG_INFO = True


logging.config.dictConfig({
    'disable_existing_loggers': False,
    'version': 1,
    'formatters': {
        'verbose': {
            'format': '%(levelname)s {ident}.%(name)s [%(filename)s:%(lineno)s] %(message)s'.format(
                ident='dosis-backup'
            )
        },
        'simple': {
            'format': '%(levelname)s [%(filename)s:%(lineno)s] %(message)s'
        }
    },
    'handlers': {
        'info': {
            'level': 'INFO',
            'class': 'logging.handlers.SysLogHandler',
            'address': '/dev/log',
            'formatter': 'verbose'
        },
        'debug': {
            'level': 'DEBUG',
            'class': 'logging.StreamHandler',
            'stream': sys.stdout,
            'formatter': 'simple'
        }
    },
    'loggers': {
        'logger': {
            'handlers': ['info', 'debug'],
            'level': 'DEBUG'
        }
    }
})


logger = logging.getLogger('logger')


class Parser(object):
    """
    Base parser interface with reload and get methods
    """
    def reload(self):
        """
        Should reload configuration

        :return: should return the configuration as a dictionary
        """
        raise NotImplementedError('reload method not implemented')

    def get(self, *args, **kwargs):
        """
        Should return information from the configuration dictionary

        :param args: whatever needs to be passed
        :param kwargs: whatever needs to be passed
        :return: information from the dictionary
        """
        raise NotImplementedError('get method not implemented')


class DosisConfigParser(Parser):
    """
    Parse dosisConfig.xml into a searchable dictionary utilizing xmltodict

    :param base: base filepath (path to base dosisConfig.xml)
    :param custom: custom filepath (path to custom dosisConfig.xml)
    """
    def __init__(self, base, custom):
        self.base = base
        self.custom = custom
        self.data = self.reload()

    def merge(self, a, b):
        """
        Recursively merge dictionaries

        :param a: base dictionary
        :param b: custom dictionary
        :return: a dictionary
        """
        c = deepcopy(a)

        def _merge(_a, _b):
            for key in b:
                if key in a:
                    if isinstance(a[key], dict) and isinstance(b[key], dict):
                        self.merge(a[key], b[key])
                    elif a[key] == b[key]:
                        pass
                    else:
                        a[key] = b[key]
                else:
                    a[key] = b[key]

        _merge(c, b)
        return c

    def reload(self):
        """
        Reload the configuration into a dictionary

        :return: a new instance of a dictionary representing the configuration
        """
        with open(self.base, 'r') as b:
            base_data = xmltodict.parse(b.read())

        with open(self.custom, 'r') as c:
            custom_data = xmltodict.parse(c.read())

        return self.merge(base_data, custom_data)

    def get(self, sect, prop, default=None):
        """
        Find a section and property and return the value, if not found, it will return default

        :param sect: section to search for
        :param prop: property to search for
        :param default: default value if not found
        :return: the tags within that section.property
        """
        sects = [section for section in self.data['config']['section'] if section['@name'] == sect]
        if sects:
            proprs = [propr for propr in sects[0]['property'] if propr['@name'] == prop]
            if proprs:
                return proprs[0]
        return default


class SSHCredentialParser(Parser):
    """
    Parse an .sshcredentials file

    :param filepath: path to the credentials file
    """
    def __init__(self, filepath):
        self.credfile = filepath
        self.data = self.reload()

    def reload(self):
        """
        Reload the configuration

        :return: dictionary containing key value pairs in the file
        """
        with open(self.credfile, 'r') as c:
            data = c.read().strip()

        data_pattern = re.compile(r'^\s*([\w\d]+)\s*=\s*(.*)\s*$')
        pairs = {}
        for line in re.split(r'[\r\n]+', data):
            matches = data_pattern.findall(line)
            if matches:
                key, value = matches[0]
                pairs[key] = value

        return pairs

    def get(self, key, default=None):
        """
        Get a single value from the parsed file

        :param key: key to get
        :param default: default value if key doesn't exist
        :return: value associated with key
        """
        return self.data.get(key, default)

    username = property(lambda self: self.get('username'))
    password = property(lambda self: self.get('password'))


class DatabaseCredentialParser(Parser):
    """
    Parse dosisConfig.xml using a DosisConfigParser and load database credentials only

    :param base: base filepath (path to base dosisConfig.xml)
    :param custom: custom filepath (path to custom dosisConfig.xml)

    """
    def __init__(self, base, custom):
        self.parser = DosisConfigParser(base, custom)
        self.credentials = self.reload()

    def reload(self):
        """
        Reload the configuration into a dictionary

        :return: a new instance of a dictionary representing the configuration
        """
        return {
            'host': self.parser.get('general', 'DBServer', {}).get('value'),
            'database': self.parser.get('general', 'DBName', {}).get('value'),
            'username': self.parser.get('general', 'DBUsername', {}).get('value'),
            'password': self.parser.get('general', 'DBPassword', {}).get('value'),
            'port': self.parser.get('general', 'DBPort', {}).get('value')
        }

    def get(self, key, default=None):
        """
        Get an element from the dictionary

        :param key: key to retrieve
        :param default: default value if one doesn't exist
        :return: value of the key
        """
        return self.credentials.get(key, default)

    host = property(lambda self: self.get('host') if self.get('host') != '127.0.0.1' else None)
    database = property(lambda self: self.get('database'))
    username = property(lambda self: self.get('username'))
    password = property(lambda self: self.get('password'))
    port = property(lambda self: self.get('port'))


class ArgumentParser(Parser):
    """
    Parse arguments from the command line with a specific set of available commands

    :param description: description of the utility
    :param epilog: epilog for the utility
    """
    def __init__(self, description, epilog):
        self.parser = argparse.ArgumentParser(description=description, epilog=epilog)
        self.args = self.reload()

    def reload(self):
        """
        Parse arguments into a usable namespace

        :return: namespace of arguments
        """
        self.parser.add_argument(
            '--no-vacuum', '-D',
            dest='no-vacuum',
            action='store_true',
            help='do not vacuum the database after backup'
        )
        self.parser.add_argument(
            '--no-archive', '-a',
            dest='no-archive',
            action='store_true',
            help='do not archive the database after backup'
        )
        self.parser.add_argument(
            '--no-labels', '-l',
            dest='no-labels',
            action='store_true',
            help='do not backup labels from the database'
        )
        self.parser.add_argument(
            '--rpm', '-r',
            dest='rpm',
            action='store_true',
            help='run with rpm configuration'
        )
        self.parser.add_argument(
            '--test-mode', '-t',
            dest='test-mode',
            action='store_true',
            help='test mode. Do not backup towers or upload'
        )
        return self.parser.parse_args()

    def get(self, key, default=None):
        """
        Get an attribute from the namespace

        :param key: key to get
        :param default: default value if not exist
        :return: value
        """
        return getattr(self.args, key, default)

    no_vacuum = property(lambda self: self.get('no-vacuum'))
    no_archive = property(lambda self: self.get('no-archive'))
    no_labels = property(lambda self: self.get('no-labels'))
    rpm = property(lambda self: self.get('rpm'))
    test_mode = property(lambda self: self.get('test-mode'))


class DatabaseConnectionError(Exception):
    """
    Thrown if there is an error in connecting to the database
    """
    pass


class DatabaseConnection(object):
    """
    Database connection class to make interfacing with database easy

    :param db_cred_parser: parser for database credentials
    :param database: override the database from the configuration
    :param username: override the username from the configuration
    :param password: override the password from the configuration
    :param port: override the port from the configuration
    :param host: override the host from the configuration
    """
    def __init__(self, db_cred_parser=None, **kwargs):
        self.db_cred_parser = db_cred_parser
        self.database = kwargs.get('database', db_cred_parser.database)
        self.username = kwargs.get('username', db_cred_parser.username)
        self.password = kwargs.get('password', db_cred_parser.password)
        self.port = kwargs.get('port', db_cred_parser.port)
        self.host = kwargs.get('host', db_cred_parser.host)
        self.connection = self.reconnect()
        self.conn_lock = Lock()

    def reconnect(self):
        """
        Reconnect to the database

        :return: database connection
        """

        tmphost = self.host
        if not tmphost:
            tmphost = 'localhost'
            
        return psycopg2.connect(
            database=self.database,
            user=self.username,
            password=self.password,
            host=tmphost,
            port=self.port,
            connection_factory=DictConnection
        )

    @property
    def connected(self):
        """
        Test whether or not the connection is alive

        :return: true or false depending on the connection state
        """
        try:
            cursor = self.connection.cursor()
            cursor.execute('SELECT 1 AS test')
            value = cursor.fetchone()
            if value['test'] != 1:
                return False
            return True
        except Exception as e:
            logger.exception(e)
            return False

    def copy(self):
        return self.__class__(
            db_cred_parser=self.db_cred_parser,
            database=self.database,
            username=self.username,
            password=self.password,
            port=self.port,
            host=self.host,
        )

    def close(self):
        self.connection.close()

    def __enter__(self):
        """
        get a cursor for python's with statement

        :return: cursor
        """
        if self.connected:
            try:
                return self.connection.cursor()
            finally:
                self.conn_lock.acquire()
        raise DatabaseConnectionError('unable to create cursor, not connected')

    def __exit__(self, exc_type, exc_val, exc_tb):
        """
        When with exits, this may contain exceptions and will rollback if any exception is caught

        :param exc_type: exception type
        :param exc_val: exception value
        :param exc_tb: exception traceback
        :return: may throw an exception
        """
        try:
            if exc_type is not None:
                logger.exception(exc_tb)
                logger.exception(exc_val)
                try:
                    self.connection.rollback()
                except Exception as e:
                    logger.exception(e)
                raise DatabaseConnectionError('database unable to process transaction')
            self.connection.commit()
            return True
        finally:
            self.conn_lock.release()


class DatabaseConnectionMethods(DatabaseConnection):
    """
    This class contains methods for interacting with the database that will be abstracted
    from its user so the user does not have to write sql

    :param args: arguments (same as DatabaseConnection)
    :param kwargs: keyword arguemnts (same as DatabaseConnection)
    """
    def __init__(self, *args, **kwargs):
        super(DatabaseConnectionMethods, self).__init__(*args, **kwargs)

    def query(self, query, params=None, isolation_level=ISOLATION_LEVEL_READ_COMMITTED, get_results=True):
        """
        Run a single query and return it's results

        :param query: query to run
        :param params: params for the query
        :param isolation_level: isolation level for this query
        :return: results
        """
        self.connection.set_isolation_level(isolation_level)
        with self as cursor:
            try:
                cursor.execute(query, {} if params is None else params)
            except psycopg2.Error as e:
                logger.exception(e)

            if get_results:
                try:
                    data = cursor.fetchall()
                    return data
                except psycopg2.Error as e:
                    if query:
                        logger.exception('query that failed: ' + str(query))
                    else:
                        logger.exception(e)

            return None

    def get_system_tag(self):
        """
        get system tag information from the database
        :return: system tag as a string
        """
        data = self.query(
            'select tag from equipment.dafe '
            'where dafe_type_id = (select id from equipment.dafe_types where tag = %(tag)s) '
            'and active',
            {'tag': 'SYSTEM'}
        )
        if not data:
            return 'SYSTEM'
        return data[0]['tag']

    def set_backup_time(self, datetime_obj=datetime.now()):
        """
        Reset the last backup time

        :param datetime_obj: datetime object to set the time with
        :return: nothing
        """
        self.query(
            'update equipment.dafe_status set status = %(dts)s '
            'where dafe_status_type_id = (select id from equipment.dafe_status_types where tag = %(tag)s)',
            {'dts': datetime_obj, 'tag': 'BACKUP_COMPLETE'}, get_results=False
        )

    def get_backup_values(self):
        """
        Get a tuple of should backup values

        :return: tuple of true false representing (usb, network, dafe)
        """
        query = \
            'select value ' \
            'from equipment.dafe_settings ' \
            'where setting_type_id = (select id from equipment.dafe_setting_types where tag = %(tag)s)'

        usb = self.query(query, {'tag': 'DO_BACKUP_USB'})
        net = self.query(query, {'tag': 'DO_BACKUP_NETWORK'})
        dafe = self.query(query, {'tag': 'DO_BACKUP_FROM_DAFE'})

        usb_val = False if not usb else (usb[0]['value'].lower() in ('true', 't', '1'))
        net_val = False if not net else (net[0]['value'].lower() in ('true', 't', '1'))
        dafe_val = False if not dafe else (dafe[0]['value'].lower() in ('true', 't', '1'))

        return usb_val, net_val, dafe_val

    def get_backup_dafes(self):
        """
        get the other dafes needed to be backed up from the database

        :return: list of rows with keys 'tag' and 'site_ip'
        """
        return self.query(
            'select d.tag, d.site_ip from equipment.dafe as d '
            'inner join equipment.dafe_types dt on d.dafe_type_id = dt.id '
            'inner join ( '
            '   select _d.tag, _d.site_ip from equipment.dafe _d '
            '   inner join equipment.dafe_types _dt on _d.dafe_type_id = _dt.id '
            '   where _dt.type_class = %(ssclass)s '
            ') sys on d.site_ip != sys.site_ip '
            'where active '
            'and d.site_ip != %(local_ip)s '
            'and dt.type_class = %(type_class)s ',
            {'ssclass': 'SITE_SERVER', 'local_ip': '127.0.0.1', 'type_class': 'L60'}
        )

    def get_cooler_summary_max_recs(self):
        return self.query(
            '''
            SELECT COALESCE(
                (SELECT value FROM site.site_settings WHERE setting_type_id = (
                    SELECT id FROM site.site_setting_types WHERE tag = 'COOLER_SUMMARY_MAX_RECS')),
                 '2'
             )::integer
            '''
        )[0][0]

    def get_cooler_num_chunks(self):
        return self.query(
            '''
            SELECT COALESCE(
                (SELECT value FROM site.site_settings WHERE setting_type_id = (
                    SELECT id FROM site.site_setting_types WHERE tag = 'COOLER_NUM_CHUNKS')),
                 '2'
             )::integer
            '''
        )[0][0]

    def get_cooler_daily_time_limit_mins(self):
        return self.query(
            '''
            SELECT COALESCE(
                (SELECT value FROM site.site_settings WHERE setting_type_id = (
                    SELECT id FROM site.site_setting_types WHERE tag = 'COOLER_DAILY_TIME_LIMIT_MINS')),
                 '120'
             )::integer
            '''
        )[0][0]

    def get_purge_archive_daily_time_limit_mins(self):
        return self.query(
            '''
            SELECT COALESCE(
                (SELECT value FROM site.site_settings WHERE setting_type_id = (
                    SELECT id FROM site.site_setting_types WHERE tag = 'ARCH_PURGE_DAILY_TIME_LIMIT_MINS')),
                 '120'
             )::integer
            '''
        )[0][0]

    def get_archive_daily_time_limit_mins(self):
        return self.query(
            '''
            SELECT COALESCE(
                (SELECT value FROM site.site_settings WHERE setting_type_id = (
                    SELECT id FROM site.site_setting_types WHERE tag = 'ARCH_DAILY_TIME_LIMIT_MINS')),
                 '120'
             )::integer
            '''
        )[0][0]

    def export_label_data(self):
        """
        Export all labels from the database

        :return: all labels in a dictionary of tag: data pairs
        """
        names = self.query('select tag from printers.label_defn_name')
        label_data = {}
        for name in names:
            tag = name['tag']
            data = self.query('select * from printers.label_defn_export(%(tag)s)', {'tag': tag})
            if data:
                label_data[tag] = '\n'.join([r.get('label_defn_export') for r in data])

        return label_data

    def export_dafes_specifics(self):
        """
        Export dafe settings from the database

        :return: dafe specific settings as a list of queries to run
        """
        names = self.query('select tag from equipment.dafe where tag not in (%(systag)s, %(unktag)s)', {
            'systag': 'SYSTEM', 'unktag': 'UNKNOWN'
        })
        settings = {}
        for name in names:
            tag = name['tag']
            data = self.query('select * from equipment.export_dafe_specifics(%(dafe)s)', {'dafe': tag})
            if data:
                settings[tag] = '\n'.join([r.get('export_dafe_specifics') for r in data])

        return settings

    def archive(self):
        """
        Run an archive on the database

        :return: success or failure in case of a database error
        """
        return_value = None
        try:
            _ = self.query('select equipment.update_dafe_status('
                               '(select id from equipment.dafe where tag = \'SYSTEM\'),'
                               '\'ARCHIVE_SUCCESSFUL\', '
                               '\'FALSE\')')
            results = self.query('select * from orders.arch_process()')
            if results:
                return_value = results[0]['arch_process'].upper()
                logger.info('ran arch_process, returned %s' % return_value)
                _ = self.query('select equipment.update_dafe_status('
                                   '(select id from equipment.dafe where tag = \'SYSTEM\'),'
                                   '\'ARCHIVE_SUCCESSFUL\', '
                                   '\'TRUE\')')
        except Exception as e:
            logger.exception(e)
            return_value = 'ERROR'

        return return_value

    def summarize(self, num_recs):
        return_value = None

        try:
            results = self.query('SELECT * FROM cooler.convert_cold_storage_to_summary(%(num_recs)s)', {'num_recs': num_recs})
            if results:
                return_value = results[0]['convert_cold_storage_to_summary'].upper()
                logger.info('ran convert_cold_storage_to_summary, returned %s' % return_value)
            return return_value
        except Exception as e:
            logger.exception(e)

    def cooler(self, chunk):
        new_self = None
        try:
            # we run multiple calls to cooler() in threads, so we need a new connection for each call
            new_self = self.copy()
            results = new_self.query('SELECT * FROM cooler.cooler_process(%(chunk)s)', {'chunk': chunk})
            return_value = results[0]['cooler_process'].upper()
            logger.info('ran cooler with chunk %s, returned %s' % (chunk, return_value))
            return return_value
        except Exception as e:
            logger.exception(e)
        finally:
            if new_self is not None:
                new_self.close()

    def cooler_prep(self):
        return_value = None
        try:
            results = self.query('SELECT * FROM cooler.prep_cooler_process()')
            if results:
                return_value = results[0]['prep_cooler_process'].upper()
                logger.info('ran prep_cooler_process, returned %s' % return_value)
            return return_value
        except Exception as e:
            logger.exception(e)

    def purge_archive_no_workorder(self):
        return_value = None
        try:
            results = self.query('SELECT * FROM orders.purge_archive_no_workorder_data()')
            if results:
                return_value = results[0]['purge_archive_no_workorder_data'].upper()
                logger.info('ran purge_archive_no_workorder_data, returned %s' % return_value)
            return return_value
        except Exception as e:
            logger.exception(e)

    def vacuum(self):
        """
        Run a vacuum and analyze on the database
        :return: nothin
        """
        self.query('vacuum analyze', isolation_level=ISOLATION_LEVEL_AUTOCOMMIT, get_results=False)


class FinishOrTimeoutResults(object):
    FINISH = 'FINISH'
    ERROR = 'ERROR'
    TIMEOUT = 'TIMEOUT'
    NO_RUN = 'NO_RUN'


def run_until_finished_or_timeout(func, time_limit_mins):
    """
    Run some incremental function over and over until it finishes
    or we run out of time, whichever comes first.
    :param func: A function that takes no arguments and returns boolean
        indicating whether the incremental process completely finished.
        Once func() returns True, this function will return and func()
        will not be run again.
        The function may also raise a RuntimeError to indicate that
        something went wrong.  In that case this function will return and
        func() will not be run again.
    :param time_limit_mins: Rough time limit for how to long to continue
        running func() over and over before this function returns and func()
        stops getting ran. Most of the time, the time limit will be overshot,
        especially if func() is some long running process.
    :return: One of the values in FinishOrTimeoutResults
    """

    start_time = datetime.now()

    while True:
        try:
            finished = func()
            if finished:
                return FinishOrTimeoutResults.FINISH
        except RuntimeError:
            return FinishOrTimeoutResults.ERROR

        time_passed_mins = (datetime.now() - start_time).total_seconds() / 60.0

        if time_passed_mins >= time_limit_mins:
            return FinishOrTimeoutResults.TIMEOUT


class DosisBackup(object):
    """
    Dosis backup class that provides a script to perform backups

    :param base_dosisconfig: filepath to base dosisConfig.xml
    :param custom_dosisconfig: filepath to custom dosisConfig.xml
    :param sshconfig: filepath to .sshcredentials file
    :param local_backup_root: where to place the backups locally
    :param usb_backup_root: where to place the backups so it goes onto a usb drive
    :param site_backup_root: where to place the backups so it goes onto the site network backup
    :param rpm_backup_root: where to place the backups so when running an RPM backup
    :param file_mode: file mode, must be a number (can enter octal like in chmod such as 0o0744)
    :param file_user: username for chowning files (should be same on local and remote)
    :param file_group: groupname for chowning files (should be same on local and remote)
    :param db_backup_name_format: format for db backup filename {system} will be replaced with
            the system name of the backup and the rest of the parameters are equivalent to datetime's
            strftime function
    :param cfg_backup_name_format: format for cfg backup filename {system} will be replaced with
            the system name of the backup and the rest of the parameters are equivalent to datetime's
            strftime function
    :param db_superuser: super user for the database
    :param cfg_directories: which configuration directories need to be backed up
    """
    def __init__(self, *args, **kwargs):
        self._args = args
        self.base_dosisconfig = kwargs.get('base_dosisconfig')
        self.custom_dosisconfig = kwargs.get('custom_dosisconfig')
        self.sshconfig = kwargs.get('sshconfig')
        self.local_backup_root = kwargs.get('local_backup_root')
        self.usb_backup_root = kwargs.get('usb_backup_root')
        self.site_backup_root = kwargs.get('site_backup_root')
        self.rpm_backup_root = kwargs.get('rpm_backup_root')
        self.file_mode = kwargs.get('file_mode')
        self.file_user = kwargs.get('file_user')
        self.file_group = kwargs.get('file_group')
        self.db_backup_name_format = kwargs.get('db_backup_name_format')
        self.cfg_backup_name_format = kwargs.get('cfg_backup_name_format')
        self.lbl_backup_name_format = kwargs.get('lbl_backup_name_format')
        self.dafe_specifis_backup_name_format = kwargs.get('dafe_specifics_backup_name_format')
        self.db_superuser = kwargs.get('db_superuser')
        self.cfg_directories = kwargs.get('cfg_directories')
        self.cfg_files = kwargs.get('cfg_files')


        logger.info('initializing backup process')
        self.dbcred = DatabaseCredentialParser(self.base_dosisconfig, self.custom_dosisconfig)
        self.sshcred = SSHCredentialParser(self.sshconfig)
        self.methods = DatabaseConnectionMethods(self.dbcred)
        self.su_methods = DatabaseConnectionMethods(self.dbcred, username=self.db_superuser, password=None)
        self.args = ArgumentParser('Backup utility for dosis database and config files', 'And thats how you do it')
        self.fail_count = 0
        self.fail_count_lock = Lock()

    @staticmethod
    def user_ids(username, groupname):
        """
        Find the uid and gid of a username and groupname and return as a tuple

        :param username: username
        :param groupname: groupname
        :return: (uid, gid) tuple
        """
        uid = pwd.getpwnam(username).pw_uid
        gid = grp.getgrnam(groupname).gr_gid
        return uid, gid

    @staticmethod
    def chmod(path, mode, recursive=False):
        """
        Chmod a directory recursively

        :param path: path to recurisve chmod
        :param mode: mode to apply: must be a number (can specify in octal using 0o0744)
        :param recursive: should the directory be chmod'd recursively
        :return: nothing
        """

        logger.info('chmod {recursive} {mode} {dir}'.format(
            mode=mode,
            dir=path,
            recursive='-R' if recursive else ''
        ))
        if not DEBUG:
            os.chmod(path, 0o0774)

        if recursive:
            for root, dirs, files in os.walk(path):
                f = set(dirs).union(set(files))
                for p in f:
                    chmod_dir = os.path.join(root, p)
                    if not DEBUG:
                        os.chmod(chmod_dir, 0o0774)

    @staticmethod
    def chown(path, user, group, recursive=False):
        """
        Recursive chown directory

        :param path: path to recursively chown
        :param user: username
        :param group: groupname
        :param recursive: should the directory be chown'd recursively
        :return: nothing
        """
        uid, gid = DosisBackup.user_ids(user, group)

        logger.info('chown {recursive} {user}.{group} {dir}'.format(
            user=user,
            group=group,
            dir=path,
            recursive='-R' if recursive else ''
        ))

        if not (uid and gid):
            return

        if not DEBUG:
            os.chown(path, uid, gid)

        if recursive:
            for root, dirs, files in os.walk(path):
                f = set(dirs).union(set(files))
                for p in f:
                    chown_dir = os.path.join(root, p)
                    if not DEBUG:
                        os.chown(chown_dir, uid, gid)

    @staticmethod
    def makedirs(directory, mode=None, user=None, group=None):
        """
        Safely call makedirs

        :param directory: directory to make
        :param user: user to own
        :param group: group to own
        :param mode: mode to apply to the directories, uses default if none
        :return: nothing
        """
        try:
            if mode is None:
                os.makedirs(directory)
            else:
                os.makedirs(directory, mode=0o0774)

            if user and group:
                DosisBackup.chown(directory, user, group)
        except Exception:
            pass

    @staticmethod
    def remake_directory(directory, mode=None, user=None, group=None):
        """
        Recreate a given directory, this implies a recursive remove, and make directory

        :param directory: directory to recreate
        :param mode: mode to apply
        :param user: username to apply
        :param group: groupname to apply
        :return: success or failure (bool)
        """
        try:
            logger.info(mode)
            if os.path.exists(directory):
                logger.info('rmtree {dir}'.format(dir=directory))
                if not DEBUG:
                    shutil.rmtree(directory)

            logger.info('makedirs {dir}'.format(dir=directory))
            if not DEBUG:
                DosisBackup.makedirs(directory, mode=mode)

            if user and group:
                DosisBackup.chown(directory, user, group)

            return True
        except OSError as e:
            logger.exception(e)
            return False

    @staticmethod
    def finalize_backup_root(tmp_backup_root, final_backup_root):
        """
        Finalize the backup root, call this when tmp_backup_root is complete and needs to be moved
        to the final directory. This will remove the old directory and rename this one to replace the old one

        :param tmp_backup_root: temporary backup root that contains the final files
        :param final_backup_root: directory to remove and replace with tmp_backup_root
        :return: success or failure (bool)
        """
        try:
            if os.path.exists(final_backup_root):
                logger.info('rmtree {dir}'.format(dir=final_backup_root))
                if not DEBUG:
                    shutil.rmtree(final_backup_root)

            logger.info('moving {tmp} -> {final}'.format(tmp=tmp_backup_root, final=final_backup_root))
            if not DEBUG:
                os.rename(tmp_backup_root, final_backup_root)
            return True
        except Exception as e:
            logger.exception(e)
            return False

    @staticmethod
    def gen_backup_root(root, datetime_obj):
        """
        generate a backup path with the correct format <root>/<day of week>

        :param root: path to backup root
        :param datetime_obj: datetime object to get day of week
        :return: backup path
        """
        backup_root = os.path.join(root, datetime_obj.strftime('%A'))
        return backup_root

    @staticmethod
    def execute_remote_script(client, commands):
        """
        Execute a remote script on a given machine

        :param client: client object SSHClient from paramiko
        :param commands: list of commands to run
        :return: nothing
        """
        for cmd in commands:
            logger.info('REMOTE_COMMAND: {cmd}'.format(cmd=cmd))

        def exec_single_command(client_, command):
            trans = client_.get_transport()
            channel = trans.open_channel(kind='session')
            channel.exec_command(command)
            while not channel.exit_status_ready():
                pass

        if not DEBUG:
            for cmd in commands:
                exec_single_command(client, cmd)

    @staticmethod
    def scp_from_remote(client, file_remote, file_local):
        """
        Secure copy a file from the remote machine to this local machine

        :param client: client object SSHClient from paramiko
        :param file_remote: file from the remote machine
        :param file_local: file location on the local machine
        :return: nothing
        """
        logger.info('remote:{file_remote} -> local:{file_local}'.format(
            file_remote=file_remote,
            file_local=file_local
        ))
        if not DEBUG:
            trans = scp.SCPClient(client.get_transport())
            trans.get(file_remote, local_path=file_local)

    def run_backup(self):
        """
        Run the backup itself
        :return: success or failure
        """
        logger.info('starting backup process')
        start_time = datetime.now()
        self.fail_count = 0

        if self.dbcred.host:
            logger.info('not running backup on team robot')
            return True

        self.makedirs(self.local_backup_root)
        self.chmod(self.local_backup_root, self.file_mode, recursive=True)
        self.chown(self.local_backup_root, self.file_user, self.file_group, recursive=True)

        system_tag = self.methods.get_system_tag()

        if self.args.rpm:
            backup_root = self.rpm_backup_root
            logger.info('setting backup_root to {path}'.format(path=backup_root))
        else:
            backup_root = self.gen_backup_root(self.local_backup_root, start_time)
            logger.info('setting backup_root to {path}'.format(path=backup_root))

        # start backup
        final_backup_root = backup_root
        backup_root += '.tmp'

        local_backup_path = os.path.join(backup_root, system_tag)

        logger.info('recreating directory for {path}'.format(path=backup_root))
        self.remake_directory(
            backup_root,
            mode=self.file_mode,
            user=self.file_user,
            group=self.file_group
        )
        self.makedirs(local_backup_path, mode=self.file_mode, user=self.file_user, group=self.file_group)

        logger.info('start spawning backup threads')

        thread_pool = ThreadPool(processes=3)
        thread_pool.apply_async(
            self.dump_database,
            callback=self.update_failed_count,
            kwds={'directory': local_backup_path, 'system_tag': system_tag, 'datetime_obj': start_time}
        )
        thread_pool.apply_async(
            self.dump_config,
            callback=self.update_failed_count,
            kwds={'directory': local_backup_path, 'system_tag': system_tag, 'datetime_obj': start_time}
        )
        thread_pool.apply_async(
            self.dump_labels,
            callback=self.update_failed_count,
            kwds={'directory': local_backup_path, 'system_tag': system_tag, 'datetime_obj': start_time}
        )
        thread_pool.apply_async(
            self.dump_dafe_specifics,
            callback=self.update_failed_count,
            kwds={'directory': local_backup_path, 'system_tag': system_tag, 'datetime_obj': start_time}
        )

        if self.args.rpm:
            # If we are running with the rpm method, then just close the thread pool and wait for it to complete
            logger.info('running with rpm method, waiting for all threads to complete')
            usb, net, dafe = False, False, False
            thread_pool.close()
            thread_pool.join()
        else:
            # If we are not running without the rpm method, we want to check the database for values associated
            # with backup using site backup, usb backup, and dafe backup
            logger.info('running without rpm method, checking usb, site, and dafe backup flags')

            usb, net, dafe = self.methods.get_backup_values()
            if dafe:
                if not self.args.test_mode:
                    # If we're running a dafe backup, then spawn a thread to dump all dafes
                    logger.info('spawning thread for dafe dump')
                    thread_pool.apply_async(
                        self.dump_dafes,
                        callback=self.update_failed_count,
                        kwds={'directory': backup_root, 'datetime_obj': start_time}
                    )
            thread_pool.close()
            thread_pool.join()

        # finalize backup directory
        if self.fail_count <= 0:
            self.finalize_backup_root(backup_root, final_backup_root)

        backup_root = final_backup_root

        if not self.args.rpm:
            if not self.args.test_mode:
                if usb:
                    self.copy_to_mountpoint(backup_root, self.usb_backup_root)

                if net:
                    self.copy_to_mountpoint(backup_root, self.site_backup_root)

            cooler_prep_result = self.cooler_prep_database()
            if cooler_prep_result in (FinishOrTimeoutResults.NO_RUN, FinishOrTimeoutResults.FINISH):
                cooler_result = self.cooler_database()
                if cooler_result in (FinishOrTimeoutResults.NO_RUN, FinishOrTimeoutResults.FINISH):
                    summary_result = self.cooler_summarize()
                    if summary_result in (FinishOrTimeoutResults.NO_RUN, FinishOrTimeoutResults.FINISH):
                        purge_result = self.purge_archive_no_workorder_database()
                        if purge_result in (FinishOrTimeoutResults.NO_RUN, FinishOrTimeoutResults.FINISH):
                            arch_result = self.archive_database()

            self.vacuum_database()

            # we only want to set the backup time if we are not running with the rpm method
            self.methods.set_backup_time()

        end_time = datetime.now()
        delta = end_time - start_time
        logger.info('backup completed in {t} seconds with {errors} errors'.format(
            errors=self.fail_count,
            t=delta.total_seconds()
        ))
        if self.fail_count == 0 and not self.args.rpm:
            if not self.args.test_mode:
                logger.info('starging transfer to aws')
                res = subprocess.check_call(['/usr/share/dosis2/bin/backup-to-aws-wrapper'])
                logger.info('backup tranfer to aws result: %s' % str(res))

        return self.fail_count == 0

    def update_failed_count(self, success):
        """
        Callback function for failures in a function, if a function returns a bad exit status
        then the fail count is incremented

        :param success: whether or not the function ran was successful
        :return: nothing
        """
        if not success:
            logger.info('a section of the backup failed')
            with self.fail_count_lock:
                self.fail_count += 1

    def dump_database(self, directory, system_tag, datetime_obj=datetime.now()):
        """
        Actually perform a database dump

        :param directory: directory to store the dumped file
        :param system_tag: system tag to format the filename with
        :param datetime_obj: datetime object to know how to format the string for the filename
        :return: success or failure
        """
        logger.info('dumping database')
        filename = datetime_obj.strftime(self.db_backup_name_format.format(system=system_tag))
        abs_filepath = os.path.join(directory, filename + '.cus')

        try:
            logger.info('starting pg_dump process for database backup {filename}'.format(filename=abs_filepath))

            if not DEBUG:
                pg_dump_command = ['/usr/bin/pg_dump', '-U', self.db_superuser, '-F', 'c']
                pg_dump_command += ['-h', self.dbcred.host] if self.dbcred.host else []
                pg_dump_command += ['-p', self.dbcred.port, '-f', abs_filepath, self.dbcred.database]
                pg_dump_p = subprocess.Popen(pg_dump_command)
                logger.info('pg_dump processes spawned, waiting for completion')
                output, _ = pg_dump_p.communicate()

                if pg_dump_p.returncode != 0:
                    raise subprocess.CalledProcessError(pg_dump_p.returncode, ' '.join(pg_dump_command), output=output)

                logger.info('pg_dump complete')

        except subprocess.CalledProcessError as e:
            logger.exception(e)
            logger.error('dumping database failure, bad return code [%s] from command [%s]' % (
                e.returncode, e.cmd
            ))
            return False

        try:
            logger.info('changing user modifications on file {filename}'.format(filename=abs_filepath))
            self.chown(abs_filepath, self.file_user, self.file_group)
            self.chmod(abs_filepath, self.file_mode)

            logger.info('dumping database success')
            return True
        except Exception as e:
            logger.exception(e)
            return False

    def dump_config(self, directory, system_tag, datetime_obj=datetime.now()):
        """
        Dump the configuration files on this local machine

        :param directory: directory to store the dumped file
        :param system_tag: system tag to format the filename with
        :param datetime_obj: datetime object to know how to format the string for the filename
        :return: success or failure
        """
        logger.info('dumping configuration')
        filename = datetime_obj.strftime(self.cfg_backup_name_format.format(system=system_tag))
        abs_filepath = os.path.join(directory, filename + '.tar')

        try:
            logger.info('creating tar for configurations {filename}'.format(filename=abs_filepath))
            if not DEBUG:
                subprocess.check_call(['tar', '-c', '-f', abs_filepath, '--files-from=/dev/null'])
                restore_script = '/opt/dosis2/bin/restore-config'
                
                if os.path.exists(restore_script):
                    subprocess.check_call([
                            'tar', '-r', '-f', abs_filepath, '-C', os.path.dirname(restore_script), os.path.basename(restore_script)
                        ])

                for d in self.cfg_directories:
                    if os.path.exists(d):
                        subprocess.check_call([
                            'tar', '-r', '-f', abs_filepath, '-C', os.path.dirname(d), os.path.basename(d)
                        ])

                for f in self.cfg_files:
                    if os.path.exists(f):
                        subprocess.check_call([
                            'tar', '-r', '-f', abs_filepath, '-C', os.path.dirname(f), os.path.basename(f)
                        ])


                subprocess.check_call(['bzip2', '-z', abs_filepath])
                abs_filepath += '.bz2'

            logger.info('changing owner and modifications on file {filename}'.format(filename=abs_filepath))
            self.chown(abs_filepath, self.file_user, self.file_group)
            self.chmod(abs_filepath, self.file_mode)

            logger.info('dumping configuration successful')
            return True
        except Exception as e:
            logger.exception(e)
            logger.error('dumping configuration failure')
            return False

    def dump_labels(self, directory, system_tag, datetime_obj=datetime.now()):
        """
        Dump the labels from the database on this local machine

        :param directory: directory to store the dumped file
        :param system_tag: system tag to format the filename with
        :param datetime_obj: datetime object to know how to format the string for the filename
        :return: success or failure
        """
        logger.info('dumping labels')
        filename = datetime_obj.strftime(self.lbl_backup_name_format.format(system=system_tag))
        abs_filepath = os.path.join(directory, filename + '.tar')

        try:
            logger.info('creating tar for configurations {filename}'.format(filename=abs_filepath))

            labels = self.methods.export_label_data()
            logger.info('exported labels: {labels}'.format(labels=', '.join(list(labels.keys()))))

            tmp_dir = mkdtemp(suffix='-labels')
            for label_name, label_data in list(labels.items()):
                lbl_path = os.path.join(tmp_dir, label_name + '.sql')
                logger.info('writing {label_path}'.format(label_path=lbl_path))
                with open(lbl_path, 'w') as lbl:
                    lbl.write(label_data)

            logger.info('creating tar for labels {filename}'.format(filename=abs_filepath))
            if not DEBUG:
                subprocess.check_call(['tar', '-c', '-f', abs_filepath, '--files-from=/dev/null'])
                for label_name in labels:
                    lbl_path = os.path.join(tmp_dir, label_name + '.sql')
                    if os.path.exists(lbl_path):
                        subprocess.check_call([
                            'tar', '-r', '-f', abs_filepath, '-C', os.path.dirname(lbl_path), os.path.basename(lbl_path)
                        ])
                subprocess.check_call(['bzip2', '-z', abs_filepath])
                abs_filepath += '.bz2'

            logger.info('removing temporary labels directory: {tmp_dir}'.format(tmp_dir=tmp_dir))
            shutil.rmtree(tmp_dir, ignore_errors=True)

            logger.info('changing owner and modification on file {filename}'.format(filename=abs_filepath))
            self.chown(abs_filepath, self.file_user, self.file_group)
            self.chmod(abs_filepath, self.file_mode)

            logger.info('dumping labels successful')
            return True
        except Exception as e:
            logger.exception(e)
            logger.error('dumping labels failed')
            return False

    def dump_dafe_specifics(self, directory, system_tag, datetime_obj=datetime.now()):
        """
        Dump the dafe specific settings from the database on this local machine

        :param directory: directory to store the dumped file
        :param system_tag: system tag to format the filename with
        :param datetime_obj: datetime object to know how to format the string for the filename
        :return: success or failure
        """
        logger.info('dumping dafe specifics')
        filename = datetime_obj.strftime(self.dafe_specifis_backup_name_format.format(system=system_tag))
        abs_filepath = os.path.join(directory, filename + '.tar')

        try:
            logger.info('creating tar for configurations {filename}'.format(filename=abs_filepath))

            spec = self.methods.export_dafes_specifics()
            logger.info('exported dafe specifics: {spec}'.format(spec=', '.join(list(spec.keys()))))

            tmp_dir = mkdtemp(suffix='-dafe-specifics')
            for dafe_spec, dafe_spec_data in list(spec.items()):
                spec_path = os.path.join(tmp_dir, dafe_spec + '.sql')
                logger.info('writing {spec_path}'.format(spec_path=spec_path))
                with open(spec_path, 'w') as spc:
                    spc.write(dafe_spec_data)

            logger.info('creating tar for dafe specifics {filename}'.format(filename=abs_filepath))
            if not DEBUG:
                subprocess.check_call(['tar', '-c', '-f', abs_filepath, '--files-from=/dev/null'])
                for dafe_spec in spec:
                    spec_path = os.path.join(tmp_dir, dafe_spec + '.sql')
                    if os.path.exists(spec_path):
                        subprocess.check_call([
                            'tar',
                            '-r',
                            '-f', abs_filepath,
                            '-C', os.path.dirname(spec_path),
                            os.path.basename(spec_path)
                        ])
                subprocess.check_call(['bzip2', '-z', abs_filepath])
                abs_filepath += '.bz2'

            logger.info('removing temporary dafe specifics directory: {tmp_dir}'.format(tmp_dir=tmp_dir))
            shutil.rmtree(tmp_dir, ignore_errors=True)

            logger.info('changing owner and modification on file {filename}'.format(filename=abs_filepath))
            self.chown(abs_filepath, self.file_user, self.file_group)
            self.chmod(abs_filepath, self.file_mode)

            logger.info('dumping dafe specifics successful')
            return True
        except Exception as e:
            logger.exception(e)
            logger.error('dumping dafe specifics failed')
            return False

    def dump_dafes(self, directory, datetime_obj=datetime.now()):
        """
        Dump all dafes if there are dafes available to dump

        :param directory: directory to base the dafe store directory on, will use dirname(directory)
        :param datetime_obj: datetime object for filename formats
        :return: success or failure
        """
        logger.info('dumping dafes')
        #thread_pool = ThreadPool(processes=2)
        dafes = self.methods.get_backup_dafes()

        for dafe_config in dafes:
            dafe = dafe_config['tag']
            ip = dafe_config['site_ip']
            logger.info('spawning thread for dafe {tag}@{ip}'.format(tag=dafe, ip=ip))
            dafe_directory = os.path.join(directory, dafe)
            dafe_remote_dir = directory
            # Took the update failed count so backup will upload and finish if a tower is down or unavailable. Really don't care
            # self.update_failed_count(
            
            if not self.dump_dafe(dafe_directory, dafe_remote_dir, dafe, ip, datetime_obj):
                logger.info("This tower did not back up correctly. Please check it further: " + str(dafe))

            #thread_pool.apply_async(
            #    self.dump_dafe,
            #    callback=self.update_failed_count,
            #    kwds={
            #        'directory': dafe_directory,
            #        'remote_dir': dafe_remote_dir,
            #        'dafe': dafe,
            #        'ip': ip,
            #        'datetime_obj': datetime_obj
            #    }
            #)
        #thread_pool.close()
        #thread_pool.join()
        logger.info('dumping dafes complete')
        return True

    def dump_dafe(self, directory, remote_directory, dafe, ip, datetime_obj=datetime.now()):
        """
        dump a single dafe backup

        :param directory: where to save the dafe dump
        :param remote_directory: remote directory on the dafe to dump
        :param dafe: dafe name
        :param ip: ip address of this dafe
        :param datetime_obj: datetime object for filename formats
        :return: success or failure
        """
        logger.info('dumping {dafe}@{ip}'.format(dafe=dafe, ip=ip))
        try:
            filename = datetime_obj.strftime(self.cfg_backup_name_format).format(system=dafe) + '.tar'
            filepath = os.path.join(remote_directory, filename)

            commands = [
                'test -d {path} && rm -rf {path}'.format(path=remote_directory),
                'install -m {mode} -o {owner} -g {group} -d {location}'.format(
                    mode='0774',
                    owner=self.file_user,
                    group=self.file_group,
                    location=remote_directory
                ),
                'tar -cf {filepath} --files-from=/dev/null'.format(filepath=filepath)
            ]
            for cfg_dir in self.cfg_directories:
                commands.append('test -d {cfg_dir} && tar -rf {filename} -C {dirn} {basen}'.format(
                    cfg_dir=cfg_dir,
                    filename=filepath,
                    dirn=os.path.dirname(cfg_dir),
                    basen=os.path.basename(cfg_dir)
                ))

            commands.append('bzip2 {filepath}'.format(filepath=filepath))

            client = paramiko.SSHClient()
            client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
            if not DEBUG:
                try:
                    client.connect(ip, username=self.sshcred.username, password=self.sshcred.password, auth_timeout=360)
                except Exception as e:
                    logger.exception(e)
                    return False
            self.execute_remote_script(client, commands)
            self.remake_directory(directory, self.file_mode, self.file_user, self.file_group)

            bzip_filepath = '{filename}.bz2'.format(filename=filepath)
            local_bzip_filepath = os.path.join(directory, os.path.basename(bzip_filepath))
            self.scp_from_remote(client, bzip_filepath, directory)

            self.chown(local_bzip_filepath, self.file_user, self.file_group)
            self.chmod(local_bzip_filepath, self.file_mode)

            logger.info('dumping dafe {dafe}@{ip} complete'.format(dafe=dafe, ip=ip))
            return True
        except Exception as e:
            logger.exception(e)
            return False

    def copy_to_mountpoint(self, path, mountpoint):
        """
        Copy the given path to a mountpoint recursively
        :param path: path to copy
        :param mountpoint: directory to copy this to
        :return: nothing
        """
        logger.info('making directory {mountpoint}'.format(mountpoint=mountpoint))
        if not DEBUG:
            self.makedirs(mountpoint)

        logger.info('mounting directory {mountpoint}'.format(mountpoint=mountpoint))
        try:
            if not DEBUG:
                with open(os.path.devnull, 'w') as devnull:
                    subprocess.call(['mount', mountpoint], stdout=devnull, stderr=subprocess.STDOUT)
        except Exception as e:
            logger.exception(e)
            logger.info('failed to mount directory {mountpoint}'.format(mountpoint=mountpoint))

        base = os.path.basename(path)
        destination = os.path.join(mountpoint, base)

        if not DEBUG:
            self.remake_directory(destination, mode=self.file_mode)

        logger.info('copying files from {backup_path} to {destination}'.format(
            backup_path=path,
            destination=destination
        ))
        if not DEBUG:
            logger.info('rmtree {destination}'.format(destination=destination))
            logger.info('copytree {source} {destination}'.format(source=path, destination=destination))
            shutil.rmtree(destination, ignore_errors=True)
            shutil.copytree(path, destination)

    def _purge_archive_no_workorder_database(self):
        result = self.methods.purge_archive_no_workorder()

        if result is None:
            raise RuntimeError('Purge archive no workorder db function returned None, indicating something went wrong')

        return result.upper() == 'FINISH'

    def _archive_database(self):
        result = self.methods.archive()

        if result is None:
            raise RuntimeError('Archive db function returned None, indicating something went wrong')

        return result.upper() == 'FINISH'

    def purge_archive_no_workorder_database(self):
        if self.args.no_archive:
            return FinishOrTimeoutResults.NO_RUN

        logger.info('running purge archive no workorder')

        if DEBUG:
            return FinishOrTimeoutResults.NO_RUN

        return run_until_finished_or_timeout(
            func=self._purge_archive_no_workorder_database,
            time_limit_mins=self.methods.get_purge_archive_daily_time_limit_mins()
        )

    def archive_database(self):
        """
        Run an archive on the database
        """
        if self.args.no_archive:
            return FinishOrTimeoutResults.NO_RUN

        logger.info('running archive')

        if DEBUG:
            return FinishOrTimeoutResults.NO_RUN

        #self.methods.archive()
        return run_until_finished_or_timeout(
            func=self._archive_database,
            time_limit_mins=self.methods.get_archive_daily_time_limit_mins()
        )

    def vacuum_database(self):
        """
        Run a vacuum on the database

        :return: nothing
        """
        if not self.args.no_vacuum:
            logger.info('running vacuum')
            if not DEBUG:
                self.su_methods.vacuum()

    def _cooler_database(self, num_chunks):
        thread_pool = ThreadPool(processes=num_chunks)

        results = thread_pool.map(self.methods.cooler, list(range(num_chunks)))
        thread_pool.close()
        thread_pool.join()

        # if we get a None back, something went wrong so lets bail out
        # so we don't just try to run the function over and over forever
        if any(result is None for result in results):
            raise RuntimeError('Cooler db function returned None, indicating that something went wrong.')

        return all(result.upper() == 'FINISH' for result in results)

    def cooler_database(self):
        if self.args.no_archive:
            return FinishOrTimeoutResults.NO_RUN

        logger.info('running cooler')

        if DEBUG:
            return FinishOrTimeoutResults.NO_RUN

        return run_until_finished_or_timeout(
            func=lambda: self._cooler_database(num_chunks=self.methods.get_cooler_num_chunks()),
            time_limit_mins=self.methods.get_cooler_daily_time_limit_mins()
        )

    def _cooler_summarize(self, num_recs):
        result = self.methods.summarize(num_recs)

        if result is None:
            raise RuntimeError('Cooler Summarize db function returned None, indicating something went wrong')

        return result.upper() == 'FINISH'

    def cooler_summarize(self):
        if self.args.no_archive:
            return FinishOrTimeoutResults.NO_RUN

        logger.info('running cooler summarize process')

        if DEBUG:
            return FinishOrTimeoutResults.NO_RUN

        return run_until_finished_or_timeout(
            func=lambda: self._cooler_summarize(num_recs=self.methods.get_cooler_summary_max_recs()),
            time_limit_mins=self.methods.get_cooler_daily_time_limit_mins()
        )

    def _cooler_prep_database(self):
        result = self.methods.cooler_prep()

        if result is None:
            raise RuntimeError('Cooler Prep db function returned None, indicating something went wrong')

        return result.upper() == 'FINISH'

    def cooler_prep_database(self):
        if self.args.no_archive:
            return FinishOrTimeoutResults.NO_RUN

        logger.info('running cooler prep')

        if DEBUG:
            return FinishOrTimeoutResults.NO_RUN

        return run_until_finished_or_timeout(
            func=self._cooler_prep_database,
            time_limit_mins=self.methods.get_cooler_daily_time_limit_mins()
        )

def is_backupable():
    rv = False
    config_path = '/opt/dosis2/etc/custom/dosisConfig.xml'
    db_server = None
    dosis_config_tree = ET.parse(config_path)
    general = dosis_config_tree.find('./section[@name="general"]')
    for child in general:
        if child.attrib['name'] == 'DBServer':
            db_server = child.find('value').text
            break

    if '127.0.0.1' == db_server:
        rv = True
    else:
        rv = False

    return rv


def check_for_updater_cron():
    updater_cron_name = '/etc/cron.d/update_handler_update_from_prep_cron'

    if os.path.exists(updater_cron_name):
        logger.info('an update is scheduled checking...')
        now = datetime.now()
        day_of_month_number = now.day

        # get UPDATE_DAY from cron job.
        update_day_from_file = None
        with open(updater_cron_name, 'r') as f:
            for line in f:
                if line.startswith("UPDATE_DAY="):
                    update_day_from_file = int(line.split('=')[1].strip())
                    break

        # Looks like backup runs around 4am. About the same as updater.
        #So if day of week_number == update_day_from_file exit backup.
        if day_of_month_number == update_day_from_file:
            logger.info('An update is scheduled for the same time as backup. holding backup...')
            sys.exit()
        else:
            logger.info('update is scheduled but not for today. continuing backup...')
    else:
        logger.info('No update scheduled, continuing.')


if __name__ == '__main__':
    if not is_backupable():
        sys.exit('system is not backupable')

    check_for_updater_cron()

    backup = DosisBackup(
        base_dosisconfig='/opt/dosis2/etc/dosisConfig.xml',
        custom_dosisconfig='/opt/dosis2/etc/custom/dosisConfig.xml',
        sshconfig='/opt/dosis2/.sshcredentials',
        local_backup_root='/opt/dosis2/backup',
        usb_backup_root='/mnt/dosisbackup',
        site_backup_root='/mnt/sitebackup/backups',
        rpm_backup_root='/opt/dosis2/rpm_backup',
        file_mode=0o0774,
        file_user='dPG',
        file_group='dosis',
        db_backup_name_format='{system}-dbBackup-%Y-%m-%d_%H-%M-%S',
        cfg_backup_name_format='{system}-cfgBackup-%Y-%m-%d_%H-%M-%S',
        lbl_backup_name_format='{system}-lblBackup-%Y-%m-%d_%H-%M-%S',
        dafe_specifics_backup_name_format='{system}-dafeSpecifics-%Y-%m-%d_%H-%M-%S',
        db_superuser='dsuper',
        cfg_directories=['/etc/cups', '/opt/dosis2/etc', '/opt/dosis2/labels', '/opt/dosis2/customer', '/etc/hosts', 
        '/etc/postgresql/10/main/pg_hba.conf', '/var/lib/pgsql/9.2/data/pg_hba.conf'],
        cfg_files=['/opt/dosis2/.smbcredentials', '/etc/auto.cifs', '/etc/auto.master', '/etc/hosts',
        '/etc/pmissync/dosis_pmis_sync.json']
    )
    if backup.run_backup():
        logger.info('backup was successful')
    else:
        logger.error('backup was not successful')
