#!/usr/bin/python2
#
# Copyright (c) 2013-2014 Rafael Martinez Guerrero / PostgreSQL-es
# rafael@postgresql.org.es / http://www.postgresql.org.es/
#
# Copyright (c) 2014 USIT-University of Oslo
#
# This file is part of PgBackMan
# https://github.com/rafaelma/pgbackman
#
# PgBackMan is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# PgBackMan is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Pgbackman.  If not, see <http://www.gnu.org/licenses/>.

'''
This program is used by PgBackMan to control the crontab and at jobs running
backups in this backup server.

pgbackman_control uses LISTEN/NOTIFY to track when it has to take action.

In addition pgbackman_control will create cache data for the Backup server and all PgSQL nodes, 
in case the pgbackman database is not available when backups have to be executed.

pgbackman_control will catch up with changes that happened during a down period of the program. 
No changes will be lose if pgbackman_control is down.
'''

import subprocess
import select
import tempfile
import shutil
import pwd
import grp
import sys
import os
import time
import datetime
import socket
import signal

from pgbackman.logs import *
from pgbackman.database import *
from pgbackman.config import *

listen_list = []
last_crontab_update = {}

# ############################################
# Function add_to_listen_channels()
# ############################################

def add_to_listen_channels(db,db_notify,backup_server_id):
    '''LISTEN to all active channels'''
    
    global listen_list
    
    old_listen_list = listen_list
    new_listen_list = []
    listen_list = []

    try:
        for channel in db_notify.get_listen_channel_names(backup_server_id): 
            listen_list.append(channel)

        new_listen_list = set(listen_list) - set(old_listen_list)    

        for channel in new_listen_list:
            db_notify.add_listen(channel)
            logs.logger.info('Listening to channel: %s',channel)

        return new_listen_list

    except Exception as e:
        logs.logger.error('Problems listening to channels - %s',e)
            

# ############################################
# Function delete_from_listen_channels()
# ############################################

def delete_from_listen_channels(db,db_notify,backup_server_id):
    '''UNLISTEN to channels'''

    global listen_list
    
    new_listen_list = []
    delete_listen_list = []

    try:
        for channel in db_notify.get_listen_channel_names(backup_server_id): 
            new_listen_list.append(channel)
        
        delete_listen_list = set(listen_list)-set(new_listen_list)
        listen_list = []
        listen_list = new_listen_list

        for channel in delete_listen_list:
            db_notify.delete_listen(channel)
            logs.logger.info('Unlistening to channel: %s',channel)

        return delete_listen_list
        
    except Exception as e:
        logs.logger.error('Problems unlistening to channels - %s',e)

                 
# ############################################
# Function get_pgsql_node_id_from_channel()
# ############################################

def get_pgsql_node_id_from_channel(channel):
    '''Extract PgSQL node ID from the channel name'''

    pgsql_id = channel.split('_pg')[1]
    return pgsql_id

            
# ############################################
# Function generate_crontab_backup_jobs()
# ############################################

def generate_crontab_backup_jobs(db,backup_server_id,pgsql_node_id):
    '''Generate a crontab file for a PgSQL node'''

    global last_crontab_update

    try:
        crontab_file = db.get_pgsql_node_config_value(pgsql_node_id,'pgnode_crontab_file')
    
        with open(crontab_file,'w') as file:
            data = db.generate_crontab_backup_jobs(backup_server_id,pgsql_node_id)
            
            file.write(data)
            file.flush()

            logs.logger.info('Crontab file: %s created/updated',crontab_file)

        last_crontab_update[str(pgsql_node_id)] = datetime.datetime.now()    
      
    except Exception as e:
            
        # If we cannot create the crontab file, we have to update
        # the job_queue in the database so we don't loose this update.
            
        try:
            logs.logger.error('Problems creating/updating the crontab file: %s - %s',crontab_file,e)
            db.update_job_queue(backup_server_id,pgsql_node_id)
            
        except Exception as e:
            logs.logger.error('Problems updating job queue for SrvID: %s and nodeID: %s after a crontab file update error - %s',backup_server_id,pgsql_node_id,e)


# ############################################
# Function generate_snapshot_at_jobs()
# ############################################

def generate_snapshot_at_jobs(db,backup_server_id,tmp_dir):
    '''Generate at jobs for snapshot jobs'''

    try:
        for record in db.get_new_snapshots(backup_server_id):
            
            snapshot_id = record[0]
            at_time = record[1]
            
            at_file_temp_file = tempfile.NamedTemporaryFile(delete=True,dir=tmp_dir)  

            with open(at_file_temp_file.name, 'r+') as at_file:
                data = db.generate_snapshot_at_file(snapshot_id)

                at_file.write(data)
                at_file.flush()
                
                at_command = 'at -f ' + at_file_temp_file.name + ' -t ' + at_time

                proc = subprocess.Popen([at_command],shell=True)
                proc.wait()
                
                if proc.returncode == 0:
                    db.update_snapshot_status(snapshot_id,'DEFINED')
                    logs.logger.info('AT job for snapshotID: %s has been defined',snapshot_id) 
                
                elif proc.returncode != 0:
                    db.update_snapshot_status(snapshot_id,'ERROR')
                    logs.logger.error('AT job for snapshotID: %s could not be defined',snapshot_id) 

    except Exception as e:
        logs.logger.error('Could not generate AT file for a snapshot - %s',e)


# ############################################
# Function generate_restore_at_jobs()
# ############################################

def generate_restore_at_jobs(db,backup_server_id,tmp_dir):
    '''Generate at jobs for restore jobs'''

    try:
        for record in db.get_new_restore(backup_server_id):
            
            restore_id = record[0]
            at_time = record[1]
            
            at_file_temp_file = tempfile.NamedTemporaryFile(delete=True,dir=tmp_dir)  

            with open(at_file_temp_file.name, 'r+') as at_file:
                data = db.generate_restore_at_file(restore_id)

                at_file.write(data)
                at_file.flush()
                
                at_command = 'at -f ' + at_file_temp_file.name + ' -t ' + at_time

                proc = subprocess.Popen([at_command],shell=True)
                proc.wait()
                
                if proc.returncode == 0:
                    db.update_restore_status(restore_id,'DEFINED')
                    logs.logger.info('AT job for restoreID: %s has been defined',restore_id) 
                
                elif proc.returncode != 0:
                    db.update_restore_status(restore_id,'ERROR')
                    logs.logger.error('AT job for restoreID: %s could not be defined',restore_id) 

    except Exception as e:
        logs.logger.error('Could not generate AT file for a restore - %s',e)


# ############################################
# Function generate_all_crontab_jobs()
# ############################################
                         
def generate_all_crontab_jobs(db,backup_server_id):
    '''
    Get all the PgSQL node IDs that need to get a new crontab file installed
    when starting pgbackman_control or after the pgbackman database has not been 
    available for pgbackman_control
    '''
    
    try:
        pgsql_node_id = db.get_next_crontab_id_to_generate(backup_server_id)
    
        while pgsql_node_id != None:
            generate_crontab_backup_jobs(db,backup_server_id,pgsql_node_id)
            pgsql_node_id = db.get_next_crontab_id_to_generate(backup_server_id)
 
        logs.logger.info('All crontab jobs in queue processed')
   
    except Exception as e:
        logs.logger.error('Problems getting next crontab file ID to generate - %s',e)
   

# ############################################
# Function create_global_directories()
# ############################################

def create_global_directories(db,backup_server_fqdn,backup_server_id):
    '''Create global directories used for cache and pending database registrations'''
    
    try:
        uid = pwd.getpwnam('pgbackman').pw_uid
        gid = grp.getgrnam('pgbackman').gr_gid

    except Exception as e:
        logs.logger.error('Problems getting UID and GID values for pgbackman - %s',e)  

    try:
        root_backup_partition = db.get_backup_server_config_value(backup_server_id,'root_backup_partition')

    except Exception as e:
        logs.logger.error('Problems getting root backup partition used in %s - %s',backup_server_fqdn,e)    

    backup_server_pending_registration_dir = root_backup_partition + '/pending_updates'
    backup_server_cache_dir = root_backup_partition +  '/cache_dir'

    #
    # Pending log registration directory
    #

    if os.path.exists(backup_server_pending_registration_dir):
        logs.logger.debug('Pending log registration directory exists: %s',backup_server_pending_registration_dir)
    else:
        logs.logger.debug('Pending log registration directory does not exist: %s',backup_server_pending_registration_dir)
        
        try:
            os.makedirs(backup_server_pending_registration_dir,0700)
            logs.logger.info('Pending log registration directory created: %s',backup_server_pending_registration_dir)
        except OSError as e:
            logs.logger.critical('OS error when creating the pending log registration directory: %s',e)
            sys.exit(1)
            
    try:
        os.chown(backup_server_pending_registration_dir, uid, gid)
        logs.logger.info('UID: %s and GID: %s defined for the directory %s',uid,gid,backup_server_pending_registration_dir)

    except OSError as e:
        logs.logger.critical('OS error when defining privileges for the pending registration directory - %s',e)

    #
    # Cache directory
    #

    if os.path.exists(backup_server_cache_dir):
        logs.logger.debug('Cache directory exists: %s',backup_server_cache_dir)
    else:
        logs.logger.debug('Cache directory does not exist: %s',backup_server_cache_dir)
        
        try:
            os.makedirs(backup_server_cache_dir,0700)
            logs.logger.info('Cache directory created: %s',backup_server_cache_dir)

        except OSError as e:
            logs.logger.critical('OS error when creating the cache directory: %s',e)
            sys.exit(1)

    try:
        os.chown(backup_server_cache_dir, uid, gid)
        logs.logger.info('UID: %s and GID: %s defined for the directory %s',uid,gid,backup_server_cache_dir)

    except OSError as e:
        logs.logger.critical('OS error when defining privileges for the cache directory - %s',e)
                

# ############################################
# Function create_pgsql_node_backup_directories()
# ############################################
    
def create_pgsql_node_backup_directories(db,pgsql_node_id):
    '''
    Create the directories needed for PgSQL nodes backups
    '''

    try:
        uid = pwd.getpwnam('pgbackman').pw_uid
        gid = grp.getgrnam('pgbackman').gr_gid

    except Exception as e:
        logs.logger.error('Problems getting UID and GID values for pgbackman - %s',e)  

    try:
        pgnode_backup_partition = db.get_pgsql_node_config_value(pgsql_node_id,'pgnode_backup_partition')

    except Exception as e:
        logs.logger.error('Problems getting backup partition used for NodeID: %s - %s',pgsql_node_id,e)   

    #
    # Dump directory
    #

    if os.path.exists(pgnode_backup_partition + '/dump'):
        logs.logger.debug('Dump directory %s exists',pgnode_backup_partition + '/dump')
    else:
        logs.logger.warning('Dump directory %s does not exist',pgnode_backup_partition + '/dump')
        
        try:
            os.makedirs(pgnode_backup_partition + '/dump',0700)
            logs.logger.info('Dump directory %s created',pgnode_backup_partition + '/dump')

        except OSError as e:
            logs.logger.critical('OS error when creating the dump directory - %s',e)
            sys.exit(1)

    try:
        os.chown(pgnode_backup_partition, uid, gid)
        os.chown(pgnode_backup_partition + '/dump', uid, gid)
        logs.logger.info('UID: %s and GID: %s defined for the directory %s',uid,gid,pgnode_backup_partition + '/dump')

    except OSError as e:
        logs.logger.critical('OS error when defining privileges for the dump directory - %s',e)
        sys.exit(1)
     
    #
    # Log directory
    #
           
    if os.path.exists(pgnode_backup_partition + '/log'):
        logs.logger.debug('Log directory %s exists',pgnode_backup_partition + '/log')
    else:
        logs.logger.warning('Log directory %s does not exist',pgnode_backup_partition + '/log')
        
        try:
            os.makedirs(pgnode_backup_partition + '/log',0700)
            logs.logger.info('Log directory %s created',pgnode_backup_partition + '/log')

        except OSError as e:
            logs.logger.critical('OS error when creating the log directory - %s',e)
            sys.exit(1)       

    try:
        os.chown(pgnode_backup_partition, uid, gid)
        os.chown(pgnode_backup_partition + '/log', uid, gid)
        logs.logger.info('UID: %s abd GID: %s defined for the directory %s',uid,gid,pgnode_backup_partition + '/log')

    except OSError as e:
        logs.logger.critical('OS error when defining privileges for the log directory - %s',e)
        sys.exit(1)       


# ############################################
# Function process_pgsql_node_to_delete()
# ############################################

def process_pgsql_node_to_delete(db,backup_server_id):
    '''Delete PgSQL node data from nodes that has been deleted.'''
    
    global last_crontab_update

    try:
        for record in db.get_pgsql_node_to_delete(backup_server_id):

            root_backup_partition = db.get_backup_server_config_value(backup_server_id,'root_backup_partition')
            pgsql_node_backup_dir = root_backup_partition + '/pgsql_node_' + str(record[1])
            crontab_file = '/etc/cron.d/pgsql_node_' + str(record[1])

            backup_server_cache_dir = root_backup_partition +  '/cache_dir'
            pgsql_node_cache_file = backup_server_cache_dir + '/pgsql_node_' + str(record[1]) + '.cache'
        
            #
            # Deleting PgSQL node crontab file
            #  

            if os.path.exists(crontab_file):
                 os.unlink(crontab_file)
                 logs.logger.info('Crontab file: %s deleted',crontab_file)

            else:
                logs.logger.warning('Crontab file: %s does not exist',crontab_file) 

            del last_crontab_update[str(record[1])]

            #
            # Deleting cache file for PgSQL node
            #

            if os.path.exists(pgsql_node_cache_file):
                os.unlink(pgsql_node_cache_file)
                logs.logger.info('Cache file: %s deleted',pgsql_node_cache_file)
        
            else:
                logs.logger.warning('Cache file: %s does not exist',pgsql_node_cache_file)
             
            #
            # Deleting PgSQL node backup dir
            #  
            
            if os.path.exists(pgsql_node_backup_dir):
                shutil.rmtree(pgsql_node_backup_dir,True)
                logs.logger.info('PgSQL node backup dir: %s deleted',pgsql_node_backup_dir)
                
            else:
                logs.logger.warning('PgSQL node backup dir: %s does not exist',pgsql_node_backup_dir)
                
            db.delete_pgsql_node_to_delete(backup_server_id,record[1])

    except Exception as e:
        logs.logger.error('Problems deleting data from PgSQL node when the node has been deleted - %s',e)   


# ############################################
# Function process_pgsql_node_stopped()
# ############################################

def process_pgsql_node_stopped(db,backup_server_id):
    '''Update crontab for PgSQL nodes stopped when pgbackman_control was down.'''
    
    try:
        for record in db.get_pgsql_node_stopped():

            crontab_file = db.get_pgsql_node_config_value(record[0],'pgnode_crontab_file')
            generate_crontab_backup_jobs(db,backup_server_id,record[0])

    except Exception as e:
        logs.logger.error('Problems updating crontab of pgsql nodes stopped when pgbackman_control was down - %s',e)   


# ############################################
# Function update_backup_server_cache_data()
# ############################################

def update_backup_server_cache_data(db,backup_server_fqdn,backup_server_id):
    '''Update the cache data for the backup server'''

    root_backup_partition = pgsql_bin_9_6 = pgsql_bin_9_5 = pgsql_bin_9_4 = pgsql_bin_9_3 = pgsql_bin_9_2 = pgsql_bin_9_1 = pgsql_bin_9_0 = None
    
    try:
        uid = pwd.getpwnam('pgbackman').pw_uid
        gid = grp.getgrnam('pgbackman').gr_gid

    except Exception as e:
        logs.logger.error('Problems getting UID and GID values for pgbackman - %s',e)  

    try:
        root_backup_partition = db.get_backup_server_config_value(backup_server_id,'root_backup_partition')

        pgsql_bin_9_6 = db.get_backup_server_config_value(backup_server_id,'pgsql_bin_9_6')
        pgsql_bin_9_5 = db.get_backup_server_config_value(backup_server_id,'pgsql_bin_9_5')
        pgsql_bin_9_4 = db.get_backup_server_config_value(backup_server_id,'pgsql_bin_9_4')
        pgsql_bin_9_3 = db.get_backup_server_config_value(backup_server_id,'pgsql_bin_9_3')
        pgsql_bin_9_2 = db.get_backup_server_config_value(backup_server_id,'pgsql_bin_9_2')
        pgsql_bin_9_1 = db.get_backup_server_config_value(backup_server_id,'pgsql_bin_9_1')
        pgsql_bin_9_0 = db.get_backup_server_config_value(backup_server_id,'pgsql_bin_9_0')
        
        backup_server_cache_dir = root_backup_partition +  '/cache_dir'
        backup_server_cache_file = backup_server_cache_dir + '/backup_server_' + backup_server_fqdn + '.cache'

        if os.path.exists(backup_server_cache_dir):
        
            with open(backup_server_cache_file,'w') as backup_server_cache:
                backup_server_cache.write('backup_server_id::' + str(backup_server_id) + '\n' +
                                          'backup_server_fqdn::' + backup_server_fqdn + '\n' +
                                          'root_backup_partition::' + root_backup_partition + '\n' +
                                          'pgsql_bin_9_6::' + pgsql_bin_9_6 + '\n' +
                                          'pgsql_bin_9_5::' + pgsql_bin_9_5 + '\n' +
                                          'pgsql_bin_9_4::' + pgsql_bin_9_4 + '\n' +
                                          'pgsql_bin_9_3::' + pgsql_bin_9_3 + '\n' +
                                          'pgsql_bin_9_2::' + pgsql_bin_9_2 + '\n' +
                                          'pgsql_bin_9_1::' + pgsql_bin_9_1 + '\n' +
                                          'pgsql_bin_9_0::' + pgsql_bin_9_0 + '\n')
                
                os.chmod(backup_server_cache_file,0644)
                os.chown(backup_server_cache_file, uid, gid)
                logs.logger.info('Cache file: %s created/updated',backup_server_cache_file)
                
    except Exception as e:
        logs.logger.error('Problems updating backup server cache data for %s - %s',backup_server_fqdn,e)   
        
    
# ############################################
# Function update_pgsql_node_cache_data()
# ############################################

def update_pgsql_node_cache_data(db,backup_server_id,pgsql_node_id):
    '''Update the cache data for a PgSQL node'''
        
    try:
        uid = pwd.getpwnam('pgbackman').pw_uid
        gid = grp.getgrnam('pgbackman').gr_gid

    except Exception as e:
        logs.logger.error('Problems getting UID and GID values for pgbackman - %s',e)  

    try:
        pgsql_node_fqdn = db.get_pgsql_node_fqdn(pgsql_node_id)
        pgsql_node_backup_dir = db.get_pgsql_node_config_value(pgsql_node_id,'pgnode_backup_partition')
        root_backup_partition = db.get_backup_server_config_value(backup_server_id,'root_backup_partition')
           
        backup_server_cache_dir = root_backup_partition +  '/cache_dir'
        pgsql_node_cache_file = backup_server_cache_dir + '/pgsql_node_' + str(pgsql_node_id) + '.cache'
        
        if os.path.exists(backup_server_cache_dir):
        
            with open(pgsql_node_cache_file,'w') as pgsql_node_cache:
                pgsql_node_cache.write('pgsql_node_id::' + str(pgsql_node_id) + '\n' +
                                       'pgsql_node_fqdn::' + pgsql_node_fqdn + '\n' +
                                       'pgnode_backup_partition::' + pgsql_node_backup_dir + '\n')

                os.chmod(pgsql_node_cache_file,0644)
                os.chown(pgsql_node_cache_file, uid, gid)
                logs.logger.info('Cache file: %s created/updated',pgsql_node_cache_file)
                   
    except Exception as e:
        logs.logger.error('Problems updating pgsql node cache data for %s - %s',pgsql_node_fqdn,e)   
   

# ############################################
# Function check_database_connection()
# ############################################
  
def check_database_connection(db):
    '''Check if we can connect to the database server and the pgbackman database'''

    try:
        db.pg_connect()
        return True
    except Exception as e:    
        return False
        

# ############################################
# Function main()
# ############################################

def main():
    global listen_list
    global last_crontab_update

    conf = PgbackmanConfiguration()
    dsn = conf.dsn
    tmp_dir = conf.tmp_dir

    logs.logger.debug('Backup server ID from config file: %s',conf.backup_server)
    logs.logger.debug('Backup server FQDN: %s',socket.getfqdn())
    logs.logger.debug('DSN: host=%s hostaddr=%s port=%s database=%s user=%s ',conf.dbhost,conf.dbhostaddr,conf.dbport,conf.dbname,conf.dbuser)

    db = PgbackmanDB(dsn, 'pgbackman_control')

    #
    # We check before starting if the database is available. 
    # If it is not available we will wait conf.pg_connect_retry_interval 
    # and try again 

    check_db = check_database_connection(db)

    while not check_db:
        logs.logger.critical('The pgbackman database is not available. Waiting %s seconds before trying again',conf.pg_connect_retry_interval)
        
        time.sleep(conf.pg_connect_retry_interval)
        check_db = check_database_connection(db)
        
    logs.logger.debug('Database server is up and running and pgbackman database is available')
    
    #
    # Instance used to have a persistance connection to the database 
    # So we can use LISTEN/NOTIFY
    #
    db_notify = PgbackmanDB(dsn, 'pgbackman_notify')
    
    try:
        db_notify.pg_connect()
    except Exception as e:
        logs.logger.critical('Problems creating a permanent connection to the pgbackman database for LISTEN/NOTIFY data - %s',e)   
        sys.exit(1)

    #
    # Checking if this backup server is registered in pgbackman
    #

    if conf.backup_server != '':
        backup_server_fqdn = conf.backup_server
    else:
        backup_server_fqdn = socket.getfqdn()
    
    try:
        backup_server_id = db.get_backup_server_id(backup_server_fqdn)
        logs.logger.info('Backup server: %s is registered in pgbackman',backup_server_fqdn)

    except psycopg2.Error as e:
        logs.logger.critical('Cannot find backup server %s in pgbackman. Stopping pgbackman_control.',backup_server_fqdn)
        logs.logger.info('**** Pgbackman_Control stopped. ****')
        sys.exit(1)     

    create_global_directories(db,backup_server_fqdn,backup_server_id)
    update_backup_server_cache_data(db,backup_server_fqdn,backup_server_id)

    #
    # Start listening to active channels when we start pgbackman_control. 
    # Create cache and backup directories for new pgsql nodes with active backup definitions 
    # created when pgbackman_control was down.
    #
    
    listen_channels = add_to_listen_channels(db,db_notify,backup_server_id)
   
    for listen_channel in listen_channels: 
        if listen_channel != 'channel_pgsql_node_running' \
                and listen_channel != 'channel_pgsql_node_stopped' \
                and listen_channel != 'channel_pgsql_node_deleted' \
                and listen_channel != 'channel_snapshot_defined' \
                and listen_channel != 'channel_restore_defined':

            pgsql_node_id = get_pgsql_node_id_from_channel(listen_channel)

            #
            # Initialize this variable for all PgSQL nodes in the system
            #
            last_crontab_update[str(pgsql_node_id)] = datetime.datetime.now()
            
            update_pgsql_node_cache_data(db,backup_server_id,pgsql_node_id)
            create_pgsql_node_backup_directories(db,pgsql_node_id)

    #
    # Check if there are some crontab/at jobs to generate or pgsql nodes that 
    # have been deleted or stopped  when we start pgbackman_control. 
    #
    # This is necessary just in case pgbackman_control has been down and missed some NOTIFYs 
    # from the central database.
    #

    generate_all_crontab_jobs(db,backup_server_id)
    process_pgsql_node_to_delete(db,backup_server_id)
    process_pgsql_node_stopped(db,backup_server_id)

    generate_snapshot_at_jobs(db,backup_server_id,tmp_dir)
    generate_restore_at_jobs(db,backup_server_id,tmp_dir)

    #
    # Main loop waiting for notifications
    #

    while True:
        channels = []        
 
        try:

            #
            # We wait for notifies from the database. The select
            # function blocks until we get a notify because the
            # timeout argument is omitted.
            #
            
            select.select([ db_notify.conn],[],[],)
                
            db_notify.conn.poll()

            while db_notify.conn.notifies:
                channel = db_notify.conn.notifies.pop().channel
                channels.append(channel)

            for channel in set(channels):
                if channel == 'channel_pgsql_node_running':
                    
                    #
                    # A PgSQL node has been registered with status RUNNING
                    #
                    logs.logger.info('Notify: PgSQL node registered with status RUNNING')
                    listen_channels = add_to_listen_channels(db,db_notify,backup_server_id)

                    #
                    # Create cache data, backup directory and crontab file 
                    # for a pgsql_node if they do not exist. 
                    #
                    
                    for listen_channel in listen_channels: 

                        if listen_channel != 'channel_pgsql_node_running' \
                                and listen_channel != 'channel_pgsql_node_stopped' \
                                and listen_channel != 'channel_pgsql_node_deleted' \
                                and listen_channel != 'channel_snapshot_defined' \
                                and listen_channel != 'channel_restore_defined':
                            
                            pgsql_node_id = get_pgsql_node_id_from_channel(listen_channel)
                            
                            update_pgsql_node_cache_data(db,backup_server_id,pgsql_node_id)
                            create_pgsql_node_backup_directories(db,pgsql_node_id)
                            generate_crontab_backup_jobs(db,backup_server_id,pgsql_node_id)

                elif channel == 'channel_pgsql_node_stopped':
                    
                    #
                    # A PgSQL node has been registered with status STOPPED
                    #
                    logs.logger.info('Notify: PgSQL node registered with status STOPPED')
                    unlisten_channels = delete_from_listen_channels(db,db_notify,backup_server_id)
                    
                    #
                    # Delete all backup jobs in the crontab file of the stopped node.
                    #

                    for unlisten_channel in unlisten_channels:
                        
                        if unlisten_channel != 'channel_pgsql_node_running' \
                                and unlisten_channel != 'channel_pgsql_node_stopped' \
                                and unlisten_channel != 'channel_pgsql_node_deleted' \
                                and unlisten_channel != 'channel_snapshot_defined' \
                                and unlisten_channel != 'channel_restore_defined':
                            
                            pgsql_node_id = get_pgsql_node_id_from_channel(unlisten_channel)
                            generate_crontab_backup_jobs(db,backup_server_id,pgsql_node_id)

                elif channel == 'channel_pgsql_node_deleted':

                    #
                    # A PgSQL node has been deleted
                    #
                    logs.logger.info('Notify: PgSQL node deleted')
                    
                    unlisten_channels = delete_from_listen_channels(db,db_notify,backup_server_id)
                    process_pgsql_node_to_delete(db,backup_server_id)

                elif channel == 'channel_snapshot_defined':
                    
                    #
                    # A snapshot backup has been defined
                    #
                    logs.logger.debug('Notify: Backup snapshot defined')
                    generate_snapshot_at_jobs(db,backup_server_id,tmp_dir)

                elif channel == 'channel_restore_defined':
                    
                    #
                    # A restore job has been defined
                    #
                    logs.logger.info('Notify: Restore defined')
                    generate_restore_at_jobs(db,backup_server_id,tmp_dir)
                    
                else:
                    #
                    # A backup job has been registered, updated or deleted
                    #                    
                    logs.logger.info('Notify: backup definition registered, updated or deleted. Channel: %s',channel)
                    pgsql_node_id = db.get_next_crontab_id_to_generate(backup_server_id)

                    # 
                    # We want to avoid the generation of a new crontab
                    # file for a PgSQL node to often.
                    #
                    # This can happen if we define a bulk backup
                    # definition for all the databases in a PgSQL node
                    # with the special dbname "#all_databases#" and
                    # the PgSQL node has many databases. The system
                    # will generate a new NOTIFY for every new
                    # database definition created.
                    #
                    # If the crontab file for a PgSQL node has been
                    # updated less than 10 seconds ago, we will wait
                    # 10 seconds before generating the file again.
                    #

                    if datetime.datetime.now() - datetime.timedelta(seconds=10) < last_crontab_update[str(pgsql_node_id)]: 

                        logs.logger.info('Controlling update ratio of the crontab file for PgSQL node: %s. Waiting 10sec before updating it.',pgsql_node_id)
                        time.sleep(10)

                    generate_crontab_backup_jobs(db,backup_server_id,pgsql_node_id)
                        
            
        except psycopg2.OperationalError as e:

            #
            # If we lose the connection to the database, we will wait conf.pg_connect_retry_interval
            # before trying to connect again. When the database is available again we will reset
            # all the listen channels and check if there are some crontab jobs in queue to be processed
            #

            logs.logger.critical('Operational error: %s',e)

            check_db = check_database_connection(db)
            
            while not check_db:
                logs.logger.critical('We have lost the connection to the database. Waiting %s seconds before trying again',conf.pg_connect_retry_interval)
                
                time.sleep(conf.pg_connect_retry_interval)
                check_db = check_database_connection(db)

            db_notify = None
            db_notify = PgbackmanDB(dsn, 'pgbackman_notify')
            
            try:
                db_notify.pg_connect()
            except Exception as e:
                logs.logger.critical('Problems creating a permanent connection to the pgbackman database for LISTEN/NOTIFY data - %s',e)   
                sys.exit(1)

            listen_list = []
 
            add_to_listen_channels(db,db_notify,backup_server_id)
            generate_all_crontab_jobs(db,backup_server_id)
            generate_snapshot_at_jobs(db,backup_server_id,tmp_dir)
            generate_restore_at_jobs(db,backup_server_id,tmp_dir)

        except Exception as e:
            logs.logger.error('General error in main loop - %s',e)   
        
    db_notify.pg_close()
    db.pg_close()

        
# ############################################
# Function signal_handler()
# ############################################
    
def signal_handler(signum, frame):
    logs.logger.info('**** Pgbackman_Control stopped. ****')
    sys.exit(0)


# ############################################
# 
# ############################################

if __name__ == '__main__':

    logs = PgbackmanLogs("pgbackman_control", "", "")
    logs.logger.info('**** pgbackman_control started. ****')

    signal.signal(signal.SIGINT,signal_handler)
    signal.signal(signal.SIGTERM,signal_handler)

    main()

