#!/usr/bin/env python
#Copyright 2004,2005,2006 Sebastian Hagen
# This file is part of eucharis.

# eucharis is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2
# as published by the Free Software Foundation

# eucharis is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

# You should have received a copy of the GNU General Public License
# along with eucharis; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA

import os
import time
import re
import logging
import socket
import optparse
import urllib2
import httplib
import threading

import __main__
import init_misc
import socket_management
from socket_management import Timer
import input_handlers
import irc_base
import dns_server
import asynchronous_processing
from authentication_system import InsufficientPermissionsError
import data_formatting

defaults = {}
defaults['rehash_data'] = {}

init_misc.set_defaults(defaults, globals())

network_connection = True

class core_base:
   def __init__(self, irc_server=None, irc_client=None, botnet_interface=None, control_connection=None):
      self.connections = {
         'irc_server':irc_server,
         'irc_client':irc_client,
         'botnet_interface':botnet_interface,
         'control_connection':control_connection
         }

      for elementname in self.connections.keys():
         if (self.connections[elementname] == None):
            del(self.connections[elementname])
            if (elementname in self.handler_bindings):
               del(self.handler_bindings[elementname])
         elif (elementname in __main__.connection_users):
            __main__.connection_users[elementname].append(self)

   def clean_up(self):
      self.handlers_modify(action='unregister')
      for elementname in self.connections.keys():
         if ((elementname in __main__.connection_users) and (self in __main__.connection_users[elementname])):
            __main__.connection_users[elementname].remove(self)

class iptrack_server:
   def __init__(self, servername, host=None, record_limit_relative = 10):
      if (host == None):
         self.host = servername
      else:
         self.host = host

      self.loggername = 'ipt_server.' + servername
      self.logger = logging.getLogger(self.loggername)
      self.servername = servername
      self.record_limit_relative = record_limit_relative
      
      self.xff_output_order = []
      self.ip_records = {}
      
      self.current_time = time.time
      self.peer_uptodate = None
      self.rehash_worked = None
      self.rehash_timestamp = None
     
   def __getinitargs__(self):
      return ((self.servername, self.host, self.record_limit_relative))
   
   def __getstate__(self):
      #Return state information supposed to be pickled.
      #We'll remove external functions, file-like objects and their containers.
      state = self.__dict__.copy()
      if ('logger' in state):
         del(state['logger'])
      if ('urllib2_od' in state):
         del(state['urllib2_od'])

      return state
      
   def integrity_check(self):
      for xff in self.xff_output_order:
         self.record_verify_xff(xff)
         self.record_verify_length(xff)

   def record_verify_length(self, xff):
      if (len(self.ip_records[xff]) > self.record_limit_relative):
         self.ip_records[xff] = self.ip_records[xff][:self.record_limit_relative]

   def record_verify_xff(self, xff):
      if not (xff in self.ip_records):
         self.ip_records[xff] = []

      if not (xff in self.xff_output_order):
         self.xff_output_order.append(xff)
      
   def ip_records_update(self, xff, ts_origin, ts_expire, ip, ts_expire_reliable=True):
      self.logger.log(10, 'IP_RECORDS_UPDATE called: xff: "%s" ts_origin: "%s"(%s) ts_expire: "%s"(%s) ip: "%s" ts_expire_reliable: "%s"' % (xff, data_formatting.seconds_hr_absolute(ts_origin), ts_origin, data_formatting.seconds_hr_absolute(ts_expire), ts_expire, ip, ts_expire_reliable))
      self.record_verify_xff(xff)
      if (ts_origin > self.current_time()):
         self.logger.log(40, 'Got an ip record with origin timestamp in the future: %s. Adjusting it to current system time. Server: %s xff: %d Ip: %s origin timestamp: %f expire timestamp: %f' % (data_formatting.seconds_hr_absolute(ts_origin), self.servername, xff, ip, ts_origin, ts_expire))
         ts_origin = self.current_time()

      if ((len(self.ip_records[xff]) > 0) and (self.ip_records[xff][0][0] == ip)):
         #The newest entry known for this xff already contains this ip.
         #We'll try to update it reasonably.
         if (self.ip_records[xff][0][1] > ts_origin):
            #The origin-time of the new record is smaller. Overwrite the old value with it.
            self.ip_records[xff][0][1] = ts_origin

         if ((ts_expire_reliable) and (self.ip_records[xff][0][2] < ts_expire)):
            #The expiration-timestamp of the new record is bigger. Overwrite the old one with it.
            self.ip_records[xff][0][2] = ts_expire

      else:
         #Either there are no entries for this xff, or the newest one differs from the input.
         #Sync with older origin and expire timestamps if we already know this ip, and insert it on the
         #beginning of the list.
         
         current_time = self.current_time()
         
         if (len(self.ip_records[xff]) > 0):
            ts_origin_old = self.ip_records[xff][0][1]
            ts_expire_old = self.ip_records[xff][0][2]

            if (ts_origin_old > current_time):
               self.logger.log(45, 'The last existing record for xff %d has a origin timestamp higher than current system time: %s. Setting it to zero; I suggest you investigate the cause (changes to system clock?).' % (xff, time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(ts_origin_old))))
               ts_origin_old = self.ip_records[xff][0][1] = 0

         else:
            ts_origin_old = 0
            ts_expire_old = 0

         if (ts_origin < ts_origin_old):
            self.logger.log(20, 'Not using new ip %s for xff %d because it is older than the latest one we have on record.' % (str(ip), xff))
         else:
            for dict_xff in self.ip_records:
               if ((len(self.ip_records[dict_xff]) > 0) and (self.ip_records[dict_xff][0][0] == ip) and (self.ip_records[dict_xff][0][2] > ts_expire)):
                  ts_expire = self.ip_records[dict_xff][0][2]

            self.logger.log(10, 'Adding record to ip_record list for xff %s: %s' % (xff, [ip, ts_origin, ts_expire]))
            self.ip_records[xff].insert(0, [ip, ts_origin, ts_expire])
            self.record_verify_length(xff)
            self.xff_output_order.sort()
            self.logger.log(10, 'Result of ip_records_update_call for ip_records for server %s and xff %s: %s' % (self.servername, xff, self.ip_records))
            return True

      self.logger.log(10, 'Result of ip_records_update_call for ip_records for server %s and xff %s: %s' % (self.servername, xff, self.ip_records))
               
   def xff_next_set(self, xff):
      if not (xff in self.xff_output_order):
         raise ValueError('Xff %d currently has no output position.' % (xff))

      if not (self.xff_output_order[0] == xff):
         self.xff_output_order.remove(xff)
         self.xff_output_order.insert(0, xff)

   def xff_rotate(self):
      self.xff_output_order = self.xff_output_order[1:] + self.xff_output_order[:1]
      
   def xff_clean_up(self):
      for xff in self.xff_output_order:
         if not (xff in self.ip_records):
            self.logger.log(30, 'Xff %s in xff_output_order does not exist in ip_records. Removing it.' % (xff,))
            self.xff_output_order.remove(xff)
         elif (len(self.ip_records[xff]) < 1):
            self.xff_output_order.remove(xff)
         elif (self.ip_records[xff][0][2] < time.time()):
            #record expired
            self.xff_output_order.remove(xff)
      
   def get_ip_record(self, authority=False):
      if (len(self.xff_output_order) > 0):
         xff = self.xff_output_order[0]
         if ((len(self.ip_records[xff])) > 0):
            self.logger.log(10, 'Got query for ip records; answering with ip %s.' % (self.ip_records[xff][0][0],))
            return self.ip_records[xff][0][:]
         else:
            self.logger.log(20, "Can't answer query for ip records.")

      return None

   def xff_choose(self):
      if (len(self.xff_output_order) > 0):
         return self.xff_output_order[0]
      else:
         assert (len(self.ip_records.keys()) == 0)

   def rehash(self):
      if not (self.servername in rehash_data):
         self.logger.log(40, "No rehash data for server %s we are supposed to rehash. Assuming it doesn't need rehashes." % (self.servername,))
         self.rehash_result_succeed()
      elif (rehash_data[self.servername][0] == True):
         self.logger.log(20, "Server %s doesn't need rehashes; ignoring request." % (self.servername,))
         self.rehash_result_succeed()
      elif (rehash_data[self.servername][0] == False):
         self.logger.log(20, "Server %s cannot be rehashed; ignoring request." % (self.servername,))
         self.rehash_result_fail()
      elif(rehash_data[self.servername][0] in ('http', 'spawn_process')):
         target_host_ip_record = self.get_ip_record()
         if not (target_host_ip_record):
            self.logger.log(20, "Can't rehash server %s because we don't have any usable ip records." % (self.servername,))
            return False
         
         target_host = target_host_ip_record[0]
         if (target_host == None):
            target_host = self.host
            
         if (rehash_data[self.servername][0] == 'http'):
            auth_data = rehash_data[self.servername][2]
            
            url = 'http://%s%s' % (target_host, rehash_data[self.servername][1])

            self.logger.log(20, 'Trying to rehash server %s with %s.' % (self.servername, url))

            od = urllib2.build_opener()
            if ((auth_data != None) and (len(auth_data) == 2)):
               password_manager = urllib2.HTTPPasswordMgrWithDefaultRealm()
               password_manager.add_password(None, target_host, auth_data[0], auth_data[1])
               auth_handler = urllib2.HTTPBasicAuthHandler(password_manager)
               od.add_handler(auth_handler)
                  
            thread = threading.Thread(group=None, target=self.rehash_execute_http, name=None, args=(), kwargs={'url':url, 'od':od})
         
         elif (rehash_data[self.servername][0] == 'spawn_process'):
            command = rehash_data[self.servername][1]
            #Evaluating the argumentsequence allows to use variables accessible here in it.
            arguments = eval(rehash_data[self.servername][2])
            thread = threading.Thread(group=None, target=self.rehash_execute_spawn_process, name=None, args=(), kwargs={'command':command, 'arguments':arguments})
            
         thread.setDaemon(True)
         thread.start()
         
   def rehash_result_fail_asynchron(self, method_string):
      Timer(interval=-1000, function=self.rehash_result_fail, parent=self)
      self.logger.log(20, 'Failed to rehash %s by %s.' % (self.servername, method_string), exc_info=True)

   def rehash_result_succeed_asynchron(self, method_string):
      Timer(interval=-1000, function=self.rehash_result_succeed, parent=self)
      self.logger.log(20, 'Successfully rehashed %s by %s.' % (self.servername, method_string))

   def rehash_result_fail(self):
      #self.rehash_timestamp = time.time()
      self.rehash_worked = False
      
   def rehash_result_succeed(self):
      self.peer_uptodate = True
      self.rehash_worked = True
     # self.rehash_timestamp = time.time()

   def rehash_execute_http(self, url, od):
      self.rehash_worked = None
      self.rehash_timestamp = time.time()
      try:
         od.open(url)
      except urllib2.URLError:
         failure=True
      except httplib.HTTPException:
         failure=True
      else:
         failure=False
         self.rehash_result_succeed_asynchron('accessing %s' % (url,))

      if (failure):
         self.rehash_result_fail_asynchron('accessing %s' % (url,))
         
      socket_management.pipe_notify.notify()
      
   def rehash_execute_spawn_process(self, command, arguments):
      return_value = os.spawnvp(os.P_WAIT, command, arguments)
      self.rehash_worked = None
      self.rehash_timestamp = time.time()
      if (return_value == 0):
         self.rehash_result_succeed_asynchron('executing %s' % (' '.join([command] + arguments[1:])))
      else:
         self.rehash_result_fail_asynchron('executing %s' % (' '.join([command] + arguments[1:])))


class iptracker(input_handlers.handlers_user, core_base):
   def __init__(self, irc_server=None, irc_client=None, botnet_interface=None, dgram_socket=None, record_limit_relative=10, maintenance_delay=600, irc_client_oper_arguments=None, SETIP_usage=False):
      self.loggername = 'iptracker'
      self.logger = logging.getLogger(self.loggername)

      self.maintenance_delay = maintenance_delay
      self.record_limit_relative = record_limit_relative
      self.irc_client_oper_arguments = irc_client_oper_arguments
      self.SETIP_usage = SETIP_usage

      self.connections = {
         'irc_server':irc_server,
         'irc_client':irc_client,
         'botnet_interface':botnet_interface
         }

      self.handler_bindings = {
         'botnet_interface':[
                            ['bot_message', re.compile('''^[GS]ETIP$''', re.IGNORECASE), self.botnet_input_handler, []],
                            ['bot_broadcast', re.compile('''^[GS]ETIP$''', re.IGNORECASE), self.botnet_input_handler, []],
                            ],
         'irc_client':[
                      ['statechange_connection', 'up', self.irc_client_init_handler, []],
                      ['snotice', True, self.irc_client_snotice_handler, []],
                      ['command', 'GETIP', self.irc_client_GETIP_handler, []]
                      ]
         }

      core_base.__init__(self, irc_server=irc_server, irc_client=irc_client, botnet_interface=botnet_interface)

      if ('irc_client' in self.connections):
         self.connections['irc_client'].connection_init()
         
      self.servers = {}

      self.handlers_modify(action='register')

      self.dns_server = dns_server.dns_server(resolveprocedure=self.query_handle_dns)
      if (type(dgram_socket) == socket._socketobject):
         self.dns_server.start_listening_udp(dgram_socket=dgram_socket)

      self.current_time = time.time

      self.refresh_timer = Timer(self.maintenance_delay, function=self.maintenance_perform, persistence=True)
      self.refresh_data_servers()
      
   def __getstate__(self):
      return {'servers':self.servers}
   
   def uplink_return(self):
      if ('irc_server' in self.connections):
         return self.connections['irc_server'].uplink_data['name']
      else:
         return None   
   
   def servers_match(self, regexp):
      return [servername for servername in self.servers.keys() if (regexp.match(servername))]
      
   def server_linked(self, servername):
      if ('irc_server' in self.connections):
         return self.connections['irc_server'].server_in_network(servername)
      else:
         return None
      
   def refresh_data_server(self, server):
      if ('botnet_interface' in self.connections):
         self.connections['botnet_interface'].send_command(command='bot_zapfbroad', data='GETIP ' + server)

      self.query_start_dns(server=server)

   def refresh_data_servers(self, servers=None):
      if (servers == None):
         servers = self.servers.keys()

      for server in servers:
         self.refresh_data_server(server=server)

   def maintenance_perform(self):
      for servername in self.servers:
         self.servers[servername].xff_clean_up()
         if (self.server_linked(servername) == False):
            self.refresh_data_server(servername)
         else:
            self.query_start_dns(server=servername)
         
   def irc_client_init_handler(self, parent, output, event_type, event, permissions, data):
      if ((event_type == 'statechange_connection') and (event == 'up') and (self.irc_client_oper_arguments)):
         output(command='OPER', arguments=self.irc_client_oper_arguments)
         
   def irc_client_snotice_handler(self, parent, output, event_type, event, permissions, data):
      if ('event2_type' in data):
         event2_type = data['event2_type']
         if (event2_type in ('connect_out_timeout', 'connect_out_fail')):
            servername = data['event2_servername']
            if ((network_connection) and (servername in self.servers)):
               self.logger.log(20, 'Unable to connect to %s with currently preferred ip. Switching.' % (servername,))
               self.xff_rotate(servername)
               self.query_start_dns(server=servername)
               
         elif (event2_type == 'connect_success'):
            if (('event2_servername' in data) and ('event2_uplink') in data):
               event2_servername, event2_uplink = data['event2_servername'], data['event2_uplink']
               
               uplink = self.uplink_return()
               if (event2_servername == uplink):
                  peername = event2_uplink
               elif (event2_uplink == uplink):
                  peername = event2_servername
               else:
                  peername = None
               
               if (peername != None):
                  self.ip_confirm(peername)
               if (peername in self.servers and not self.servers[peername].peer_uptodate):
                  self.logger.log(20, "We have linked to %s; setting it's up-to-date status to true." % (peername,))
                  self.servers[peername].peer_uptodate = True
               
         elif (event2_type == 'connect_out_rejected_ip'):
            if('error_servername' in data):
               peername = data['error_servername']
               if (peername in self.servers):
                  self.servers[peername].peer_uptodate = False
                  self.logger.log(30, 'Server %s appears to be in need of a rehash; trying to provide it.' % (peername,))
                  self.servers[peername].rehash()
                  
                  self.ip_confirm(peername)
                  
         elif (event2_type == 'connect_in_denied_ip'):
            peername = data['event2_servername2']
            if (peername in self.servers):
               self.logger.log(30, 'Connection from server %s has been denied because of unknown ip. Adding ip with xff 7.' % (peername,))
               current_time = self.current_time()
               self.ip_records_update(servername=peername, xff=7, ts_origin=current_time, ts_expire=current_time+86400, ip=data['event2_host2'], ts_expire_reliable=False)
               self.xff_rotate(peername)

   def irc_client_GETIP_handler(self, parent, output, event_type, event, permissions, data):
      command = data['command']
      arguments = data['arguments']
      if (command != 'GETIP'):
         raise ValueError('Got unexpected value for command %s (expected GETIP).' % (command,))
      
      if (len(arguments) < 3):
         self.logger.log(35, 'Got invalid GETIP request from server: Too few arguments (expected three). Dump: %s' % (data,))
         return False
      
      servername = arguments[0]
      hostname = arguments[1]
      reason = arguments[2]
      
      if (servername in self.servers):
         self.uplink_update(servername, connect=True)
      else:
         asynchronous_processing.gethostbyname(request=hostname, callback_target=self.response_handle_gethostbyname, callback_kwargs={'request_type':'GETIP', 'callback_data':servername})
         
   def ip_confirm(self, servername):
      if (servername in self.servers):
         server = self.servers[servername]
         ip_record = server.get_ip_record()
         if (ip_record):
            self.logger.log(20, 'Currently assumed ip %s for server %s with xff %d appears to be correct.' % (ip_record[0], servername, server.xff_choose()))
         else:
            self.logger.log(30, "Last submitted ip for server %s appears to be correct, but we can't currently retrieve it." % (servername,))
         
   def xff_rotate(self, servername):
      if (servername in self.servers):
         server = self.servers[servername]
         server.xff_rotate()
         if (server.peer_uptodate == False):
            server.rehash()
         self.uplink_update(servername=servername)
         return True
      else:
         return False

   def query_handle_dns(self, request_data, response_data):
      """Answers queries sent to the pseudo-dns-server.

      dns_server should handle the protocol and socket details."""

      #The dns server passes queries in the request_data-dictionary. The most
      #important element of that dictionary is probably 'questions', which is a list
      #of the queries. The first element of each query_entry is the domain-name queried
      #as a list of strings  (use '.'.join(request_data['questions'][x][0]) to get it as
      #a string in the usual notation.), the other two elements are integers, specifying
      #query type and class. The response_data-dictionary passed is valid, and will
      #cause a dns response claiming that there are no entries matching the query.
      #
      #The response_data-elements 'answers', 'authoritative_nameservers' and
      #'additional_records' are lists that can be modified in place or replaced. Any
      #entries existing in these lists when this procedure finishes execution will be
      #returned to the querying program by the pseudo-dns-server.
      #
      #valid element names for dictionaries in these lists are 'domain_name', 'data', 'ttl',
      #'type' and 'class'. For any further questions read the dns_server code and rfc1035.
      
      self.logger.log(10, 'Received dns query for ip:' + str(request_data) + '\t' + str(response_data))

      for query_entry in request_data['questions']:
         if (len(query_entry) != 3):
            self.logger.log(40, 'Got invalid query_entry (not exactly 3 elements) in dns-request. Dump: %s' % (request_data,))
         else:
            (domain_splitted, query_type, query_class) = query_entry

         if (query_type == 1):
            #"A"; Unreal uses this for a lot of hosts, including ones that aren't
            #listed in its config file. Being asked for one does not mean that it
            #has any long-term relevance.
            domain_string = '.'.join(domain_splitted)
            ip_record = self.get_ip_record(servername=domain_string, authority=True)
            if (ip_record == None):
               callback_data=response_data.copy()
               callback_data['request_sequence'] = domain_string
               asynchronous_processing.gethostbyname(request=domain_string, callback_target=self.response_handle_gethostbyname, callback_kwargs={'request_type':'DNS', 'callback_data':callback_data})
               response_data['send_response'] = False
               
            else:
               response_ip, response_origin_ts, response_expire = ip_record
               current_time = self.current_time()
               #if (response_expire > current_time):
               #   response_ttl = response_expire - current_time
               #else:
               #   response_ttl = 5
               response_ttl = 10
               
               response_data['answers'].append({
                  'domain_name':domain_splitted,
                  'data':response_ip,
                  'ttl':response_ttl
                  })

         elif (query_type == 12):
            #"PTR"; should only occur in reverse-lookups.
            if (len(domain_splitted) != 6):
               self.logger.log(35, 'Got invalid splitted domain (not exactly 6 elements) of type PTR in dns-request. Ignoring. Dump: %s' % (request_data,))
            elif (list(domain_splitted[-2:]) != ['in-addr', 'arpa']):
               self.logger.log(35, 'Got invalid splitted domain of type PTR that does not appear to be an inverse lookup (last two elements are not "in-addr" and "arpa", respectively). Ignoring. Dump: %s' % (request_data,))
            else:
             target_ip_splitted = domain_splitted[:4]
             target_ip_splitted.reverse()
             target_ip_string = '.'.join(target_ip_splitted)
             callback_data=response_data.copy()
             callback_data['request_sequence'] = domain_splitted
             callback_data['request_ip'] = target_ip_splitted
             asynchronous_processing.gethostbyaddr(request=target_ip_string, callback_target=self.response_handle_gethostbyaddr, callback_kwargs={'callback_data':callback_data})
             #Don't answer this query immediately; we'll trigger the response when we get a result.
             response_data['send_response'] = False

   def response_handle_gethostbyaddr(self, request, response, callback_data):
      request_sequence = callback_data['request_sequence']
      request_ip = callback_data['request_ip']
      response_data = callback_data
      (primary_hostname, alternative_hostnames) = response[:2]
      response_data['answers'].append({
         'domain_name':request_sequence,
         'data':primary_hostname,
         'type':12,
         })

      for alternative_hostname in alternative_hostnames:
         response_data['additional_records'].append({
            'domain_name':alternative_hostname,
            'data':request_ip,
            'type':12,
            })

      self.dns_server.output_send_response(response_data=response_data)
         
   def response_handle_gethostbyname(self, request_type, request, response, callback_data):
      if (request_type == 'GETIP'):
         if (self.SETIP_usage):
            if ('irc_client' in self.connections):
               servername = callback_data
               self.connections['irc_client'].send_output_heuristic(command='SETIP', arguments=[servername, response])
               self.connections['irc_client'].send_output_heuristic(command='CONNECT', arguments=[servername])
         else:
            self.logger.log(35, 'response_handle_gethostbyname was called with request_type GETIP while self.SETIP_usage evaluates to False. Ignoring. Callback data is: %s' % (callback_data,))

      elif (request_type == 'DNS'):
         request_sequence = callback_data['request_sequence']
         response_data = callback_data
         assert (type(response) == str)
         primary_ip = response
         response_data['answers'].append({
            'domain_name':request_sequence,
            'data':primary_ip,
            })
         
         self.dns_server.output_send_response(response_data=response_data)
         
      else:
         self.logger.log(40, 'response_handle_gethostbyname was called with an invalid argument for request_type: %s. Ignoring. Callback data is: %s' % (request_type, callback_data))

   def ip_records_update(self, servername, xff, ts_origin, ts_expire, ip, ts_expire_reliable=True):
      self.existence_server_verify(servername)
      return_data = self.servers[servername].ip_records_update(xff=xff, ts_origin=ts_origin, ts_expire=ts_expire, ip=ip, ts_expire_reliable=ts_expire_reliable)
         
      if ((not (4 in self.servers[servername].ip_records)) or (len(self.servers[servername].ip_records[4]) < 1) or (self.servers[servername].ip_records[4][0][0] != ip)):
         self.logger.log(20, 'Got new ip %s for server %s and xff %d which does not equal the newest xff 4 record for this server. Performing a dns lookup.' % (ip, servername, xff))
         self.query_start_dns(server=servername)

      if (return_data):
         self.uplink_update(servername=servername)
         self.setip_send(servername=servername, xff=xff, output=self.connections['botnet_interface'].handlers.output)
   
   def botnet_input_handler(self, parent, output, event_type, event, permissions, data, trusted=False):
      if (len(data['arguments']) > 0):
         arguments = data['arguments']
         command = arguments[0]
         if (command == 'SETIP'):
            #A valid SETIP line looks like this:
            #SETIP <servername> <ip> <age> <x-files-factor> <time_to_expire>
            #xff parameter name taken from rrdtool.
            if (len(arguments) < 6):
               self.logger.log(30, 'Received invalid SETIP command (too few arguments) over botnet_interface. Dump: ' + str(data))

            else:
               try:
                  setip_servername = arguments[1]
                  setip_ip = arguments[2]
                  setip_age = float(arguments[3])
                  setip_xff = int(arguments[4])
                  setip_expire_relative = float(arguments[5])
               except ValueError:
                  self.logger.log(30, 'Received invalid SETIP command (invalid argument) over botnet_interface. Dump: ' + str(data), exc_info=True)

               else:
                  setip_ts_origin = self.current_time() - setip_age
                  setip_ts_expire = self.current_time() + setip_expire_relative
                  
                  if ((setip_xff < 1) and not trusted):
                     self.logger.log(30, 'Received SETIP command with invalid xff %d; setting it to 2. Dump: %s ' % (setip_xff, str(data)))
                     setip_xff = 2

                  self.ip_records_update(servername=setip_servername, xff=setip_xff, ts_origin=setip_ts_origin, ts_expire=setip_ts_expire, ip=setip_ip)
                  if (setip_xff == 4):
                     self.logger.log(10, 'Received SETIP command for xff 4 and server %s. Performing dns lookup.' % (setip_servername,))
                     self.query_start_dns(server=setip_servername)

         elif (command == 'GETIP'):
            #A valid GETIP command looks like this:
            #GETIP <servername>
            if (len(arguments) < 2):
               self.logger.log(30, 'Received invalid GETIP command (too few arguments) over botnet_interface. Dump: ' + str(data))
            else:
               servername = arguments[1]
               if (servername in self.servers):
                  current_time = time.time()
                  for xff in self.servers[servername].xff_output_order:
                     self.setip_send(servername, xff, output, current_time=current_time)

   def setip_send(self, servername, xff, output, current_time=None):
      if not (current_time):
         current_time = time.time()
      
      if (self.servers[servername].ip_records[xff][0][2] > current_time):
         #Xff 0 ip records are only used internally
         if (xff < 1):
            xff_out = 1
         else:
            xff_out = xff

         #output(command='bot_zapfbroad', data='SETIP %s %s %i %i %i' % (servername, self.servers[servername].ip_records[xff][0][0], current_time - self.servers[servername].ip_records[xff][0][1], xff_out, self.servers[servername].ip_records[xff][0][2] - current_time))

         age = current_time - self.servers[servername].ip_records[xff][0][1]
         expire_ts = self.servers[servername].ip_records[xff][0][2] - current_time
         
         self.logger.log(10, 'Outputting SETIP for record of server %s and xff %s: %s ; age: %s expire_ts: %s current_time: %s(%s)' % (servername, xff, self.servers[servername].ip_records[xff][0], age, expire_ts, current_time, data_formatting.seconds_hr_absolute(current_time)))
         
         output(command='bot_zapfbroad', data='SETIP %s %s %i %i %i' % (servername, self.servers[servername].ip_records[xff][0][0], age, xff_out, expire_ts))
         
         
   def get_ip(self, servername, authority=False):
      return_value = self.get_ip_record(servername=servername, authority=authority)
      if (return_value == None):
         return return_value
      else:
         return return_value[0]

   def get_ip_record(self, servername, authority=False):
      if (servername in self.servers):
         return self.servers[servername].get_ip_record()
      else:
         return None

   def xff_choose(self, servername):
      if (servername in self.servers):
         return self.servers[servername].xff_choose()
      
   def query_start_dns(self, server):
      if (server in self.servers):
         hostname = self.servers[server].host
         if (hostname == None):
            return False
      else:
         hostname = server

      asynchronous_processing.gethostbyname(request=hostname, callback_target=self.response_handle_dns, callback_kwargs={'servername':server})

   def response_handle_dns(self, request, response, servername):
      if not (servername in self.servers):
         self.logger.log(40, "Got dns lookup result for server %s, which I don't remember tracking. What am I supposed to do with this ? (rhetorical question) ip: %s" % (servername, ip))
         return False

      ts_origin = self.current_time()
      ts_expire = ts_origin + 86400.0
      self.ip_records_update(servername=servername, xff=4, ts_origin=ts_origin, ts_expire=ts_expire, ip=response, ts_expire_reliable=False)

   def existence_server_verify(self, servername):
      if not (servername in self.servers):
         self.servers[servername] = iptrack_server(servername=servername, record_limit_relative=self.record_limit_relative)

   def uplink_update(self, servername, connect=False):
      if ('irc_client' in self.connections):
         if (self.SETIP_usage):
            server_ip = self.get_ip(servername=servername)
            if (server_ip):
               self.connections['irc_client'].send_output_heuristic(command='SETIP', arguments=[servername, server_ip])
            else:
               connect = False
         else:
            self.connections['irc_client'].send_output_heuristic(command='DNS', arguments=['c'])
            
         if (connect):
            self.connections['irc_client'].send_output_heuristic(command='CONNECT', arguments=[servername])
                  
   def integrity_check(self):
      for servername in self.servers:
         self.servers[servername].integrity_check()
      
   def clean_up(self):
      self.refresh_timer.stop()
      core_base.clean_up(self)


class iptrack_user_interface(input_handlers.handlers_user, core_base):
   def __init__(self, iptrack_instance, irc_server=None, control_connection=None):
      self.loggername = 'iptrack_user_interface'
      self.logger = logging.getLogger(self.loggername)

      self.iptrack_instance = iptrack_instance
      
      self.connections = {
         'irc_server':irc_server,
         'control_connection':control_connection
         }

      self.patterns = {
         'iptrack': re.compile('^iptrack', re.IGNORECASE),
         'disc': re.compile('^disc', re.IGNORECASE),
         'rehash': re.compile('^rehash', re.IGNORECASE),
         'setip': re.compile('^setip', re.IGNORECASE),
         'self_disconnect': re.compile('^self_disconnect', re.IGNORECASE),
         'self_reconnect': re.compile('^self_reconnect', re.IGNORECASE),
         }

      self.handler_bindings = {
         'irc_server':[
                      ['trigger', self.patterns['iptrack'], self.irc_trigger_handler, []],
                      ['trigger', self.patterns['disc'], self.irc_trigger_handler, []],
                      ['trigger', self.patterns['rehash'], self.irc_trigger_handler, ['o']],
                      ],
         'control_connection':[
            ['command', self.patterns['iptrack'], self.control_connection_trigger_handler, []],
            ['command', self.patterns['disc'], self.control_connection_trigger_handler, []],
            ['command', self.patterns['setip'], self.control_connection_trigger_handler, ['IPCONTROL']],
            ['command', self.patterns['self_disconnect'], self.control_connection_trigger_handler, ['CONNECTIONCONTROL']],
            ['command', self.patterns['self_reconnect'], self.control_connection_trigger_handler, ['CONNECTIONCONTROL']],
            ]
         }

      core_base.__init__(self, irc_server=irc_server, control_connection=control_connection, irc_client=None, botnet_interface=None)

      self.handlers_modify(action='register')

      self.irc_reply = {
         'target':'',
         'output':None
         }

      self.input_parser = {}
         
      self.input_parser = {
         'iptrack':input_handlers.input_parser(custom_output=self.optparse_error_output, program_name='iptrack', usage='%prog ACTION [options]', option_list=[
            optparse.make_option('-s', '--server', action='store', type='string', dest='server', default=None, help='Server to get info for (default:all)'),
            optparse.make_option('-x', '--xff', action='store', type='int', dest='xff', default=None, help='X-Files factor'),
            optparse.make_option('-n', '--entries', type='int', dest='entries', default=1, help='Maximum number of entries to display or manipulate per server and xff')
            ]
         ),
         'disc':input_handlers.input_parser(custom_output=self.optparse_error_output, program_name='disc', usage='%prog [server]', option_list = [
            optparse.make_option('-x', '--xff', action='store', type='int', dest='xff', default=None, help='X-Files factor to use as basis for guesstimate'),
            ]
         ),
         'rehash':input_handlers.input_parser(custom_output=self.optparse_error_output, program_name='disc', usage='%prog <server>')
         }

      self.permissions = {
         'iptrack': {
            'drop_server':['o'],
            'add_server':['o'],
            'drop_records':['o'],
            }
         }
         
      self.current_time = time.time

   def optparse_error_output(self, output_string):
      self.optparse_error_report(output_string)

   def match_event(self, patternname, event):
      self.logger.log(10, 'trying to match object %s against pattern named %s.' % (event, patternname))
      if (self.patterns[patternname].search(event) == None):
         return False
      else:
         return True

   def permissions_verify(self, required_permissions, actual_permissions):
      for permission in required_permissions:
         if (not (permission in actual_permissions)):
            raise InsufficientPermissionsError()

   def uplink_return(self):
      if (self.iptrack_instance):
         return self.iptrack_instance.uplink_return()
               
   def control_connection_trigger_handler(self, parent, output, event_type, event, permissions, data):
      if ((self.match_event('iptrack', event)) or (self.match_event('disc', event))):
         output, connection = data['output'], data['connection']
      
         def control_output(string_data):
            output(command='IPTRACK_REPLY', arguments=[string_data])
      
         self.general_trigger_handler(event=data['command'], permissions=permissions, output=control_output, arguments=data['arguments'])
         
      elif (self.match_event('setip', event)):
         self.iptrack_instance.botnet_input_handler(parent=None, output=output, event_type=event_type, event=event, permissions=permissions, data={
            'command':data['command'],
            'arguments':['SETIP'] + data['arguments']
            }, trusted=True)
            
      elif (self.match_event('self_disconnect', event)):
         network_connection = False
         self.logger.log(30, 'Our internet link just went down.')
         if (self.iptrack_instance):
            if (self.iptrack_instance.connections['irc_server'] != None):
               self.iptrack_instance.connections['irc_server'].squit_servers(scope='remote_linked')
               
            for server in self.iptrack_instance.servers.values():
               server.peer_uptodate = False
         
      elif (self.match_event('self_reconnect', event)):
         network_connection = True
         self_uplink = self.uplink_return()
         if (self_uplink):
            self_ip = self.iptrack_instance.get_ip(servername=self_uplink)
         else:
            self_ip = None
         
         if (self_ip):
            self.logger.log(30, 'Our internet link has come up. Global ip appears to be %s.' % (self_ip,))
         else:
            self.logger.log(30, 'Our internet link has come up. Global ip is unknown.')
        
         if (self.iptrack_instance):
            for server in self.iptrack_instance.servers.values():
               server.rehash()
               
         if (self.iptrack_instance.connections['irc_server']):
            for servername in self.iptrack_instance.servers.keys():
               self.iptrack_instance.connections['irc_server'].send_command(command='CONNECT', arguments=[servername])
         
   def irc_trigger_handler(self, parent, output, event_type, event, permissions, data):
      reply_target, source = data['reply_target'], data['source']
      
      def irc_output(string_data):
         output(command='PRIVMSG', arguments=[reply_target, string_data])
      
      self.general_trigger_handler(event=event, output=irc_output, permissions=permissions, arguments=data['arguments'])
         
       
   def general_trigger_handler(self, event, permissions, output, arguments):
      ipt_servers = self.iptrack_instance.servers
      self.optparse_error_report = output

      if (self.match_event('iptrack', event)):
         try:
            (input_options, input_arguments) = self.input_parser['iptrack'].parse_args(arguments)
         except ValueError:
            return

         if (len(input_arguments) < 1):
            output('Missing positional argument ACTION.')
         else:
            iptrack_action = input_arguments[0].lower()
            try:
               if (iptrack_action in self.permissions['iptrack']):
                  self.permissions_verify(required_permissions=self.permissions['iptrack'][iptrack_action], actual_permissions=permissions)
            
            except InsufficientPermissionsError:
               output("Negative on that; you don't have sufficient permissions.")
            else:
               if (iptrack_action == 'dump'):
                  def output_entries(server, xff, limit):
                     output('server \002%s\002 xff \002%d\002:' % (server, xff))
                     for data_entry in ipt_servers[server].ip_records[xff][:limit]:
                        output('%15s  TS: %s  expires: %s' % (data_entry[0], data_formatting.seconds_hr_absolute(data_entry[1]), data_formatting.seconds_hr_absolute(data_entry[2])))

                  if (input_options.server == None):
                     target_servers = ipt_servers.keys()
                  elif (input_options.server in ipt_servers):
                     target_servers = [input_options.server]
                  else:
                     target_servers = []
                     target_servers_re = re.compile('''.*''' + input_options.server + '''.*''',re.IGNORECASE)
                     target_servers = [servername for servername in ipt_servers if (target_servers_re.search(servername) != None)]

                  if (input_options.xff == None):
                     input_options.xff = 4
                     
                  for server in target_servers:
                     if (input_options.xff < 0):
                        for xff in ipt_servers[server].ip_records:
                           output_entries(server, xff, input_options.entries)

                     elif (input_options.xff in ipt_servers[server].ip_records):
                        output_entries(server, input_options.xff, input_options.entries)

               elif (iptrack_action == 'report'):
                  if (input_options.server == None):
                     if (len(input_arguments) > 0):
                        target_servers = [input_argument for input_argument in input_arguments if (input_argument in self.iptrack_instance.servers)]
                     else:
                        target_servers = [self.uplink_return()]
                  else:
                     target_servers = self.iptrack_instance.servers_match(re.compile(input_options.server, re.IGNORECASE))
            
                  if (len(target_servers) == 0):
                     output('No matching servers.')
               
                  for servername in target_servers:
                     output('servername: \002%s\002' % (servername,))
                     server = self.iptrack_instance.servers[servername]
                     output('  preferred xff: %s' % (server.xff_choose(),))
                     ip_record = server.get_ip_record()
                     if (ip_record == None):
                        output('  Unable to get ip record information.')
                     else:
                        output('  best-guess ip: %s' % (ip_record[0],))
                        output('  valid since: %s (which was %s ago)' % (data_formatting.seconds_hr_absolute(ip_record[1]), data_formatting.seconds_hr_relative(time.time() - ip_record[1])))
                        output('  estimated to expire at: %s' % (data_formatting.seconds_hr_absolute(ip_record[2]),))
                        if not (servername in rehash_data):
                           rehash_method_out_string = "no idea (uh-oh, not a good situation)"
                        else:
                           rehash_method_out_string = rehash_data[servername][0]
                           if (rehash_method_out_string == True):
                              rehash_method_out_string = "doesn't need it"
                           elif (rehash_method_out_string == False):
                              rehash_method_out_string = 'not possible (blame my admin or that of the server)'

                        output('  rehash_method: %s' % (rehash_method_out_string,))
                        output('  last rehash attempt: %s' % (data_formatting.seconds_hr_absolute(server.rehash_timestamp,)))
                        output('  last rehash attempt worked?: %s' % (server.rehash_worked,))
                        output('  server up-to-date?: %s' % (server.peer_uptodate,))
         
               elif (iptrack_action == 'list'):
                  output('Currently tracking the following servers: %s' % (' '.join(ipt_servers),))
                  
               elif (iptrack_action in ('add_server', 'drop_server', 'drop_records')):
                  if (input_options.server != None):
                     target_servername = input_options.server
                  elif (len(input_arguments) > 1):
                     target_servername = input_arguments[1]
                  else:
                     output('Error: expected at least one argument (the servername).')
                     target_servername = None
               
                  if (target_servername != None):
                     if (iptrack_action == 'add_server'):
                        if (target_servername in self.iptrack_instance.servers):
                           output('Already tracking %s.' % (target_servername,))
                        else:
                           output('Confirmed, adding %s to observed servers.' % (target_servername,))
                           self.iptrack_instance.servers[target_servername] = iptrack_server(servername=target_servername)
                           self.iptrack_instance.refresh_data_server(server=target_servername)
                     elif (iptrack_action == 'drop_server'):
                        if not (target_servername in self.iptrack_instance.servers):
                           output('Already not tracking %s.' % (target_servername,))
                        else:
                           output('Confirmed, removing %s from observed servers.' % (target_servername,))
                           del(self.iptrack_instance.servers[target_servername])
                           
                     elif (iptrack_action == 'drop_records'):
                        if (input_options.xff != None):
                           target_xff = input_options.xff
                        elif (len(input_arguments) > 2):
                           try:
                              target_xff = int(input_arguments[2])
                           except ValueError:
                              output('Invalid value %s for integer argument.' % (input_arguments[2],))
                        else:
                           output('Error: expected at least two arguments (servername and xff).')
                           target_xff = None
                           
                        if (target_xff != None):
                           if not (target_servername in self.iptrack_instance.servers):
                              output('Error: unknown server %s.' % (target_servername,))
                           elif not (target_xff in self.iptrack_instance.servers[target_servername].ip_records):
                              output("Error: We currently don't don't have any ip-records with xff %d for server %s." % (target_xff, target_servername))
                           else:
                              output('Confirmed, dropping first %d ip_records of server %s and xff %d.' % (input_options.entries, target_servername, target_xff))
                              del(self.iptrack_instance.servers[target_servername].ip_records[target_xff][:input_options.entries])
                     else:
                        raise NotImplementedError
                     
               else:
                  output("Invalid value '%s' for ACTION argument." % (iptrack_action,))

      elif (self.match_event('disc', event)):
         (input_options, input_arguments) = self.input_parser['disc'].parse_args(arguments)
         
         xff = input_options.xff

         if (len(input_arguments) < 1):
            irc_server = self.uplink_return()
         else:
            irc_server = input_arguments[0]

         if (xff == None):
            ip_record = self.iptrack_instance.get_ip_record(servername=irc_server, authority=False)
            xff = self.iptrack_instance.xff_choose(servername=irc_server)
            
         elif ((irc_server in self.iptrack_instance.servers) and (xff in self.iptrack_instance.servers[irc_server].ip_records) and (len(self.iptrack_instance.servers[irc_server].ip_records[xff]) > 0)):
            ip_record = self.iptrack_instance.servers[irc_server].ip_records[xff][0]
         else:
            ip_record = None

         if (ip_record == None):
            output('Unable to estimate time to desynch for server %s.' % (irc_server,))
            
         else:
            expire_ts = ip_record[2]
            current_time = self.current_time()

            if (expire_ts > current_time):
               output('Next desynch of %s according to data for xff %d should happen approximately at %s, which is in %s.' % (irc_server, xff, data_formatting.seconds_hr_absolute(expire_ts), data_formatting.seconds_hr_relative(expire_ts-current_time, 1)))
            else: 
               output('Desynch of %s according to data for xff %d should have happened around %s, which was %s ago.' % (irc_server, xff, data_formatting.seconds_hr_absolute(expire_ts), data_formatting.seconds_hr_relative(current_time-expire_ts, 1)))

      elif (self.match_event('rehash', event)):
         (input_options, input_arguments) = self.input_parser['rehash'].parse_args(arguments)
         if (len(input_arguments) < 1):
            output('Error: expected at least one argument (the servername).')
         else:
            servername = input_arguments[0]
            if (not servername in self.iptrack_instance.servers):
               output('Error: unknown server %s.' % (servername,))
            elif not (servername in rehash_data):
               output('Error: no rehash method entry for server %s present.' % (servername,))
            else:
               if (rehash_data[servername][0] == True):
                  output("Negative on that; server %s doesn't need manual rehashes." % (servername,))
               elif (rehash_data[servername][0] == False):
                  output('Error: server %s cannot currently be rehashed.' % (servername,))
               else:
                  self.iptrack_instance.servers[servername].rehash()
                  output('Confirmed. Trying to rehash server %s with method %s.' % (servername, rehash_data[servername][0]))
            
      else:
         self.logger.log(40, 'Unknown event passed to irc_trigger_handler. event_type: %s, event: %s, data: %s' % (str(event_type), str(event), str(data)))