Refactor io to separate process. Add tests
This commit is contained in:
Alec Leamas
parent
5144523cfd
commit
b128e3bcc6
|
|
@ -8,6 +8,7 @@ Dependencies
|
|||
------------
|
||||
- python-twisted (tested with 12.1)
|
||||
- supybot (tested with 0.83.4)
|
||||
- ncat for unit tests
|
||||
|
||||
Getting started
|
||||
---------------
|
||||
|
|
@ -153,4 +154,8 @@ pylint: (in the Git directory):
|
|||
```
|
||||
$ pylint --rcfile pylint.conf \*.py > pylint.log
|
||||
```
|
||||
Unit tests are currently not in place.
|
||||
Unit tests:
|
||||
```
|
||||
$ supybot-test plugins/Irccat
|
||||
```
|
||||
|
||||
|
|
|
|||
152
plugin.py
152
plugin.py
|
|
@ -29,9 +29,28 @@
|
|||
# https://chris-lamb.co.uk/posts/irccat-plugin-supybot
|
||||
# http://www.jibble.org/pircbot.php
|
||||
|
||||
''' Main plugin module. See README for usage and configuration. '''
|
||||
'''
|
||||
Main plugin module. See README for usage and configuration.
|
||||
|
||||
Here is two processes, the main process and the io_process.
|
||||
The main process has a separate listener_thread.
|
||||
|
||||
The io_process gets data from a port and forwards it to the
|
||||
main process. The main process handles user commands. A separate
|
||||
thread gets data from the io_process and forwards to irc.
|
||||
|
||||
Somewhat messy. Design effected by need to run twisted in a process
|
||||
so it vcan be restarted, and that the irc state can't be shared
|
||||
i. e., the separate process can't shuffle data to irc.
|
||||
|
||||
Here is no critical zones, this is pure message passing. The
|
||||
io_process gets updated configurations from main. Main gets data
|
||||
to print from io_process.
|
||||
'''
|
||||
|
||||
import multiprocessing
|
||||
import pickle
|
||||
import sys
|
||||
import time
|
||||
|
||||
from twisted.internet import reactor, protocol
|
||||
|
|
@ -40,28 +59,36 @@ from twisted.protocols import basic
|
|||
from supybot import callbacks
|
||||
from supybot import ircmsgs
|
||||
from supybot import log
|
||||
from supybot import world
|
||||
from supybot.commands import commalist
|
||||
from supybot.commands import threading
|
||||
from supybot.commands import wrap
|
||||
|
||||
import config
|
||||
|
||||
|
||||
_HELP_URL = "https://github.com/leamas/supybot-irccat"
|
||||
|
||||
|
||||
class _Section(object):
|
||||
''' Section representation in _data. '''
|
||||
def io_process(port, pipe):
|
||||
''' Run the twisted-governed data flow from port -> irc. '''
|
||||
# pylint: disable=E1101
|
||||
|
||||
def __init__(self, password, channels):
|
||||
self.password = password
|
||||
self.channels = channels
|
||||
logger = log.getPluginLogger('irccat.io')
|
||||
logger.debug("Starting IO process on %d" % port)
|
||||
reactor.listenTCP(port, IrccatFactory(pipe))
|
||||
try:
|
||||
reactor.run()
|
||||
except Exception as ex: # pylint: disable=W0703
|
||||
logger.error("Exception in io_process: " + str(ex), exc_info = True)
|
||||
logger.info(" io_process: exiting")
|
||||
|
||||
|
||||
class _Blacklist(object):
|
||||
''' Handles blacklisting of aggressive clients. '''
|
||||
''' Handles blacklisting of faulty clients. '''
|
||||
|
||||
FailMax = 4 # Max # of times
|
||||
BlockTime = 20 # Time we wait in blacklisted state (seconds).
|
||||
FailMax = 8 # Max # of times
|
||||
BlockTime = 500 # Time we wait in blacklisted state (seconds).
|
||||
|
||||
def __init__(self):
|
||||
self._state = {}
|
||||
|
|
@ -95,23 +122,31 @@ class _Blacklist(object):
|
|||
return False
|
||||
|
||||
|
||||
class _Section(object):
|
||||
''' Section representation in _Config._data. '''
|
||||
|
||||
def __init__(self, password, channels):
|
||||
self.password = password
|
||||
self.channels = channels
|
||||
|
||||
|
||||
class _Config(object):
|
||||
''' Persistent stored, critical zone section data. '''
|
||||
''' Persistent stored section data. '''
|
||||
|
||||
def __init__(self):
|
||||
self.log = log.getPluginLogger('irccat.config')
|
||||
self._lock = threading.Lock()
|
||||
self._path = config.global_option('sectionspath').value
|
||||
self.port = config.global_option('port').value
|
||||
self._path = config.global_option('sectionspath').value
|
||||
try:
|
||||
self._data = pickle.load(open(self._path))
|
||||
except IOError:
|
||||
self._data = {}
|
||||
self.log.warning("Can't find stored config, creating empty.")
|
||||
logger = log.getPluginLogger('irccat.config')
|
||||
logger.warning("Can't find stored config, creating empty.")
|
||||
self._dump()
|
||||
except Exception: # Unpickle throws just anything.
|
||||
self._data = {}
|
||||
self.log.warning("Bad stored config, creating empty.")
|
||||
logger = log.getPluginLogger('irccat.config')
|
||||
logger.warning("Bad stored config, creating empty.")
|
||||
self._dump()
|
||||
|
||||
def _dump(self):
|
||||
|
|
@ -120,26 +155,22 @@ class _Config(object):
|
|||
|
||||
def get(self, section_name):
|
||||
''' Return (password, channels) tuple or raise KeyError. '''
|
||||
with self._lock:
|
||||
s = self._data[section_name]
|
||||
s = self._data[section_name]
|
||||
return s.password, s.channels
|
||||
|
||||
def update(self, section_name, password, channels):
|
||||
''' Store section data for name, creating it if required. '''
|
||||
with self._lock:
|
||||
self._data[section_name] = _Section(password, channels)
|
||||
self._dump()
|
||||
self._data[section_name] = _Section(password, channels)
|
||||
self._dump()
|
||||
|
||||
def remove(self, section_name):
|
||||
''' Remove existing section or raise KeyError. '''
|
||||
with self._lock:
|
||||
del(self._data[section_name])
|
||||
self._dump()
|
||||
del(self._data[section_name])
|
||||
self._dump()
|
||||
|
||||
def keys(self):
|
||||
''' Return list of section names. '''
|
||||
with self._lock:
|
||||
return list(self._data.keys())
|
||||
return list(self._data.keys())
|
||||
|
||||
|
||||
class IrccatProtocol(basic.LineOnlyReceiver):
|
||||
|
|
@ -147,15 +178,14 @@ class IrccatProtocol(basic.LineOnlyReceiver):
|
|||
|
||||
delimiter = '\n'
|
||||
|
||||
def __init__(self, irc, config_, blacklist):
|
||||
self.irc = irc
|
||||
def __init__(self, config_, blacklist, msg_conn):
|
||||
self.config = config_
|
||||
self.blacklist = blacklist
|
||||
self.log = log.getPluginLogger('irccat.protocol')
|
||||
self.msg_conn = msg_conn
|
||||
self.peer = None
|
||||
self.log = log.getPluginLogger('irccat.protocol')
|
||||
|
||||
def connectionMade(self):
|
||||
# if blacklisted: self.transport.abortConnection()
|
||||
self.peer = self.transport.getPeer()
|
||||
if self.blacklist.onList(self.peer.host):
|
||||
self.transport.abortConnection()
|
||||
|
|
@ -167,10 +197,12 @@ class IrccatProtocol(basic.LineOnlyReceiver):
|
|||
''' Handle one line of input from client. '''
|
||||
|
||||
def warning(what):
|
||||
''' Log a warning about bad input. '''
|
||||
''' Log and register bad input warning. '''
|
||||
if self.peer:
|
||||
what += ' from: ' + str(self.peer.host)
|
||||
self.log.warning(what)
|
||||
if world.testing:
|
||||
self.msg_conn.send((what, ['#test']))
|
||||
self.blacklist.register(self.peer.host, False)
|
||||
|
||||
try:
|
||||
|
|
@ -188,21 +220,24 @@ class IrccatProtocol(basic.LineOnlyReceiver):
|
|||
return
|
||||
if not channels:
|
||||
warning('Empty channel list: ' + section)
|
||||
for channel in channels:
|
||||
self.irc.queueMsg(ircmsgs.notice(channel, data))
|
||||
self.log.debug("Sending " + data + " to: " + str(channels))
|
||||
self.msg_conn.send((data, channels))
|
||||
self.blacklist.register(self.peer.host, True)
|
||||
|
||||
|
||||
class IrccatFactory(protocol.Factory):
|
||||
''' Twisted factory producing a Protocol using buildProtocol. '''
|
||||
|
||||
def __init__(self, irc, config_):
|
||||
self.irc = irc
|
||||
self.config = config_
|
||||
def __init__(self, pipe):
|
||||
self.pipe = pipe
|
||||
self.blacklist = _Blacklist()
|
||||
assert self.pipe[0].poll(), "No initial config!"
|
||||
self.config = self.pipe[0].recv()
|
||||
|
||||
def buildProtocol(self, addr):
|
||||
return IrccatProtocol(self.irc, self.config, self.blacklist)
|
||||
if self.pipe[0].poll():
|
||||
self.config = self.pipe[0].recv()
|
||||
return IrccatProtocol(self.config, self.blacklist, self.pipe[0])
|
||||
|
||||
|
||||
class Irccat(callbacks.Plugin):
|
||||
|
|
@ -220,19 +255,46 @@ class Irccat(callbacks.Plugin):
|
|||
|
||||
def __init__(self, irc):
|
||||
callbacks.Plugin.__init__(self, irc)
|
||||
self.irc = irc
|
||||
self.log = log.getPluginLogger('irccat.irccat')
|
||||
self.config = _Config()
|
||||
self.server = reactor.listenTCP(self.config.port,
|
||||
IrccatFactory(irc, self.config))
|
||||
self.thread = \
|
||||
threading.Thread(target = reactor.run,
|
||||
kwargs = {'installSignalHandlers': False})
|
||||
|
||||
self.pipe = multiprocessing.Pipe()
|
||||
self.pipe[1].send(self.config)
|
||||
self.process = multiprocessing.Process(
|
||||
target = io_process,
|
||||
args = (self.config.port, self.pipe))
|
||||
self.process.start()
|
||||
|
||||
self.listen_abort = False
|
||||
self.thread = threading.Thread(target = self.listener_thread)
|
||||
self.thread.start()
|
||||
|
||||
def die(self):
|
||||
def listener_thread(self):
|
||||
''' Take messages from process, write them to irc.'''
|
||||
while not self.listen_abort:
|
||||
try:
|
||||
if not self.pipe[1].poll(0.5):
|
||||
continue
|
||||
msg, channels = self.pipe[1].recv()
|
||||
for channel in channels:
|
||||
self.irc.queueMsg(ircmsgs.notice(channel, msg))
|
||||
except EOFError:
|
||||
self.listen_abort = True
|
||||
except Exception:
|
||||
self.log.debug("LISTEN: Exception", exc_info = True)
|
||||
self.listen_abort = True
|
||||
self.log.debug("LISTEN: exiting")
|
||||
|
||||
def die(self, cmd = False): # pylint: disable=W0221
|
||||
''' Tear down reactor thread and die. '''
|
||||
reactor.callFromThread(reactor.stop)
|
||||
|
||||
self.log.debug("Dying...")
|
||||
self.process.terminate()
|
||||
self.listen_abort = True
|
||||
self.thread.join()
|
||||
callbacks.Plugin.die(self)
|
||||
if not cmd:
|
||||
callbacks.Plugin.die(self)
|
||||
|
||||
def sectiondata(self, irc, msg, args, section_name, password, channels):
|
||||
""" <section name> <password> <channel[,channel...]>
|
||||
|
|
@ -243,6 +305,7 @@ class Irccat(callbacks.Plugin):
|
|||
"""
|
||||
|
||||
self.config.update(section_name, password, channels)
|
||||
self.pipe[1].send(self.config)
|
||||
irc.replySuccess()
|
||||
|
||||
sectiondata = wrap(sectiondata, [admin,
|
||||
|
|
@ -261,6 +324,7 @@ class Irccat(callbacks.Plugin):
|
|||
except KeyError:
|
||||
irc.reply("Error: no such section")
|
||||
return
|
||||
self.pipe[1].send(self.config)
|
||||
irc.replySuccess()
|
||||
|
||||
sectionkill = wrap(sectionkill, [admin, 'somethingWithoutSpaces'])
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ load-plugins=
|
|||
# can either give multiple identifier separated by comma (,) or put this option
|
||||
# multiple time (only on the command line, not in the configuration file where
|
||||
# it should appear only once).
|
||||
disable=R0903,W0141,W0232,E1103,I0011,W0311,W0403,W0613,R0201
|
||||
disable=R0903,W0141,W0232,E1103,I0011,W0311,W0403,W0613,R0201,C0103
|
||||
|
||||
#E1103: *%s %r has no %r member (but some types could not be inferred)*
|
||||
# Adds noise like "type list has no member split()", not useful
|
||||
|
|
@ -50,6 +50,7 @@ disable=R0903,W0141,W0232,E1103,I0011,W0311,W0403,W0613,R0201
|
|||
#W0613: Unused arguments warning. Will always happen here due to protyping
|
||||
# in plugin methods.
|
||||
#R0201: Method could be a function. Also happens due to plugin prototyping.
|
||||
#C0103: Invalid name. Supybot's camelCase style is flagged as bad.
|
||||
|
||||
|
||||
[REPORTS]
|
||||
|
|
|
|||
123
test.py
123
test.py
|
|
@ -33,16 +33,131 @@
|
|||
# Missing docstrings:
|
||||
# pylint: disable=C0111
|
||||
# supybot's typenames are irregular
|
||||
# pylint: disable=C0103
|
||||
# Too many public methods:
|
||||
# pylint: disable=R0904
|
||||
|
||||
|
||||
import os
|
||||
import os.path
|
||||
import subprocess
|
||||
|
||||
from supybot.test import *
|
||||
|
||||
|
||||
class IrccatTestCase(PluginTestCase):
|
||||
plugins = ('Irccat',)
|
||||
import config
|
||||
import plugin as irccat
|
||||
|
||||
|
||||
def clear_sections(testcase):
|
||||
if os.path.exists('test-sections.pickle'):
|
||||
os.unlink('test-sections.pickle')
|
||||
config.global_option('sectionspath').setValue('test-sections.pickle')
|
||||
config.global_option('port').setValue(23456)
|
||||
|
||||
|
||||
class IrccatTestList(PluginTestCase):
|
||||
plugins = ('Irccat', 'User')
|
||||
|
||||
def setUp(self, nick='test'): # pylint: disable=W0221
|
||||
clear_sections(self)
|
||||
PluginTestCase.setUp(self)
|
||||
self.assertNotError('reload Irccat')
|
||||
self.assertNotError('register suptest suptest')
|
||||
self.assertNotError('sectiondata ivar ivar #al-bot-test')
|
||||
|
||||
def testList(self):
|
||||
self.assertResponse('sectionlist', 'ivar')
|
||||
|
||||
def testReload(self):
|
||||
self.assertResponse('reload Irccat', 'The operation succeeded.')
|
||||
|
||||
|
||||
class IrccatTestCopy(ChannelPluginTestCase):
|
||||
plugins = ('Irccat', 'User')
|
||||
channel = '#test'
|
||||
cmd_tmpl = "echo '%s' | nc --send-only localhost 23456"
|
||||
|
||||
def setUp(self, nick='test'): # pylint: disable=W0221
|
||||
clear_sections(self)
|
||||
ChannelPluginTestCase.setUp(self)
|
||||
self.assertNotError('reload Irccat', private = True)
|
||||
self.assertNotError('register suptest suptest', private = True)
|
||||
self.assertNotError('sectiondata ivar ivarpw #test', private = True)
|
||||
|
||||
def testCopy(self):
|
||||
cmd = self.cmd_tmpl % 'ivar;ivarpw;ivar data'
|
||||
subprocess.check_call(cmd, shell = True)
|
||||
result = self.getMsg(' ')
|
||||
self.assertEqual(result.args[1], 'ivar data')
|
||||
|
||||
def testBadFormat(self):
|
||||
cmd = self.cmd_tmpl % 'ivar;ivarpw data'
|
||||
subprocess.check_call(cmd, shell = True)
|
||||
result = self.getMsg(' ')
|
||||
self.assertTrue(result.args[1].startswith('Illegal format'))
|
||||
|
||||
def testBadPw(self):
|
||||
cmd = self.cmd_tmpl % 'ivar;ivarpw22;ivar data'
|
||||
subprocess.check_call(cmd, shell = True)
|
||||
result = self.getMsg(' ')
|
||||
self.assertTrue(result.args[1].startswith('Bad password'))
|
||||
|
||||
def testBadSection(self):
|
||||
cmd = self.cmd_tmpl % 'ivaru22;ivarpw22;ivar data'
|
||||
subprocess.check_call(cmd, shell = True)
|
||||
result = self.getMsg(' ')
|
||||
self.assertTrue(result.args[1].startswith('No such section'))
|
||||
|
||||
|
||||
class IrccatTestData(PluginTestCase):
|
||||
plugins = ('Irccat', 'User')
|
||||
|
||||
def setUp(self, nick='test'): # pylint: disable=W0221
|
||||
clear_sections(self)
|
||||
PluginTestCase.setUp(self)
|
||||
self.assertNotError('reload Irccat')
|
||||
self.assertNotError('sectiondata ivar ivar #al-bot-test')
|
||||
self.assertNotError('sectiondata yngve yngve #al-bot-test')
|
||||
|
||||
def testList(self):
|
||||
self.assertResponse('sectionlist', 'yngve ivar')
|
||||
|
||||
def testReload(self):
|
||||
self.assertResponse('reload Irccat', 'The operation succeeded.')
|
||||
|
||||
def testShow(self):
|
||||
self.assertResponse('sectionshow yngve', 'yngve #al-bot-test')
|
||||
|
||||
def testKill(self):
|
||||
self.assertNotError('sectionkill yngve')
|
||||
self.assertResponse('sectionlist', 'ivar')
|
||||
|
||||
def testKillBadSection(self):
|
||||
self.assertResponse('sectionkill tore', 'Error: no such section')
|
||||
|
||||
|
||||
class BlacklistTest(SupyTestCase):
|
||||
|
||||
def setUp(self):
|
||||
SupyTestCase.setUp(self)
|
||||
self.blacklist = None
|
||||
|
||||
def testBlock(self):
|
||||
self.blacklist = irccat._Blacklist() # pylint: disable=W0212
|
||||
self.blacklist.FailMax = 5
|
||||
self.blacklist.BlockTime = 0.2
|
||||
|
||||
host = '132.132.132.132'
|
||||
self.assertFalse(self.blacklist.onList(host))
|
||||
for i in [1, 2, 3, 4]: # pylint: disable=W0612
|
||||
self.blacklist.register(host, False)
|
||||
self.assertFalse(self.blacklist.onList(host))
|
||||
self.blacklist.register(host, False)
|
||||
self.assertTrue(self.blacklist.onList(host))
|
||||
time.sleep(0.25)
|
||||
self.assertFalse(self.blacklist.onList(host))
|
||||
for i in [1, 2, 3, 4, 5]:
|
||||
self.blacklist.register(host, False)
|
||||
self.assertTrue(self.blacklist.onList(host))
|
||||
|
||||
#
|
||||
# vim:set shiftwidth=4 tabstop=4 expandtab textwidth=79:
|
||||
|
|
|
|||
Loading…
Reference in New Issue