diff --git a/msrplib/protocol.py b/msrplib/protocol.py index 43922bc..51e2791 100644 --- a/msrplib/protocol.py +++ b/msrplib/protocol.py @@ -1,777 +1,815 @@ # Copyright (C) 2008-2012 AG Projects. See LICENSE for details import random import re from application.system import host as host_module from collections import deque, namedtuple from gnutls.interfaces.twisted import X509Credentials from twisted.internet.protocol import connectionDone from twisted.protocols.basic import LineReceiver from msrplib import MSRPError class ParsingError(MSRPError): pass class HeaderParsingError(ParsingError): def __init__(self, header, reason): self.header = header ParsingError.__init__(self, 'Error parsing {} header: {}'.format(header, reason)) # Header value data types (for decoded values) # ByteRange = namedtuple('ByteRange', ['start', 'end', 'total']) Status = namedtuple('Status', ['code', 'comment']) ContentDisposition = namedtuple('ContentDisposition', ['disposition', 'parameters']) # Header value types (describe how to encode/decode the value) # class SimpleHeaderType(object): data_type = object @staticmethod def decode(encoded): return encoded @staticmethod def encode(decoded): return decoded class UTF8HeaderType(object): data_type = str @staticmethod def decode(encoded): return encoded.decode('utf-8') @staticmethod def encode(decoded): return decoded.encode('utf-8') class URIHeaderType(object): data_type = deque @staticmethod def decode(encoded): return deque(URI.parse(uri) for uri in encoded.split(' ')) @staticmethod def encode(decoded): return ' '.join(str(uri) for uri in decoded) class IntegerHeaderType(object): data_type = int @staticmethod def decode(encoded): return int(encoded) @staticmethod def encode(decoded): return str(decoded) class LimitedChoiceHeaderType(SimpleHeaderType): allowed_values = frozenset() @classmethod def decode(cls, encoded): if encoded not in cls.allowed_values: raise ValueError('Invalid value: {!r}'.format(encoded)) return encoded class SuccessReportHeaderType(LimitedChoiceHeaderType): allowed_values = frozenset({'yes', 'no'}) class FailureReportHeaderType(LimitedChoiceHeaderType): allowed_values = frozenset({'yes', 'no', 'partial'}) class ByteRangeHeaderType(object): data_type = ByteRange regex = re.compile(r'(?P\d+)-(?P\*|\d+)/(?P\*|\d+)') @classmethod def decode(cls, encoded): match = cls.regex.match(encoded) if match is None: raise ValueError('Invalid byte range value: {!r}'.format(encoded)) start, end, total = match.groups() start = int(start) end = int(end) if end != '*' else None total = int(total) if total != '*' else None return ByteRange(start, end, total) @staticmethod def encode(decoded): start, end, total = decoded return '{}-{}/{}'.format(start, '*' if end is None else end, '*' if total is None else total) class StatusHeaderType(object): data_type = Status @staticmethod def decode(encoded): namespace, sep, rest = encoded.partition(' ') if namespace != '000' or sep != ' ': raise ValueError('Invalid status value: {!r}'.format(encoded)) code, _, comment = rest.partition(' ') if not code.isdigit() or len(code) != 3: raise ValueError('Invalid status code: {!r}'.format(code)) return Status(int(code), comment or None) @staticmethod def encode(decoded): code, comment = decoded if comment is None: return '000 {:03d}'.format(code) else: return '000 {:03d} {}'.format(code, comment) class ContentDispositionHeaderType(object): data_type = ContentDisposition regex = re.compile(r'(\w+)=("[^"]+"|[^";]+)') @classmethod def decode(cls, encoded): disposition, _, parameters = encoded.partition(';') if not disposition: raise ValueError('Invalid content disposition: {!r}'.format(encoded)) return ContentDisposition(disposition, {name: value.strip('"') for name, value in cls.regex.findall(parameters)}) @staticmethod def encode(decoded): disposition, parameters = decoded return '; '.join([disposition] + ['{}="{}"'.format(name, value) for name, value in parameters.items()]) class ParameterListHeaderType(object): data_type = dict regex = re.compile(r'(\w+)=("[^"]+"|[^",]+)') @classmethod def decode(cls, encoded): return {name: value.strip('"') for name, value in cls.regex.findall(encoded)} @staticmethod def encode(decoded): return ', '.join('{}="{}"'.format(name, value) for name, value in decoded.items()) class DigestHeaderType(ParameterListHeaderType): @classmethod def decode(cls, encoded): algorithm, sep, parameters = encoded.partition(' ') if algorithm != 'Digest' or sep != ' ': raise ValueError('Invalid Digest header value') return super(DigestHeaderType, cls).decode(parameters) @staticmethod def encode(decoded): return 'Digest ' + super(DigestHeaderType, DigestHeaderType).encode(decoded) # Header classes # class MSRPHeaderMeta(type): __classmap__ = {} name = None def __init__(cls, name, bases, dictionary): type.__init__(cls, name, bases, dictionary) if cls.name is not None: cls.__classmap__[cls.name] = cls def __call__(cls, *args, **kw): if cls.name is not None: return super(MSRPHeaderMeta, cls).__call__(*args, **kw) # specialized class, instantiated directly. else: return cls._instantiate_specialized_class(*args, **kw) # non-specialized class, instantiated as a more specialized class if available. def _instantiate_specialized_class(cls, name, value): if name in cls.__classmap__: return super(MSRPHeaderMeta, cls.__classmap__[name]).__call__(value) else: return super(MSRPHeaderMeta, cls).__call__(name, value) class MSRPHeader(object, metaclass=MSRPHeaderMeta): name = None type = SimpleHeaderType def __init__(self, name, value): self.name = name if isinstance(value, str): self.encoded = value else: self.decoded = value def __eq__(self, other): if isinstance(other, MSRPHeader): return self.name == other.name and self.decoded == other.decoded return NotImplemented def __ne__(self, other): return not self == other @property def encoded(self): if self._encoded is None: self._encoded = self.type.encode(self._decoded) return self._encoded @encoded.setter def encoded(self, encoded): self._encoded = encoded self._decoded = None @property def decoded(self): if self._decoded is None: try: self._decoded = self.type.decode(self._encoded) except Exception as e: raise HeaderParsingError(self.name, str(e)) return self._decoded @decoded.setter def decoded(self, decoded): if not isinstance(decoded, self.type.data_type): try: # noinspection PyArgumentList decoded = self.type.data_type(decoded) except Exception: raise TypeError('value must be an instance of {}'.format(self.type.data_type.__name__)) self._decoded = decoded self._encoded = None class MSRPNamedHeader(MSRPHeader): def __init__(self, value): MSRPHeader.__init__(self, self.name, value) class ToPathHeader(MSRPNamedHeader): name = 'To-Path' type = URIHeaderType class FromPathHeader(MSRPNamedHeader): name = 'From-Path' type = URIHeaderType class MessageIDHeader(MSRPNamedHeader): name = 'Message-ID' type = SimpleHeaderType class SuccessReportHeader(MSRPNamedHeader): name = 'Success-Report' type = SuccessReportHeaderType class FailureReportHeader(MSRPNamedHeader): name = 'Failure-Report' type = FailureReportHeaderType class ByteRangeHeader(MSRPNamedHeader): name = 'Byte-Range' type = ByteRangeHeaderType @property def start(self): return self.decoded.start @property def end(self): return self.decoded.end @property def total(self): return self.decoded.total class StatusHeader(MSRPNamedHeader): name = 'Status' type = StatusHeaderType @property def code(self): return self.decoded.code @property def comment(self): return self.decoded.comment class ExpiresHeader(MSRPNamedHeader): name = 'Expires' type = IntegerHeaderType class MinExpiresHeader(MSRPNamedHeader): name = 'Min-Expires' type = IntegerHeaderType class MaxExpiresHeader(MSRPNamedHeader): name = 'Max-Expires' type = IntegerHeaderType class UsePathHeader(MSRPNamedHeader): name = 'Use-Path' type = URIHeaderType class WWWAuthenticateHeader(MSRPNamedHeader): name = 'WWW-Authenticate' type = DigestHeaderType class AuthorizationHeader(MSRPNamedHeader): name = 'Authorization' type = DigestHeaderType class AuthenticationInfoHeader(MSRPNamedHeader): name = 'Authentication-Info' type = ParameterListHeaderType class ContentTypeHeader(MSRPNamedHeader): name = 'Content-Type' type = SimpleHeaderType class ContentIDHeader(MSRPNamedHeader): name = 'Content-ID' type = SimpleHeaderType class ContentDescriptionHeader(MSRPNamedHeader): name = 'Content-Description' type = SimpleHeaderType class ContentDispositionHeader(MSRPNamedHeader): name = 'Content-Disposition' type = ContentDispositionHeaderType class UseNicknameHeader(MSRPNamedHeader): name = 'Use-Nickname' type = UTF8HeaderType class HeaderOrderMapping(dict): __levels__ = { 0: ['To-Path'], 1: ['From-Path'], 2: ['Status', 'Message-ID', 'Byte-Range', 'Success-Report', 'Failure-Report'] + ['Authorization', 'Authentication-Info', 'WWW-Authenticate', 'Expires', 'Min-Expires', 'Max-Expires', 'Use-Path', 'Use-Nickname'], 3: ['Content-ID', 'Content-Description', 'Content-Disposition'], 4: ['Content-Type'] } def __init__(self): super(HeaderOrderMapping, self).__init__({name: level for level, name_list in list(self.__levels__.items()) for name in name_list}) def __missing__(self, key): return 3 if key.startswith('Content-') else 2 sort_key = dict.__getitem__ class HeaderOrdering(object): name_map = HeaderOrderMapping() sort_key = name_map.sort_key class MissingHeader(object): decoded = None class HeaderMapping(dict): def __init__(self, *args, **kw): super(HeaderMapping, self).__init__(*args, **kw) self.__modified__ = True def __repr__(self): return '{}({})'.format(self.__class__.__name__, super(HeaderMapping, self).__repr__()) def __setitem__(self, key, value): super(HeaderMapping, self).__setitem__(key, value) self.__modified__ = True def __delitem__(self, key): super(HeaderMapping, self).__delitem__(key) self.__modified__ = True def __copy__(self): return self.__class__(self) def clear(self): super(HeaderMapping, self).clear() self.__modified__ = True def copy(self): return self.__class__(self) def pop(self, *args): result = super(HeaderMapping, self).pop(*args) self.__modified__ = True return result def popitem(self): result = super(HeaderMapping, self).popitem() self.__modified__ = True return result def setdefault(self, *args): result = super(HeaderMapping, self).setdefault(*args) self.__modified__ = True return result def update(self, *args, **kw): super(HeaderMapping, self).update(*args, **kw) self.__modified__ = True class MSRPData(object): __immutable__ = frozenset({'method', 'code', 'comment', 'headers'}) # Immutable attributes (cannot be overwritten) - def __init__(self, transaction_id, method=None, code=None, comment=None, headers=None, data='', contflag='$'): + def __init__(self, transaction_id, method=None, code=None, comment=None, headers=None, data=b'', contflag='$'): if method is None and code is None: raise ValueError('either method or code must be specified') elif method is not None and code is not None: raise ValueError('method and code cannot be both specified') elif code is None and comment is not None: raise ValueError('comment should only be specified when code is specified') self.transaction_id = transaction_id self.method = method self.code = code self.comment = comment self.headers = HeaderMapping(headers or {}) self.data = data self.contflag = contflag self.chunk_header = None # the chunk header (if the data was received from the network) self.chunk_footer = None # the chunk footer (if the data was received from the network) if method is not None: self.first_line = 'MSRP {} {}'.format(transaction_id, method) elif comment is None: self.first_line = 'MSRP {} {:03d}'.format(transaction_id, code) else: self.first_line = 'MSRP {} {:03d} {}'.format(transaction_id, code, comment) self.__modified__ = True def __setattr__(self, name, value): if name in self.__dict__: if name in self.__immutable__: raise AttributeError('Cannot overwrite attribute') elif name == 'transaction_id': self.first_line = self.first_line.replace(self.transaction_id, value) self.__modified__ = True super(MSRPData, self).__setattr__(name, value) def __delattr__(self, name): if name in self.__immutable__: raise AttributeError('Cannot delete attribute') super(MSRPData, self).__delattr__(name) def __str__(self): # TODO: make __str__ == encode()? return self.first_line def __repr__(self): description = self.first_line for name in sorted(self.headers, key=HeaderOrdering.sort_key): description += ' {}={!r}'.format(name, self.headers[name].encoded) description += ' len={}'.format(self.size) return '<{} at {:#x} {} {}>'.format(self.__class__.__name__, id(self), description, self.contflag) def __eq__(self, other): if isinstance(other, MSRPData): return self.first_line == other.first_line and self.headers == other.headers and self.data == other.data and self.contflag == other.contflag return NotImplemented def __ne__(self, other): return not self == other def copy(self): return self.__class__(self.transaction_id, self.method, self.code, self.comment, self.headers.copy(), self.data, self.contflag) def add_header(self, header): self.headers[header.name] = header def verify_headers(self): if 'To-Path' not in self.headers: raise HeaderParsingError('To-Path', 'header is missing') if 'From-Path' not in self.headers: raise HeaderParsingError('From-Path', 'header is missing') for header in self.headers.values(): _ = header.decoded @property def from_path(self): return self.headers.get('From-Path', MissingHeader).decoded @property def to_path(self): return self.headers.get('To-Path', MissingHeader).decoded @property def content_type(self): return self.headers.get('Content-Type', MissingHeader).decoded @property def message_id(self): return self.headers.get('Message-ID', MissingHeader).decoded @property def byte_range(self): return self.headers.get('Byte-Range', MissingHeader).decoded @property def status(self): return self.headers.get('Status', MissingHeader).decoded @property def failure_report(self): return self.headers.get('Failure-Report', MissingHeader).decoded or 'yes' @property def success_report(self): return self.headers.get('Success-Report', MissingHeader).decoded or 'no' @property def size(self): return len(self.data) @property def encoded_header(self): if self.__modified__ or self.headers.__modified__: lines = [self.first_line] + ['{}: {}'.format(name, self.headers[name].encoded) for name in sorted(self.headers, key=HeaderOrdering.sort_key)] if 'Content-Type' in self.headers: lines.append('\r\n') self.__dict__['encoded_header'] = '\r\n'.join(lines) self.__modified__ = self.headers.__modified__ = False return self.__dict__['encoded_header'] @property def encoded_footer(self): return '\r\n-------{}{}\r\n'.format(self.transaction_id, self.contflag) def encode(self): - return self.encoded_header + self.data + self.encoded_footer + encoded_header = self.encoded_header if isinstance(self.encoded_header, bytes) else self.encoded_header.encode() + data = self.data if isinstance(self.data, bytes) else self.data.encode() + encoded_footer = self.encoded_footer if isinstance(self.encoded_footer, bytes) else self.encoded_footer.encode() + return encoded_header + data + encoded_footer # noinspection PyProtectedMember class MSRPProtocol(LineReceiver): # TODO: _ in the method name is not legal, but sipsimple defined the FILE_OFFSET method first_line_re = re.compile(r'^MSRP ([A-Za-z0-9][A-Za-z0-9.+%=-]{3,31}) (?:([A-Z_]+)|(\d{3})(?: (.+))?)$') MAX_LENGTH = 16384 MAX_LINES = 64 def __init__(self, msrp_transport): self.msrp_transport = msrp_transport - self.term_buf = '' + self.term_buf = b'' self.term_re = None self.term_substrings = [] self.data = None self.line_count = 0 def _reset(self): self.data = None self.line_count = 0 def connectionMade(self): self.msrp_transport._got_transport(self.transport) + def lineLengthExceeded(self, line): + self._reset() + def lineReceived(self, line): - line = line.decode('utf-8') + + try: + decoded_line = line.decode('utf-8') + except UnicodeDecodeError: + decoded_line = None + if self.data: if len(line) == 0: +# The end-line that terminates the request MUST be composed of seven +# "-" (minus sign) characters, the transaction ID as used in the start +# line, and a flag character. If a body is present, the end-line MUST +# be preceded by a CRLF that is not part of the body. If the chunk +# represents the data that forms the end of the complete message, the +# flag value MUST be a "$". If the sender is aborting an incomplete +# message, and intends to send no further chunks in that message, the +# flag MUST be a "#". Otherwise, the flag MUST be a "+". +# +# If the request contains a body, the sender MUST ensure that the end- +# line (seven hyphens, the transaction identifier, and a continuation +# flag) is not present in the body. If the end-line is present in the +# body, the sender MUST choose a new transaction identifier that is not +# present in the body, and add a CRLF if needed, and the end-line, +# including the "$", "#", or "+" character. + terminator = '\r\n-------' + self.data.transaction_id continue_flags = [c+'\r\n' for c in '$#+'] - self.term_buf = '' + self.term_buf = b'' self.term_re = re.compile("^(.*)%s([$#+])\r\n(.*)$" % re.escape(terminator), re.DOTALL) self.term_substrings = [terminator[:i] for i in range(1, len(terminator)+1)] + [terminator+cont[:i] for cont in continue_flags for i in range(1, len(cont))] self.term_substrings.reverse() - self.data.chunk_header += self.delimiter.decode('utf-8') + self.data.chunk_header += self.delimiter self.msrp_transport._data_start(self.data) self.setRawMode() else: - match = self.term_re.match(line) + match = self.term_re.match(decoded_line) if decoded_line else None if match: continuation = match.group(1) - self.data.chunk_footer = line + self.delimiter.decode('utf-8') + self.data.chunk_footer = line + self.delimiter self.msrp_transport._data_start(self.data) - self.msrp_transport._data_end(continuation) + self.msrp_transport._data_end(continuation.encode()) self._reset() else: - self.data.chunk_header += line + self.delimiter.decode('utf-8') + self.data.chunk_header += line + self.delimiter self.line_count += 1 if self.line_count > self.MAX_LINES: self.msrp_transport.logger.received_illegal_data(self.data.chunk_header, self.msrp_transport) self._reset() return try: - name, value = line.split(': ', 1) - except ValueError: + name, value = decoded_line.split(': ', 1) + except (ValueError, AttributeError) as e: return # let this pass silently, we'll just not read this line. TODO: review this (ignore or drop connection?) else: + #print('MSRP Header: %s=%s' % (name, value)) self.data.add_header(MSRPHeader(name, value)) else: # this is a new message. TODO: drop connection if first line cannot be parsed? - match = self.first_line_re.match(line) + match = self.first_line_re.match(decoded_line) if decoded_line else None if match: transaction_id, method, code, comment = match.groups() code = int(code) if code is not None else None + #print('MSRP Data start %s %s' % (method, transaction_id)) self.data = MSRPData(transaction_id, method, code, comment) - self.data.chunk_header = line + self.delimiter.decode('utf-8') + self.data.chunk_header = line + self.delimiter self.term_re = re.compile(r'^-------{}([$#+])$'.format(re.escape(transaction_id))) else: - self.msrp_transport.logger.received_illegal_data(line + self.delimiter.decode('utf-8'), self.msrp_transport) - - def lineLengthExceeded(self, line): - self._reset() + self.msrp_transport.logger.received_illegal_data(line + self.delimiter, self.msrp_transport) def rawDataReceived(self, data): - data = self.term_buf + data.decode('utf-8') - match = self.term_re.match(data) - if match: # we got the last data for this message - contents, continuation, extra = match.groups() + data = self.term_buf + data + + terminator = '\r\n-------' + self.data.transaction_id + terminator = terminator.encode() + contents_position = data.find(terminator) + + if contents_position > -1: # we got the last data for this message + contents = data[:contents_position] + leftover = data[contents_position+len(terminator):] + continuation_position = leftover.find(b'\r\n') + continuation = leftover[:continuation_position] + extra = leftover[continuation_position+len(continuation)+1:] + if contents: self.msrp_transport._data_write(contents, final=True) - self.data.chunk_footer = '\r\n-------{}{}\r\n'.format(self.data.transaction_id, continuation) + + chunk_footer = '\r\n-------{}{}\r\n'.format(self.data.transaction_id, continuation.decode()) + self.data.chunk_footer = chunk_footer.encode() self.msrp_transport._data_end(continuation) self._reset() - self.setLineMode(extra.encode('utf-8')) + self.setLineMode(extra) else: for term in self.term_substrings: - if data.endswith(term): - self.term_buf = data[-len(term):] - data = data[:-len(term)] + if data and data.endswith(term.encode()): + self.term_buf = data[-len(term.encode()):] + data = data[:-len(term.encode())] break else: - self.term_buf = '' + self.term_buf = b'' + self.msrp_transport._data_write(data, final=False) def connectionLost(self, reason=connectionDone): self.msrp_transport.connection_lost(reason) class ConnectInfo(object): host = None use_tls = True port = 2855 def __init__(self, host=None, use_tls=None, port=None, credentials=None): if host is not None: self.host = host if use_tls is not None: self.use_tls = use_tls if port is not None: self.port = port self.credentials = credentials if self.use_tls and self.credentials is None: self.credentials = X509Credentials(None, None) @property def scheme(self): if self.use_tls: return 'msrps' else: return 'msrp' # use TLS_URI and TCP_URI ? class URI(ConnectInfo): _uri_re = re.compile(r'^(?P.*?)://(((?P.*?)@)?(?P.*?)(:(?P[0-9]+?))?)(/(?P.*?))?;(?P.*?)(;(?P.*))?$') def __init__(self, host=None, use_tls=None, user=None, port=None, session_id=None, transport="tcp", parameters=None, credentials=None): ConnectInfo.__init__(self, host or host_module.default_ip, use_tls=use_tls, port=port, credentials=credentials) if session_id is None: session_id = '%x' % random.getrandbits(80) if parameters is None: parameters = {} self.user = user self.transport = transport self.session_id = session_id self.parameters = parameters # noinspection PyTypeChecker @classmethod def parse(cls, value): match = cls._uri_re.match(value) if match is None: raise ParsingError('Cannot parse URI') uri_params = match.groupdict() scheme = uri_params.pop('scheme') if scheme not in ('msrp', 'msrps'): raise ParsingError('Invalid URI scheme: %r' % scheme) if uri_params['transport'] != 'tcp': raise ParsingError("Invalid URI transport: %r (only 'tcp' is accepted)" % uri_params['transport']) uri_params['use_tls'] = scheme == 'msrps' if uri_params['port'] is not None: uri_params['port'] = int(uri_params['port']) if uri_params['parameters'] is not None: try: uri_params['parameters'] = dict(param.split('=') for param in uri_params['parameters'].split(';')) except ValueError: raise ParsingError('Cannot parse URI parameters') return cls(**uri_params) def __repr__(self): arguments = 'host', 'use_tls', 'user', 'port', 'session_id', 'transport', 'parameters', 'credentials' return '{}({})'.format(self.__class__.__name__, ', '.join('{}={!r}'.format(name, getattr(self, name)) for name in arguments)) def __str__(self): user_part = '{}@'.format(self.user) if self.user else '' port_part = ':{}'.format(self.port) if self.port else '' session_part = '/{}'.format(self.session_id) if self.session_id else '' parameter_parts = [';{}={}'.format(name, value) for name, value in self.parameters.items()] if self.parameters else [] return ''.join([self.scheme, '://', user_part, self.host, port_part, session_part, ';', self.transport] + parameter_parts) def __eq__(self, other): if self is other: return True if isinstance(other, URI): # MSRP URI comparison according to section 6.1 of RFC 4975 self_items = self.use_tls, self.host.lower(), self.port, self.session_id, self.transport.lower() other_items = other.use_tls, other.host.lower(), other.port, other.session_id, other.transport.lower() return self_items == other_items return NotImplemented def __ne__(self, other): return not self == other def __hash__(self): return hash((self.use_tls, self.host.lower(), self.port, self.session_id, self.transport.lower())) diff --git a/msrplib/session.py b/msrplib/session.py index 1ce301f..eea9666 100644 --- a/msrplib/session.py +++ b/msrplib/session.py @@ -1,350 +1,350 @@ # Copyright (C) 2008-2012 AG Projects. See LICENSE for details # import traceback from time import time from twisted.internet.error import ConnectionClosed, ConnectionDone from twisted.python.failure import Failure from gnutls.errors import GNUTLSError from eventlib import api, coros, proc from eventlib.twistedutil.protocol import ValueQueue from msrplib import protocol, MSRPError from msrplib.transport import make_report, make_response, MSRPTransactionError ConnectionClosedErrors = (ConnectionClosed, GNUTLSError) class MSRPSessionError(MSRPError): pass class MSRPBadContentType(MSRPTransactionError): code = 415 comment = 'Unsupported media type' class LocalResponse(MSRPTransactionError): def __repr__(self): return '' % (self.code, self.comment) Response200OK = LocalResponse("OK", 200) Response408Timeout = LocalResponse("Timed out while waiting for transaction response", 408) def contains_mime_type(mimetypelist, mimetype): """Return True if mimetypelist contains mimetype. mimietypelist either contains the complete mime types, such as 'text/plain', or simple patterns, like 'text/*', or simply '*'. """ mimetype = mimetype.lower().partition(';')[0] for pattern in mimetypelist: pattern = pattern.lower() if pattern == '*': return True if pattern == mimetype: return True if pattern.endswith('/*') and mimetype.startswith(pattern[:-1]): return True return False class OutgoingChunk(object): __slots__ = ('chunk', 'response_callback') - def __init__(self, chunk, response_callback=None): + def __init__(self, chunk, response_callback=None): self.chunk = chunk self.response_callback = response_callback class MSRPSession(object): RESPONSE_TIMEOUT = 30 SHUTDOWN_TIMEOUT = 1 KEEPALIVE_INTERVAL = 60 def __init__(self, msrptransport, accept_types=['*'], on_incoming_cb=None, automatic_reports=True): self.msrp = msrptransport self.accept_types = accept_types self.automatic_reports = automatic_reports if not callable(on_incoming_cb): raise TypeError('on_incoming_cb must be callable: %r' % on_incoming_cb) self._on_incoming_cb = on_incoming_cb self.expected_responses = {} self.outgoing = coros.queue() self.reader_job = proc.spawn(self._reader) self.writer_job = proc.spawn(self._writer) self.state = 'CONNECTED' # -> 'FLUSHING' -> 'CLOSING' -> 'DONE' # in FLUSHING writer sends only while there's something in the outgoing queue # then it exits and sets state to 'CLOSING' which makes reader only pay attention # to responses and success reports. (XXX it could now discard incoming data chunks # with direct write() since writer is dead) self.reader_job.link(self.writer_job) self.last_expected_response = 0 self.keepalive_proc = proc.spawn(self._keepalive) def _get_logger(self): return self.msrp.logger def _set_logger(self, logger): self.msrp.logger = logger logger = property(_get_logger, _set_logger) def set_state(self, state): self.logger.debug('%s (was %s)', state, self.state) self.state = state @property def connected(self): return self.state=='CONNECTED' def shutdown(self, wait=True): """Send the messages already in queue then close the connection""" self.set_state('FLUSHING') self.keepalive_proc.kill() self.keepalive_proc = None self.outgoing.send(None) if wait: self.writer_job.wait(None, None) self.reader_job.wait(None, None) def _keepalive(self): while True: api.sleep(self.KEEPALIVE_INTERVAL) if not self.connected: return try: chunk = self.msrp.make_send_request() chunk.add_header(protocol.MSRPHeader('Keep-Alive', 'yes')) self.deliver_chunk(chunk) except MSRPTransactionError as e: if e.code == 408: self.msrp.loseConnection(wait=False) self.set_state('CLOSING') return def _handle_incoming_response(self, chunk): try: response_cb, timer = self.expected_responses.pop(chunk.transaction_id) except KeyError: pass else: if timer is not None: timer.cancel() response_cb(chunk) def _check_incoming_SEND(self, chunk): error = self.msrp.check_incoming_SEND_chunk(chunk) if error is not None: return error if chunk.data: if chunk.content_type is None: return MSRPBadContentType('Content-Type header missing') if not contains_mime_type(self.accept_types, chunk.content_type): return MSRPBadContentType def _handle_incoming_SEND(self, chunk): error = self._check_incoming_SEND(chunk) if error is None: code, comment = 200, 'OK' else: code, comment = error.code, error.comment response = make_response(chunk, code, comment) if response is not None: self.outgoing.send(OutgoingChunk(response)) if code == 200: self._on_incoming_cb(chunk) if self.automatic_reports: report = make_report(chunk, 200, 'OK') if report is not None: self.outgoing.send(OutgoingChunk(report)) def _handle_incoming_REPORT(self, chunk): self._on_incoming_cb(chunk) def _handle_incoming_NICKNAME(self, chunk): if 'Use-Nickname' not in chunk.headers or 'Success-Report' in chunk.headers or 'Failure-Report' in chunk.headers: response = make_response(chunk, 400, 'Bad request') self.outgoing.send(OutgoingChunk(response)) return self._on_incoming_cb(chunk) def _reader(self): """Wait forever for new chunks. Notify the user about the good ones through self._on_incoming_cb. If a response to a previously sent chunk is received, pop the corresponding response_cb from self.expected_responses and send the response there. """ error = Failure(ConnectionDone()) try: self.writer_job.link(self.reader_job) try: while self.state in ['CONNECTED', 'FLUSHING']: chunk = self.msrp.read_chunk() if chunk.method is None: # response self._handle_incoming_response(chunk) else: method = getattr(self, '_handle_incoming_%s' % chunk.method, None) if method is not None: method(chunk) else: response = make_response(chunk, 501, 'Method unknown') self.outgoing.send(OutgoingChunk(response)) except proc.LinkedExited: # writer has exited pass finally: self.writer_job.unlink(self.reader_job) self.writer_job.kill() self.logger.debug('reader: expecting responses only') delay = time() - self.last_expected_response if delay>=0 and self.expected_responses: # continue read the responses until the last timeout expires with api.timeout(delay, None): while self.expected_responses: chunk = self.msrp.read_chunk() if chunk.method is None: self._handle_incoming_response(chunk) else: self.logger.debug('dropping incoming %r', chunk) # read whatever left in the queue with api.timeout(0, None): while self.msrp._queue: chunk = self.msrp.read_chunk() if chunk.method is None: self._handle_incoming_response(chunk) else: self.logger.debug('dropping incoming %r', chunk) self.logger.debug('reader: done') except ConnectionClosedErrors as e: self.logger.debug('reader: exiting because of %r', e) error = Failure(e) except Exception: self.logger.exception('reader: captured unhandled exception:') error = Failure() raise finally: self._on_incoming_cb(error=error) self.msrp.loseConnection(wait=False) self.set_state('DONE') def _writer(self): try: while self.state=='CONNECTED' or (self.state=='FLUSHING' and self.outgoing): item = self.outgoing.wait() if item is None: break self._write_chunk(item.chunk, item.response_callback) except ConnectionClosedErrors + (proc.LinkedExited, proc.ProcExit) as e: self.logger.debug('writer: exiting because of %r' % e) except: self.logger.exception('writer: captured unhandled exception:') raise finally: self.msrp.loseConnection(wait=False) self.set_state('CLOSING') def _write_chunk(self, chunk, response_cb=None): assert chunk.transaction_id not in self.expected_responses, "MSRP transaction %r is already in progress" % chunk.transaction_id self.msrp.write_chunk(chunk) if response_cb is not None: timer = api.get_hub().schedule_call_global(self.RESPONSE_TIMEOUT, self._response_timeout, chunk.transaction_id, Response408Timeout) self.expected_responses[chunk.transaction_id] = (response_cb, timer) self.last_expected_response = time() + self.RESPONSE_TIMEOUT def _response_timeout(self, id, timeout_error): response_cb, timer = self.expected_responses.pop(id, (None, None)) if response_cb is not None: response_cb(timeout_error) if timer is not None: timer.cancel() def send_chunk(self, chunk, response_cb=None): """Send `chunk'. Report the result via `response_cb'. When `response_cb' argument is present, it will be used to report the transaction response to the caller. When a response is received or generated locally, `response_cb' is called with one argument. The function must do something quickly and must not block, because otherwise it would the reader greenlet. If no response was received after RESPONSE_TIMEOUT seconds, * 408 response is generated if Failure-Report was 'yes' or absent * 200 response is generated if Failure-Report was 'partial' or 'no' Note that it's rather wasteful to provide `response_cb' argument other than None for chunks with Failure-Report='no' since it will always fire 30 seconds later with 200 result (unless the other party is broken and ignores Failure-Report header) If sending is impossible raise MSRPSessionError. """ assert chunk.transaction_id not in self.expected_responses, "MSRP transaction %r is already in progress" % chunk.transaction_id if response_cb is not None and not callable(response_cb): raise TypeError('response_cb must be callable: %r' % (response_cb, )) if self.state != 'CONNECTED': raise MSRPSessionError('Cannot send chunk because MSRPSession is %s' % self.state) if self.msrp._disconnected_event.ready(): raise MSRPSessionError(str(self.msrp._disconnected_event.wait())) self.outgoing.send(OutgoingChunk(chunk, response_cb)) def send_report(self, chunk, code, reason): if chunk.method != 'SEND': raise ValueError('reports may only be sent for SEND chunks') report = make_report(chunk, code, reason) if report is not None: self.send_chunk(report) def deliver_chunk(self, chunk, event=None): """Send chunk, wait for the transaction response (if Failure-Report header is not 'no'). Return the transaction response if it's a success, raise MSRPTransactionError if it's not. If chunk's Failure-Report is 'no', return None immediately. """ if chunk.failure_report!='no' and event is None: event = coros.event() self.send_chunk(chunk, event.send) if event is not None: response = event.wait() if isinstance(response, Exception): raise response elif 200 <= response.code <= 299: return response raise MSRPTransactionError(comment=response.comment, code=response.code) def make_message(self, msg, content_type, message_id=None): chunk = self.msrp.make_send_request(data=msg, message_id=message_id) chunk.add_header(protocol.ContentTypeHeader(content_type)) return chunk def send_message(self, msg, content_type): chunk = self.make_message(msg, content_type) self.send_chunk(chunk) return chunk def deliver_message(self, msg, content_type): chunk = self.make_message(msg, content_type) self.deliver_chunk(chunk) return chunk class GreenMSRPSession(MSRPSession): def __init__(self, msrptransport, accept_types=['*']): MSRPSession.__init__(self, msrptransport, accept_types, on_incoming_cb=self._incoming_cb) self.incoming = ValueQueue() def receive_chunk(self): return self.incoming.wait() def _incoming_cb(self, value=None, error=None): if error is not None: self.incoming.send_exception(error.type, error.value, error.tb) else: self.incoming.send(value) # TODO: # 413 - requires special action both in reader and in writer # continuation: # # All MSRP endpoints MUST be able to receive the multipart/mixed [15] and multipart/alternative [15] media-types. diff --git a/msrplib/transport.py b/msrplib/transport.py index 5985478..efcd811 100644 --- a/msrplib/transport.py +++ b/msrplib/transport.py @@ -1,311 +1,339 @@ # Copyright (C) 2008-2012 AG Projects. See LICENSE for details import random from application import log from eventlib.twistedutil.protocol import GreenTransportBase from twisted.internet.error import ConnectionDone from msrplib import protocol, MSRPError from msrplib.trafficlog import Logger log = log.get_logger('msrplib') class ChunkParseError(MSRPError): """Failed to parse incoming chunk""" class MSRPTransactionError(MSRPError): def __init__(self, comment=None, code=None): if comment is not None: self.comment = comment if code is not None: self.code = code if not hasattr(self, 'code'): raise TypeError("must provide 'code'") def __str__(self): if hasattr(self, 'comment'): return '%s %s' % (self.code, self.comment) else: return str(self.code) class MSRPBadRequest(MSRPTransactionError): code = 400 comment = 'Bad Request' def __str__(self): return 'Remote party sent bogus data' class MSRPNoSuchSessionError(MSRPTransactionError): code = 481 comment = 'No such session' data_start, data_end, data_write, data_final_write = list(range(4)) def make_report(chunk, code, comment): if chunk.success_report == 'yes' or (chunk.failure_report in ('yes', 'partial') and code != 200): report = protocol.MSRPData(transaction_id='%x' % random.getrandbits(64), method='REPORT') report.add_header(protocol.ToPathHeader(chunk.from_path)) report.add_header(protocol.FromPathHeader([chunk.to_path[0]])) report.add_header(protocol.StatusHeader(protocol.Status(code, comment))) report.add_header(protocol.MessageIDHeader(chunk.message_id)) if chunk.byte_range is None: start = 1 total = chunk.size else: start, end, total = chunk.byte_range report.add_header(protocol.ByteRangeHeader(protocol.ByteRange(start, start+chunk.size-1, total))) return report else: return None def make_response(chunk, code, comment): """Construct a response to a request as described in RFC4975 Section 7.2. If the response is not needed, return None. If a required header missing, raise ChunkParseError. """ if chunk.failure_report == 'no': return if chunk.failure_report == 'partial' and code == 200: return if chunk.to_path is None: raise ChunkParseError('missing To-Path header: %r' % chunk) if chunk.from_path is None: raise ChunkParseError('missing From-Path header: %r' % chunk) if chunk.method == 'SEND': to_path = [chunk.from_path[0]] else: to_path = chunk.from_path from_path = [chunk.to_path[0]] response = protocol.MSRPData(chunk.transaction_id, code=code, comment=comment) response.add_header(protocol.ToPathHeader(to_path)) response.add_header(protocol.FromPathHeader(from_path)) return response class MSRPTransport(GreenTransportBase): protocol_class = protocol.MSRPProtocol def __init__(self, local_uri, logger, use_sessmatch=False): GreenTransportBase.__init__(self) if local_uri is not None and not isinstance(local_uri, protocol.URI): raise TypeError('Not MSRP URI instance: %r' % (local_uri, )) # The following members define To-Path and From-Path headers as following: # * Outgoing request: # From-Path: local_uri # To-Path: local_path + remote_path + [remote_uri] # * Incoming request: # From-Path: remote_path + remote_uri # To-Path: remote_path + local_path + [local_uri] # XXX self.local_uri = local_uri if logger is None: logger = Logger() self.logger = logger self.local_path = [] self.remote_uri = None self.remote_path = [] self.use_sessmatch = use_sessmatch def next_host(self): if self.local_path: return self.local_path[0] return self.full_remote_path[0] def set_local_path(self, lst): self.local_path = lst @property def full_local_path(self): # suitable to put into INVITE return self.local_path + [self.local_uri] @property def full_remote_path(self): return self.remote_path + [self.remote_uri] def make_request(self, method): transaction_id = '%x' % random.getrandbits(64) chunk = protocol.MSRPData(transaction_id=transaction_id, method=method) chunk.add_header(protocol.ToPathHeader(self.local_path + self.remote_path + [self.remote_uri])) chunk.add_header(protocol.FromPathHeader([self.local_uri])) return chunk def make_send_request(self, message_id=None, data='', start=1, end=None, length=None): chunk = self.make_request('SEND') if end is None: end = start - 1 + len(data) if length is None: length = start - 1 + len(data) if end == length != '*': contflag = '$' else: contflag = '+' chunk.add_header(protocol.ByteRangeHeader(protocol.ByteRange(start, end if length <= 2048 else None, length))) if message_id is None: message_id = '%x' % random.getrandbits(64) chunk.add_header(protocol.MessageIDHeader(message_id)) chunk.data = data chunk.contflag = contflag return chunk def _data_start(self, data): self._queue.send((data_start, data)) def _data_end(self, continuation): self._queue.send((data_end, continuation)) def _data_write(self, contents, final): if final: self._queue.send((data_final_write, contents)) else: self._queue.send((data_write, contents)) def write_chunk(self, chunk, wait=True): self.write(chunk.encode(), wait=wait) self.logger.sent_chunk(chunk, transport=self) def read_chunk(self, max_size=1024*1024*4): """Wait for a new chunk and return it. If there was an error, close the connection and raise ChunkParseError. In case of unintelligible input, lose the connection and return None. When the connection is closed, raise the reason of the closure (e.g. ConnectionDone). """ - assert max_size > 0 func, msrpdata = self._wait() if func != data_start: self.logger.debug('Bad data: %r %r', func, msrpdata) self.loseConnection() - raise ChunkParseError + raise ChunkParseError('Bad start data') + data = msrpdata.data - func, param = self._wait() + func, chunk = self._wait() + + try: + param = chunk.decode() if isinstance(chunk, bytes) else chunk + except UnicodeDecodeError: + param = None + + #data_start, data_end, data_write, data_final_write = list(range(4)) + while func == data_write: - data += param - if len(data) > max_size: - self.logger.debug('Chunk is too big (max_size=%d bytes)', max_size) + data += chunk + if len(chunk) > max_size: + msg = 'Chunk is too big (max_size=%d bytes)', max_size + self.logger.debug(msg) self.loseConnection() - raise ChunkParseError - func, param = self._wait() - if func == data_final_write: - data += param - func, param = self._wait() + raise ChunkParseError(msg) + + func, chunk = self._wait() + + if func == data_final_write: + data += chunk + func, chunk = self._wait() + try: + param = chunk.decode() if isinstance(chunk, bytes) else chunk + except UnicodeDecodeError: + param = None + if func != data_end: - self.logger.debug('Bad data: %r %s', func, repr(param)[:100]) + bad_data = repr(param)[:100] if param else None + msg = 'Bad data: %r %s', func, bad_data + self.logger.debug(msg) self.loseConnection() - raise ChunkParseError + raise ChunkParseError(msg) + + try: + param = chunk.decode() if isinstance(chunk, bytes) else chunk + except UnicodeDecodeError: + param = None + if param not in "$+#": - self.logger.debug('Bad data: %r %s', func, repr(param)[:100]) + bad_data = repr(param)[:100] if param else None + self.logger.debug(bad_data) self.loseConnection() - raise ChunkParseError + raise ChunkParseError(bad_data) + msrpdata.data = data msrpdata.contflag = param + self.logger.received_chunk(msrpdata, transport=self) return msrpdata def _set_full_remote_path(self, full_remote_path): # as received in response to INVITE if not all(isinstance(x, protocol.URI) for x in full_remote_path): raise TypeError('Not all elements are MSRP URI: %r' % full_remote_path) self.remote_uri = full_remote_path[-1] self.remote_path = full_remote_path[:-1] def bind(self, full_remote_path): self._set_full_remote_path(full_remote_path) chunk = self.make_send_request() self.write_chunk(chunk) # With some ACM implementations both parties may think they are active, # so they will both send an empty SEND request. -Saul while True: chunk = self.read_chunk() if chunk.code is None: # This was not a response, it was a request if chunk.method == 'SEND' and not chunk.data: self.write_response(chunk, 200, 'OK') else: self.loseConnection(wait=False) raise MSRPNoSuchSessionError('Chunk received while binding session: %s' % chunk) elif chunk.code != 200: self.loseConnection(wait=False) raise MSRPNoSuchSessionError('Cannot bind session: %s' % chunk) else: break def write_response(self, chunk, code, comment, wait=True): """Generate and write the response, lose the connection in case of error""" try: response = make_response(chunk, code, comment) except ChunkParseError as ex: log.error('Failed to generate a response: %s' % ex) self.loseConnection(wait=False) raise except Exception: log.exception('Failed to generate a response') self.loseConnection(wait=False) raise else: if response is not None: self.write_chunk(response, wait=wait) def accept_binding(self, full_remote_path): self._set_full_remote_path(full_remote_path) chunk = self.read_chunk() error = self.check_incoming_SEND_chunk(chunk) if error is None: code, comment = 200, 'OK' else: code, comment = error.code, error.comment self.write_response(chunk, code, comment) if 'Content-Type' in chunk.headers or chunk.size > 0: # deliver chunk to read_chunk data = chunk.data chunk.data = '' self._data_start(chunk) self._data_write(data, final=True) self._data_end(chunk.contflag) def check_incoming_SEND_chunk(self, chunk): """Check the 'To-Path' and 'From-Path' of the incoming SEND chunk. Return None is the paths are valid for this connection. If an error is detected and MSRPError is created and returned. """ assert chunk.method == 'SEND', repr(chunk) if chunk.to_path is None: return MSRPBadRequest('To-Path header missing') if chunk.from_path is None: return MSRPBadRequest('From-Path header missing') to_path = list(chunk.to_path) from_path = list(chunk.from_path) expected_to = [self.local_uri] expected_from = self.local_path + self.remote_path + [self.remote_uri] # Match only session ID when use_sessmatch is set (http://tools.ietf.org/html/draft-ietf-simple-msrp-sessmatch-10) if self.use_sessmatch: if to_path[0].session_id != expected_to[0].session_id: log.error('To-Path: expected session_id %s, got %s' % (expected_to[0].session_id, to_path[0].session_id)) return MSRPNoSuchSessionError('Invalid To-Path') if from_path[0].session_id != expected_from[0].session_id: log.error('From-Path: expected session_id %s, got %s' % (expected_from[0].session_id, from_path[0].session_id)) return MSRPNoSuchSessionError('Invalid From-Path') else: if to_path != expected_to: log.error('To-Path: expected %r, got %r' % (expected_to, to_path)) return MSRPNoSuchSessionError('Invalid To-Path') if from_path != expected_from: log.error('From-Path: expected %r, got %r' % (expected_from, from_path)) return MSRPNoSuchSessionError('Invalid From-Path') def connection_lost(self, reason): - message = 'Closed connection to {0.host}:{0.port}'.format(self.getPeer()) + #message = 'Closed connection to {0.host}:{0.port}'.format(self.transport.getPeer()) + message = 'Closed connection' if not isinstance(reason.value, ConnectionDone): message += ' ({})'.format(reason.getErrorMessage()) self.logger.info(message) self._connectionLost(reason)