1"""An NNTP client class based on: 2- RFC 977: Network News Transfer Protocol 3- RFC 2980: Common NNTP Extensions 4- RFC 3977: Network News Transfer Protocol (version 2) 5 6Example: 7 8>>> from nntplib import NNTP 9>>> s = NNTP('news') 10>>> resp, count, first, last, name = s.group('comp.lang.python') 11>>> print('Group', name, 'has', count, 'articles, range', first, 'to', last) 12Group comp.lang.python has 51 articles, range 5770 to 5821 13>>> resp, subs = s.xhdr('subject', '{0}-{1}'.format(first, last)) 14>>> resp = s.quit() 15>>> 16 17Here 'resp' is the server response line. 18Error responses are turned into exceptions. 19 20To post an article from a file: 21>>> f = open(filename, 'rb') # file containing article, including header 22>>> resp = s.post(f) 23>>> 24 25For descriptions of all methods, read the comments in the code below. 26Note that all arguments and return values representing article numbers 27are strings, not numbers, since they are rarely used for calculations. 28""" 29 30# RFC 977 by Brian Kantor and Phil Lapsley. 31# xover, xgtitle, xpath, date methods by Kevan Heydon 32 33# Incompatible changes from the 2.x nntplib: 34# - all commands are encoded as UTF-8 data (using the "surrogateescape" 35# error handler), except for raw message data (POST, IHAVE) 36# - all responses are decoded as UTF-8 data (using the "surrogateescape" 37# error handler), except for raw message data (ARTICLE, HEAD, BODY) 38# - the `file` argument to various methods is keyword-only 39# 40# - NNTP.date() returns a datetime object 41# - NNTP.newgroups() and NNTP.newnews() take a datetime (or date) object, 42# rather than a pair of (date, time) strings. 43# - NNTP.newgroups() and NNTP.list() return a list of GroupInfo named tuples 44# - NNTP.descriptions() returns a dict mapping group names to descriptions 45# - NNTP.xover() returns a list of dicts mapping field names (header or metadata) 46# to field values; each dict representing a message overview. 47# - NNTP.article(), NNTP.head() and NNTP.body() return a (response, ArticleInfo) 48# tuple. 49# - the "internal" methods have been marked private (they now start with 50# an underscore) 51 52# Other changes from the 2.x/3.1 nntplib: 53# - automatic querying of capabilities at connect 54# - New method NNTP.getcapabilities() 55# - New method NNTP.over() 56# - New helper function decode_header() 57# - NNTP.post() and NNTP.ihave() accept file objects, bytes-like objects and 58# arbitrary iterables yielding lines. 59# - An extensive test suite :-) 60 61# TODO: 62# - return structured data (GroupInfo etc.) everywhere 63# - support HDR 64 65# Imports 66import re 67import socket 68import collections 69import datetime 70import sys 71import warnings 72 73try: 74 import ssl 75except ImportError: 76 _have_ssl = False 77else: 78 _have_ssl = True 79 80from email.header import decode_header as _email_decode_header 81from socket import _GLOBAL_DEFAULT_TIMEOUT 82 83__all__ = ["NNTP", 84 "NNTPError", "NNTPReplyError", "NNTPTemporaryError", 85 "NNTPPermanentError", "NNTPProtocolError", "NNTPDataError", 86 "decode_header", 87 ] 88 89warnings._deprecated(__name__, remove=(3, 13)) 90 91# maximal line length when calling readline(). This is to prevent 92# reading arbitrary length lines. RFC 3977 limits NNTP line length to 93# 512 characters, including CRLF. We have selected 2048 just to be on 94# the safe side. 95_MAXLINE = 2048 96 97 98# Exceptions raised when an error or invalid response is received 99class NNTPError(Exception): 100 """Base class for all nntplib exceptions""" 101 def __init__(self, *args): 102 Exception.__init__(self, *args) 103 try: 104 self.response = args[0] 105 except IndexError: 106 self.response = 'No response given' 107 108class NNTPReplyError(NNTPError): 109 """Unexpected [123]xx reply""" 110 pass 111 112class NNTPTemporaryError(NNTPError): 113 """4xx errors""" 114 pass 115 116class NNTPPermanentError(NNTPError): 117 """5xx errors""" 118 pass 119 120class NNTPProtocolError(NNTPError): 121 """Response does not begin with [1-5]""" 122 pass 123 124class NNTPDataError(NNTPError): 125 """Error in response data""" 126 pass 127 128 129# Standard port used by NNTP servers 130NNTP_PORT = 119 131NNTP_SSL_PORT = 563 132 133# Response numbers that are followed by additional text (e.g. article) 134_LONGRESP = { 135 '100', # HELP 136 '101', # CAPABILITIES 137 '211', # LISTGROUP (also not multi-line with GROUP) 138 '215', # LIST 139 '220', # ARTICLE 140 '221', # HEAD, XHDR 141 '222', # BODY 142 '224', # OVER, XOVER 143 '225', # HDR 144 '230', # NEWNEWS 145 '231', # NEWGROUPS 146 '282', # XGTITLE 147} 148 149# Default decoded value for LIST OVERVIEW.FMT if not supported 150_DEFAULT_OVERVIEW_FMT = [ 151 "subject", "from", "date", "message-id", "references", ":bytes", ":lines"] 152 153# Alternative names allowed in LIST OVERVIEW.FMT response 154_OVERVIEW_FMT_ALTERNATIVES = { 155 'bytes': ':bytes', 156 'lines': ':lines', 157} 158 159# Line terminators (we always output CRLF, but accept any of CRLF, CR, LF) 160_CRLF = b'\r\n' 161 162GroupInfo = collections.namedtuple('GroupInfo', 163 ['group', 'last', 'first', 'flag']) 164 165ArticleInfo = collections.namedtuple('ArticleInfo', 166 ['number', 'message_id', 'lines']) 167 168 169# Helper function(s) 170def decode_header(header_str): 171 """Takes a unicode string representing a munged header value 172 and decodes it as a (possibly non-ASCII) readable value.""" 173 parts = [] 174 for v, enc in _email_decode_header(header_str): 175 if isinstance(v, bytes): 176 parts.append(v.decode(enc or 'ascii')) 177 else: 178 parts.append(v) 179 return ''.join(parts) 180 181def _parse_overview_fmt(lines): 182 """Parse a list of string representing the response to LIST OVERVIEW.FMT 183 and return a list of header/metadata names. 184 Raises NNTPDataError if the response is not compliant 185 (cf. RFC 3977, section 8.4).""" 186 fmt = [] 187 for line in lines: 188 if line[0] == ':': 189 # Metadata name (e.g. ":bytes") 190 name, _, suffix = line[1:].partition(':') 191 name = ':' + name 192 else: 193 # Header name (e.g. "Subject:" or "Xref:full") 194 name, _, suffix = line.partition(':') 195 name = name.lower() 196 name = _OVERVIEW_FMT_ALTERNATIVES.get(name, name) 197 # Should we do something with the suffix? 198 fmt.append(name) 199 defaults = _DEFAULT_OVERVIEW_FMT 200 if len(fmt) < len(defaults): 201 raise NNTPDataError("LIST OVERVIEW.FMT response too short") 202 if fmt[:len(defaults)] != defaults: 203 raise NNTPDataError("LIST OVERVIEW.FMT redefines default fields") 204 return fmt 205 206def _parse_overview(lines, fmt, data_process_func=None): 207 """Parse the response to an OVER or XOVER command according to the 208 overview format `fmt`.""" 209 n_defaults = len(_DEFAULT_OVERVIEW_FMT) 210 overview = [] 211 for line in lines: 212 fields = {} 213 article_number, *tokens = line.split('\t') 214 article_number = int(article_number) 215 for i, token in enumerate(tokens): 216 if i >= len(fmt): 217 # XXX should we raise an error? Some servers might not 218 # support LIST OVERVIEW.FMT and still return additional 219 # headers. 220 continue 221 field_name = fmt[i] 222 is_metadata = field_name.startswith(':') 223 if i >= n_defaults and not is_metadata: 224 # Non-default header names are included in full in the response 225 # (unless the field is totally empty) 226 h = field_name + ": " 227 if token and token[:len(h)].lower() != h: 228 raise NNTPDataError("OVER/XOVER response doesn't include " 229 "names of additional headers") 230 token = token[len(h):] if token else None 231 fields[fmt[i]] = token 232 overview.append((article_number, fields)) 233 return overview 234 235def _parse_datetime(date_str, time_str=None): 236 """Parse a pair of (date, time) strings, and return a datetime object. 237 If only the date is given, it is assumed to be date and time 238 concatenated together (e.g. response to the DATE command). 239 """ 240 if time_str is None: 241 time_str = date_str[-6:] 242 date_str = date_str[:-6] 243 hours = int(time_str[:2]) 244 minutes = int(time_str[2:4]) 245 seconds = int(time_str[4:]) 246 year = int(date_str[:-4]) 247 month = int(date_str[-4:-2]) 248 day = int(date_str[-2:]) 249 # RFC 3977 doesn't say how to interpret 2-char years. Assume that 250 # there are no dates before 1970 on Usenet. 251 if year < 70: 252 year += 2000 253 elif year < 100: 254 year += 1900 255 return datetime.datetime(year, month, day, hours, minutes, seconds) 256 257def _unparse_datetime(dt, legacy=False): 258 """Format a date or datetime object as a pair of (date, time) strings 259 in the format required by the NEWNEWS and NEWGROUPS commands. If a 260 date object is passed, the time is assumed to be midnight (00h00). 261 262 The returned representation depends on the legacy flag: 263 * if legacy is False (the default): 264 date has the YYYYMMDD format and time the HHMMSS format 265 * if legacy is True: 266 date has the YYMMDD format and time the HHMMSS format. 267 RFC 3977 compliant servers should understand both formats; therefore, 268 legacy is only needed when talking to old servers. 269 """ 270 if not isinstance(dt, datetime.datetime): 271 time_str = "000000" 272 else: 273 time_str = "{0.hour:02d}{0.minute:02d}{0.second:02d}".format(dt) 274 y = dt.year 275 if legacy: 276 y = y % 100 277 date_str = "{0:02d}{1.month:02d}{1.day:02d}".format(y, dt) 278 else: 279 date_str = "{0:04d}{1.month:02d}{1.day:02d}".format(y, dt) 280 return date_str, time_str 281 282 283if _have_ssl: 284 285 def _encrypt_on(sock, context, hostname): 286 """Wrap a socket in SSL/TLS. Arguments: 287 - sock: Socket to wrap 288 - context: SSL context to use for the encrypted connection 289 Returns: 290 - sock: New, encrypted socket. 291 """ 292 # Generate a default SSL context if none was passed. 293 if context is None: 294 context = ssl._create_stdlib_context() 295 return context.wrap_socket(sock, server_hostname=hostname) 296 297 298# The classes themselves 299class NNTP: 300 # UTF-8 is the character set for all NNTP commands and responses: they 301 # are automatically encoded (when sending) and decoded (and receiving) 302 # by this class. 303 # However, some multi-line data blocks can contain arbitrary bytes (for 304 # example, latin-1 or utf-16 data in the body of a message). Commands 305 # taking (POST, IHAVE) or returning (HEAD, BODY, ARTICLE) raw message 306 # data will therefore only accept and produce bytes objects. 307 # Furthermore, since there could be non-compliant servers out there, 308 # we use 'surrogateescape' as the error handler for fault tolerance 309 # and easy round-tripping. This could be useful for some applications 310 # (e.g. NNTP gateways). 311 312 encoding = 'utf-8' 313 errors = 'surrogateescape' 314 315 def __init__(self, host, port=NNTP_PORT, user=None, password=None, 316 readermode=None, usenetrc=False, 317 timeout=_GLOBAL_DEFAULT_TIMEOUT): 318 """Initialize an instance. Arguments: 319 - host: hostname to connect to 320 - port: port to connect to (default the standard NNTP port) 321 - user: username to authenticate with 322 - password: password to use with username 323 - readermode: if true, send 'mode reader' command after 324 connecting. 325 - usenetrc: allow loading username and password from ~/.netrc file 326 if not specified explicitly 327 - timeout: timeout (in seconds) used for socket connections 328 329 readermode is sometimes necessary if you are connecting to an 330 NNTP server on the local machine and intend to call 331 reader-specific commands, such as `group'. If you get 332 unexpected NNTPPermanentErrors, you might need to set 333 readermode. 334 """ 335 self.host = host 336 self.port = port 337 self.sock = self._create_socket(timeout) 338 self.file = None 339 try: 340 self.file = self.sock.makefile("rwb") 341 self._base_init(readermode) 342 if user or usenetrc: 343 self.login(user, password, usenetrc) 344 except: 345 if self.file: 346 self.file.close() 347 self.sock.close() 348 raise 349 350 def _base_init(self, readermode): 351 """Partial initialization for the NNTP protocol. 352 This instance method is extracted for supporting the test code. 353 """ 354 self.debugging = 0 355 self.welcome = self._getresp() 356 357 # Inquire about capabilities (RFC 3977). 358 self._caps = None 359 self.getcapabilities() 360 361 # 'MODE READER' is sometimes necessary to enable 'reader' mode. 362 # However, the order in which 'MODE READER' and 'AUTHINFO' need to 363 # arrive differs between some NNTP servers. If _setreadermode() fails 364 # with an authorization failed error, it will set this to True; 365 # the login() routine will interpret that as a request to try again 366 # after performing its normal function. 367 # Enable only if we're not already in READER mode anyway. 368 self.readermode_afterauth = False 369 if readermode and 'READER' not in self._caps: 370 self._setreadermode() 371 if not self.readermode_afterauth: 372 # Capabilities might have changed after MODE READER 373 self._caps = None 374 self.getcapabilities() 375 376 # RFC 4642 2.2.2: Both the client and the server MUST know if there is 377 # a TLS session active. A client MUST NOT attempt to start a TLS 378 # session if a TLS session is already active. 379 self.tls_on = False 380 381 # Log in and encryption setup order is left to subclasses. 382 self.authenticated = False 383 384 def __enter__(self): 385 return self 386 387 def __exit__(self, *args): 388 is_connected = lambda: hasattr(self, "file") 389 if is_connected(): 390 try: 391 self.quit() 392 except (OSError, EOFError): 393 pass 394 finally: 395 if is_connected(): 396 self._close() 397 398 def _create_socket(self, timeout): 399 if timeout is not None and not timeout: 400 raise ValueError('Non-blocking socket (timeout=0) is not supported') 401 sys.audit("nntplib.connect", self, self.host, self.port) 402 return socket.create_connection((self.host, self.port), timeout) 403 404 def getwelcome(self): 405 """Get the welcome message from the server 406 (this is read and squirreled away by __init__()). 407 If the response code is 200, posting is allowed; 408 if it 201, posting is not allowed.""" 409 410 if self.debugging: print('*welcome*', repr(self.welcome)) 411 return self.welcome 412 413 def getcapabilities(self): 414 """Get the server capabilities, as read by __init__(). 415 If the CAPABILITIES command is not supported, an empty dict is 416 returned.""" 417 if self._caps is None: 418 self.nntp_version = 1 419 self.nntp_implementation = None 420 try: 421 resp, caps = self.capabilities() 422 except (NNTPPermanentError, NNTPTemporaryError): 423 # Server doesn't support capabilities 424 self._caps = {} 425 else: 426 self._caps = caps 427 if 'VERSION' in caps: 428 # The server can advertise several supported versions, 429 # choose the highest. 430 self.nntp_version = max(map(int, caps['VERSION'])) 431 if 'IMPLEMENTATION' in caps: 432 self.nntp_implementation = ' '.join(caps['IMPLEMENTATION']) 433 return self._caps 434 435 def set_debuglevel(self, level): 436 """Set the debugging level. Argument 'level' means: 437 0: no debugging output (default) 438 1: print commands and responses but not body text etc. 439 2: also print raw lines read and sent before stripping CR/LF""" 440 441 self.debugging = level 442 debug = set_debuglevel 443 444 def _putline(self, line): 445 """Internal: send one line to the server, appending CRLF. 446 The `line` must be a bytes-like object.""" 447 sys.audit("nntplib.putline", self, line) 448 line = line + _CRLF 449 if self.debugging > 1: print('*put*', repr(line)) 450 self.file.write(line) 451 self.file.flush() 452 453 def _putcmd(self, line): 454 """Internal: send one command to the server (through _putline()). 455 The `line` must be a unicode string.""" 456 if self.debugging: print('*cmd*', repr(line)) 457 line = line.encode(self.encoding, self.errors) 458 self._putline(line) 459 460 def _getline(self, strip_crlf=True): 461 """Internal: return one line from the server, stripping _CRLF. 462 Raise EOFError if the connection is closed. 463 Returns a bytes object.""" 464 line = self.file.readline(_MAXLINE +1) 465 if len(line) > _MAXLINE: 466 raise NNTPDataError('line too long') 467 if self.debugging > 1: 468 print('*get*', repr(line)) 469 if not line: raise EOFError 470 if strip_crlf: 471 if line[-2:] == _CRLF: 472 line = line[:-2] 473 elif line[-1:] in _CRLF: 474 line = line[:-1] 475 return line 476 477 def _getresp(self): 478 """Internal: get a response from the server. 479 Raise various errors if the response indicates an error. 480 Returns a unicode string.""" 481 resp = self._getline() 482 if self.debugging: print('*resp*', repr(resp)) 483 resp = resp.decode(self.encoding, self.errors) 484 c = resp[:1] 485 if c == '4': 486 raise NNTPTemporaryError(resp) 487 if c == '5': 488 raise NNTPPermanentError(resp) 489 if c not in '123': 490 raise NNTPProtocolError(resp) 491 return resp 492 493 def _getlongresp(self, file=None): 494 """Internal: get a response plus following text from the server. 495 Raise various errors if the response indicates an error. 496 497 Returns a (response, lines) tuple where `response` is a unicode 498 string and `lines` is a list of bytes objects. 499 If `file` is a file-like object, it must be open in binary mode. 500 """ 501 502 openedFile = None 503 try: 504 # If a string was passed then open a file with that name 505 if isinstance(file, (str, bytes)): 506 openedFile = file = open(file, "wb") 507 508 resp = self._getresp() 509 if resp[:3] not in _LONGRESP: 510 raise NNTPReplyError(resp) 511 512 lines = [] 513 if file is not None: 514 # XXX lines = None instead? 515 terminators = (b'.' + _CRLF, b'.\n') 516 while 1: 517 line = self._getline(False) 518 if line in terminators: 519 break 520 if line.startswith(b'..'): 521 line = line[1:] 522 file.write(line) 523 else: 524 terminator = b'.' 525 while 1: 526 line = self._getline() 527 if line == terminator: 528 break 529 if line.startswith(b'..'): 530 line = line[1:] 531 lines.append(line) 532 finally: 533 # If this method created the file, then it must close it 534 if openedFile: 535 openedFile.close() 536 537 return resp, lines 538 539 def _shortcmd(self, line): 540 """Internal: send a command and get the response. 541 Same return value as _getresp().""" 542 self._putcmd(line) 543 return self._getresp() 544 545 def _longcmd(self, line, file=None): 546 """Internal: send a command and get the response plus following text. 547 Same return value as _getlongresp().""" 548 self._putcmd(line) 549 return self._getlongresp(file) 550 551 def _longcmdstring(self, line, file=None): 552 """Internal: send a command and get the response plus following text. 553 Same as _longcmd() and _getlongresp(), except that the returned `lines` 554 are unicode strings rather than bytes objects. 555 """ 556 self._putcmd(line) 557 resp, list = self._getlongresp(file) 558 return resp, [line.decode(self.encoding, self.errors) 559 for line in list] 560 561 def _getoverviewfmt(self): 562 """Internal: get the overview format. Queries the server if not 563 already done, else returns the cached value.""" 564 try: 565 return self._cachedoverviewfmt 566 except AttributeError: 567 pass 568 try: 569 resp, lines = self._longcmdstring("LIST OVERVIEW.FMT") 570 except NNTPPermanentError: 571 # Not supported by server? 572 fmt = _DEFAULT_OVERVIEW_FMT[:] 573 else: 574 fmt = _parse_overview_fmt(lines) 575 self._cachedoverviewfmt = fmt 576 return fmt 577 578 def _grouplist(self, lines): 579 # Parse lines into "group last first flag" 580 return [GroupInfo(*line.split()) for line in lines] 581 582 def capabilities(self): 583 """Process a CAPABILITIES command. Not supported by all servers. 584 Return: 585 - resp: server response if successful 586 - caps: a dictionary mapping capability names to lists of tokens 587 (for example {'VERSION': ['2'], 'OVER': [], LIST: ['ACTIVE', 'HEADERS'] }) 588 """ 589 caps = {} 590 resp, lines = self._longcmdstring("CAPABILITIES") 591 for line in lines: 592 name, *tokens = line.split() 593 caps[name] = tokens 594 return resp, caps 595 596 def newgroups(self, date, *, file=None): 597 """Process a NEWGROUPS command. Arguments: 598 - date: a date or datetime object 599 Return: 600 - resp: server response if successful 601 - list: list of newsgroup names 602 """ 603 if not isinstance(date, (datetime.date, datetime.date)): 604 raise TypeError( 605 "the date parameter must be a date or datetime object, " 606 "not '{:40}'".format(date.__class__.__name__)) 607 date_str, time_str = _unparse_datetime(date, self.nntp_version < 2) 608 cmd = 'NEWGROUPS {0} {1}'.format(date_str, time_str) 609 resp, lines = self._longcmdstring(cmd, file) 610 return resp, self._grouplist(lines) 611 612 def newnews(self, group, date, *, file=None): 613 """Process a NEWNEWS command. Arguments: 614 - group: group name or '*' 615 - date: a date or datetime object 616 Return: 617 - resp: server response if successful 618 - list: list of message ids 619 """ 620 if not isinstance(date, (datetime.date, datetime.date)): 621 raise TypeError( 622 "the date parameter must be a date or datetime object, " 623 "not '{:40}'".format(date.__class__.__name__)) 624 date_str, time_str = _unparse_datetime(date, self.nntp_version < 2) 625 cmd = 'NEWNEWS {0} {1} {2}'.format(group, date_str, time_str) 626 return self._longcmdstring(cmd, file) 627 628 def list(self, group_pattern=None, *, file=None): 629 """Process a LIST or LIST ACTIVE command. Arguments: 630 - group_pattern: a pattern indicating which groups to query 631 - file: Filename string or file object to store the result in 632 Returns: 633 - resp: server response if successful 634 - list: list of (group, last, first, flag) (strings) 635 """ 636 if group_pattern is not None: 637 command = 'LIST ACTIVE ' + group_pattern 638 else: 639 command = 'LIST' 640 resp, lines = self._longcmdstring(command, file) 641 return resp, self._grouplist(lines) 642 643 def _getdescriptions(self, group_pattern, return_all): 644 line_pat = re.compile('^(?P<group>[^ \t]+)[ \t]+(.*)$') 645 # Try the more std (acc. to RFC2980) LIST NEWSGROUPS first 646 resp, lines = self._longcmdstring('LIST NEWSGROUPS ' + group_pattern) 647 if not resp.startswith('215'): 648 # Now the deprecated XGTITLE. This either raises an error 649 # or succeeds with the same output structure as LIST 650 # NEWSGROUPS. 651 resp, lines = self._longcmdstring('XGTITLE ' + group_pattern) 652 groups = {} 653 for raw_line in lines: 654 match = line_pat.search(raw_line.strip()) 655 if match: 656 name, desc = match.group(1, 2) 657 if not return_all: 658 return desc 659 groups[name] = desc 660 if return_all: 661 return resp, groups 662 else: 663 # Nothing found 664 return '' 665 666 def description(self, group): 667 """Get a description for a single group. If more than one 668 group matches ('group' is a pattern), return the first. If no 669 group matches, return an empty string. 670 671 This elides the response code from the server, since it can 672 only be '215' or '285' (for xgtitle) anyway. If the response 673 code is needed, use the 'descriptions' method. 674 675 NOTE: This neither checks for a wildcard in 'group' nor does 676 it check whether the group actually exists.""" 677 return self._getdescriptions(group, False) 678 679 def descriptions(self, group_pattern): 680 """Get descriptions for a range of groups.""" 681 return self._getdescriptions(group_pattern, True) 682 683 def group(self, name): 684 """Process a GROUP command. Argument: 685 - group: the group name 686 Returns: 687 - resp: server response if successful 688 - count: number of articles 689 - first: first article number 690 - last: last article number 691 - name: the group name 692 """ 693 resp = self._shortcmd('GROUP ' + name) 694 if not resp.startswith('211'): 695 raise NNTPReplyError(resp) 696 words = resp.split() 697 count = first = last = 0 698 n = len(words) 699 if n > 1: 700 count = words[1] 701 if n > 2: 702 first = words[2] 703 if n > 3: 704 last = words[3] 705 if n > 4: 706 name = words[4].lower() 707 return resp, int(count), int(first), int(last), name 708 709 def help(self, *, file=None): 710 """Process a HELP command. Argument: 711 - file: Filename string or file object to store the result in 712 Returns: 713 - resp: server response if successful 714 - list: list of strings returned by the server in response to the 715 HELP command 716 """ 717 return self._longcmdstring('HELP', file) 718 719 def _statparse(self, resp): 720 """Internal: parse the response line of a STAT, NEXT, LAST, 721 ARTICLE, HEAD or BODY command.""" 722 if not resp.startswith('22'): 723 raise NNTPReplyError(resp) 724 words = resp.split() 725 art_num = int(words[1]) 726 message_id = words[2] 727 return resp, art_num, message_id 728 729 def _statcmd(self, line): 730 """Internal: process a STAT, NEXT or LAST command.""" 731 resp = self._shortcmd(line) 732 return self._statparse(resp) 733 734 def stat(self, message_spec=None): 735 """Process a STAT command. Argument: 736 - message_spec: article number or message id (if not specified, 737 the current article is selected) 738 Returns: 739 - resp: server response if successful 740 - art_num: the article number 741 - message_id: the message id 742 """ 743 if message_spec: 744 return self._statcmd('STAT {0}'.format(message_spec)) 745 else: 746 return self._statcmd('STAT') 747 748 def next(self): 749 """Process a NEXT command. No arguments. Return as for STAT.""" 750 return self._statcmd('NEXT') 751 752 def last(self): 753 """Process a LAST command. No arguments. Return as for STAT.""" 754 return self._statcmd('LAST') 755 756 def _artcmd(self, line, file=None): 757 """Internal: process a HEAD, BODY or ARTICLE command.""" 758 resp, lines = self._longcmd(line, file) 759 resp, art_num, message_id = self._statparse(resp) 760 return resp, ArticleInfo(art_num, message_id, lines) 761 762 def head(self, message_spec=None, *, file=None): 763 """Process a HEAD command. Argument: 764 - message_spec: article number or message id 765 - file: filename string or file object to store the headers in 766 Returns: 767 - resp: server response if successful 768 - ArticleInfo: (article number, message id, list of header lines) 769 """ 770 if message_spec is not None: 771 cmd = 'HEAD {0}'.format(message_spec) 772 else: 773 cmd = 'HEAD' 774 return self._artcmd(cmd, file) 775 776 def body(self, message_spec=None, *, file=None): 777 """Process a BODY command. Argument: 778 - message_spec: article number or message id 779 - file: filename string or file object to store the body in 780 Returns: 781 - resp: server response if successful 782 - ArticleInfo: (article number, message id, list of body lines) 783 """ 784 if message_spec is not None: 785 cmd = 'BODY {0}'.format(message_spec) 786 else: 787 cmd = 'BODY' 788 return self._artcmd(cmd, file) 789 790 def article(self, message_spec=None, *, file=None): 791 """Process an ARTICLE command. Argument: 792 - message_spec: article number or message id 793 - file: filename string or file object to store the article in 794 Returns: 795 - resp: server response if successful 796 - ArticleInfo: (article number, message id, list of article lines) 797 """ 798 if message_spec is not None: 799 cmd = 'ARTICLE {0}'.format(message_spec) 800 else: 801 cmd = 'ARTICLE' 802 return self._artcmd(cmd, file) 803 804 def slave(self): 805 """Process a SLAVE command. Returns: 806 - resp: server response if successful 807 """ 808 return self._shortcmd('SLAVE') 809 810 def xhdr(self, hdr, str, *, file=None): 811 """Process an XHDR command (optional server extension). Arguments: 812 - hdr: the header type (e.g. 'subject') 813 - str: an article nr, a message id, or a range nr1-nr2 814 - file: Filename string or file object to store the result in 815 Returns: 816 - resp: server response if successful 817 - list: list of (nr, value) strings 818 """ 819 pat = re.compile('^([0-9]+) ?(.*)\n?') 820 resp, lines = self._longcmdstring('XHDR {0} {1}'.format(hdr, str), file) 821 def remove_number(line): 822 m = pat.match(line) 823 return m.group(1, 2) if m else line 824 return resp, [remove_number(line) for line in lines] 825 826 def xover(self, start, end, *, file=None): 827 """Process an XOVER command (optional server extension) Arguments: 828 - start: start of range 829 - end: end of range 830 - file: Filename string or file object to store the result in 831 Returns: 832 - resp: server response if successful 833 - list: list of dicts containing the response fields 834 """ 835 resp, lines = self._longcmdstring('XOVER {0}-{1}'.format(start, end), 836 file) 837 fmt = self._getoverviewfmt() 838 return resp, _parse_overview(lines, fmt) 839 840 def over(self, message_spec, *, file=None): 841 """Process an OVER command. If the command isn't supported, fall 842 back to XOVER. Arguments: 843 - message_spec: 844 - either a message id, indicating the article to fetch 845 information about 846 - or a (start, end) tuple, indicating a range of article numbers; 847 if end is None, information up to the newest message will be 848 retrieved 849 - or None, indicating the current article number must be used 850 - file: Filename string or file object to store the result in 851 Returns: 852 - resp: server response if successful 853 - list: list of dicts containing the response fields 854 855 NOTE: the "message id" form isn't supported by XOVER 856 """ 857 cmd = 'OVER' if 'OVER' in self._caps else 'XOVER' 858 if isinstance(message_spec, (tuple, list)): 859 start, end = message_spec 860 cmd += ' {0}-{1}'.format(start, end or '') 861 elif message_spec is not None: 862 cmd = cmd + ' ' + message_spec 863 resp, lines = self._longcmdstring(cmd, file) 864 fmt = self._getoverviewfmt() 865 return resp, _parse_overview(lines, fmt) 866 867 def date(self): 868 """Process the DATE command. 869 Returns: 870 - resp: server response if successful 871 - date: datetime object 872 """ 873 resp = self._shortcmd("DATE") 874 if not resp.startswith('111'): 875 raise NNTPReplyError(resp) 876 elem = resp.split() 877 if len(elem) != 2: 878 raise NNTPDataError(resp) 879 date = elem[1] 880 if len(date) != 14: 881 raise NNTPDataError(resp) 882 return resp, _parse_datetime(date, None) 883 884 def _post(self, command, f): 885 resp = self._shortcmd(command) 886 # Raises a specific exception if posting is not allowed 887 if not resp.startswith('3'): 888 raise NNTPReplyError(resp) 889 if isinstance(f, (bytes, bytearray)): 890 f = f.splitlines() 891 # We don't use _putline() because: 892 # - we don't want additional CRLF if the file or iterable is already 893 # in the right format 894 # - we don't want a spurious flush() after each line is written 895 for line in f: 896 if not line.endswith(_CRLF): 897 line = line.rstrip(b"\r\n") + _CRLF 898 if line.startswith(b'.'): 899 line = b'.' + line 900 self.file.write(line) 901 self.file.write(b".\r\n") 902 self.file.flush() 903 return self._getresp() 904 905 def post(self, data): 906 """Process a POST command. Arguments: 907 - data: bytes object, iterable or file containing the article 908 Returns: 909 - resp: server response if successful""" 910 return self._post('POST', data) 911 912 def ihave(self, message_id, data): 913 """Process an IHAVE command. Arguments: 914 - message_id: message-id of the article 915 - data: file containing the article 916 Returns: 917 - resp: server response if successful 918 Note that if the server refuses the article an exception is raised.""" 919 return self._post('IHAVE {0}'.format(message_id), data) 920 921 def _close(self): 922 try: 923 if self.file: 924 self.file.close() 925 del self.file 926 finally: 927 self.sock.close() 928 929 def quit(self): 930 """Process a QUIT command and close the socket. Returns: 931 - resp: server response if successful""" 932 try: 933 resp = self._shortcmd('QUIT') 934 finally: 935 self._close() 936 return resp 937 938 def login(self, user=None, password=None, usenetrc=True): 939 if self.authenticated: 940 raise ValueError("Already logged in.") 941 if not user and not usenetrc: 942 raise ValueError( 943 "At least one of `user` and `usenetrc` must be specified") 944 # If no login/password was specified but netrc was requested, 945 # try to get them from ~/.netrc 946 # Presume that if .netrc has an entry, NNRP authentication is required. 947 try: 948 if usenetrc and not user: 949 import netrc 950 credentials = netrc.netrc() 951 auth = credentials.authenticators(self.host) 952 if auth: 953 user = auth[0] 954 password = auth[2] 955 except OSError: 956 pass 957 # Perform NNTP authentication if needed. 958 if not user: 959 return 960 resp = self._shortcmd('authinfo user ' + user) 961 if resp.startswith('381'): 962 if not password: 963 raise NNTPReplyError(resp) 964 else: 965 resp = self._shortcmd('authinfo pass ' + password) 966 if not resp.startswith('281'): 967 raise NNTPPermanentError(resp) 968 # Capabilities might have changed after login 969 self._caps = None 970 self.getcapabilities() 971 # Attempt to send mode reader if it was requested after login. 972 # Only do so if we're not in reader mode already. 973 if self.readermode_afterauth and 'READER' not in self._caps: 974 self._setreadermode() 975 # Capabilities might have changed after MODE READER 976 self._caps = None 977 self.getcapabilities() 978 979 def _setreadermode(self): 980 try: 981 self.welcome = self._shortcmd('mode reader') 982 except NNTPPermanentError: 983 # Error 5xx, probably 'not implemented' 984 pass 985 except NNTPTemporaryError as e: 986 if e.response.startswith('480'): 987 # Need authorization before 'mode reader' 988 self.readermode_afterauth = True 989 else: 990 raise 991 992 if _have_ssl: 993 def starttls(self, context=None): 994 """Process a STARTTLS command. Arguments: 995 - context: SSL context to use for the encrypted connection 996 """ 997 # Per RFC 4642, STARTTLS MUST NOT be sent after authentication or if 998 # a TLS session already exists. 999 if self.tls_on: 1000 raise ValueError("TLS is already enabled.") 1001 if self.authenticated: 1002 raise ValueError("TLS cannot be started after authentication.") 1003 resp = self._shortcmd('STARTTLS') 1004 if resp.startswith('382'): 1005 self.file.close() 1006 self.sock = _encrypt_on(self.sock, context, self.host) 1007 self.file = self.sock.makefile("rwb") 1008 self.tls_on = True 1009 # Capabilities may change after TLS starts up, so ask for them 1010 # again. 1011 self._caps = None 1012 self.getcapabilities() 1013 else: 1014 raise NNTPError("TLS failed to start.") 1015 1016 1017if _have_ssl: 1018 class NNTP_SSL(NNTP): 1019 1020 def __init__(self, host, port=NNTP_SSL_PORT, 1021 user=None, password=None, ssl_context=None, 1022 readermode=None, usenetrc=False, 1023 timeout=_GLOBAL_DEFAULT_TIMEOUT): 1024 """This works identically to NNTP.__init__, except for the change 1025 in default port and the `ssl_context` argument for SSL connections. 1026 """ 1027 self.ssl_context = ssl_context 1028 super().__init__(host, port, user, password, readermode, 1029 usenetrc, timeout) 1030 1031 def _create_socket(self, timeout): 1032 sock = super()._create_socket(timeout) 1033 try: 1034 sock = _encrypt_on(sock, self.ssl_context, self.host) 1035 except: 1036 sock.close() 1037 raise 1038 else: 1039 return sock 1040 1041 __all__.append("NNTP_SSL") 1042 1043 1044# Test retrieval when run as a script. 1045if __name__ == '__main__': 1046 import argparse 1047 1048 parser = argparse.ArgumentParser(description="""\ 1049 nntplib built-in demo - display the latest articles in a newsgroup""") 1050 parser.add_argument('-g', '--group', default='gmane.comp.python.general', 1051 help='group to fetch messages from (default: %(default)s)') 1052 parser.add_argument('-s', '--server', default='news.gmane.io', 1053 help='NNTP server hostname (default: %(default)s)') 1054 parser.add_argument('-p', '--port', default=-1, type=int, 1055 help='NNTP port number (default: %s / %s)' % (NNTP_PORT, NNTP_SSL_PORT)) 1056 parser.add_argument('-n', '--nb-articles', default=10, type=int, 1057 help='number of articles to fetch (default: %(default)s)') 1058 parser.add_argument('-S', '--ssl', action='store_true', default=False, 1059 help='use NNTP over SSL') 1060 args = parser.parse_args() 1061 1062 port = args.port 1063 if not args.ssl: 1064 if port == -1: 1065 port = NNTP_PORT 1066 s = NNTP(host=args.server, port=port) 1067 else: 1068 if port == -1: 1069 port = NNTP_SSL_PORT 1070 s = NNTP_SSL(host=args.server, port=port) 1071 1072 caps = s.getcapabilities() 1073 if 'STARTTLS' in caps: 1074 s.starttls() 1075 resp, count, first, last, name = s.group(args.group) 1076 print('Group', name, 'has', count, 'articles, range', first, 'to', last) 1077 1078 def cut(s, lim): 1079 if len(s) > lim: 1080 s = s[:lim - 4] + "..." 1081 return s 1082 1083 first = str(int(last) - args.nb_articles + 1) 1084 resp, overviews = s.xover(first, last) 1085 for artnum, over in overviews: 1086 author = decode_header(over['from']).split('<', 1)[0] 1087 subject = decode_header(over['subject']) 1088 lines = int(over[':lines']) 1089 print("{:7} {:20} {:42} ({})".format( 1090 artnum, cut(author, 20), cut(subject, 42), lines) 1091 ) 1092 1093 s.quit() 1094