diff -r f16f0500b49b Doc/library/netrc.rst --- a/Doc/library/netrc.rst Fri Nov 25 17:38:13 2016 +0300 +++ b/Doc/library/netrc.rst Sat Nov 26 19:01:42 2016 +0800 @@ -18,7 +18,7 @@ .. class:: netrc([file]) - A :class:`netrc` instance or subclass instance encapsulates data from a netrc + A :class:`netrc` instance or subclass instance encapsulates data from a netrc file. The initialization argument, if present, specifies the file to parse. If no argument is given, the file :file:`.netrc` in the user's home directory will be read. Parse errors will raise :exc:`NetrcParseError` with diagnostic @@ -32,6 +32,12 @@ .. versionchanged:: 3.4 Added the POSIX permission check. + .. versionchanged:: 3.7 + The entry in the netrc file no longer needs to contain all tokens. The missing + tokens' value default to an empty string. All the tokens and their values now + can contain arbitrary characters, like whitespace and non-ASCII characters. + If the login name is anonymous, it won't trigger the security check. + .. exception:: NetrcParseError @@ -75,11 +81,3 @@ .. attribute:: netrc.macros Dictionary mapping macro names to string lists. - -.. note:: - - Passwords are limited to a subset of the ASCII character set. All ASCII - punctuation is allowed in passwords, however, note that whitespace and - non-printable characters are not allowed in passwords. This is a limitation - of the way the .netrc file is parsed and may be removed in the future. - diff -r f16f0500b49b Lib/netrc.py --- a/Lib/netrc.py Fri Nov 25 17:38:13 2016 +0300 +++ b/Lib/netrc.py Sat Nov 26 19:01:42 2016 +0800 @@ -19,6 +19,51 @@ return "%s (%s, line %s)" % (self.msg, self.filename, self.lineno) +class _netrclex: + def __init__(self, fp): + self.lineno = 1 + self.instream = fp + self.whitespace = "\n\t\r " + self._stack = [] + + def _read_char(self): + ch = self.instream.read(1) + if ch == "\n": + self.lineno += 1 + return ch + + def get_token(self): + if self._stack: + return self._stack.pop(0) + token = "" + fiter = iter(self._read_char, "") + for ch in fiter: + if ch in self.whitespace: + continue + if ch == "\"": + for ch in fiter: + if ch != "\"": + if ch == "\\": + ch = self._read_char() + token += ch + continue + return token + else: + if ch == "\\": + ch = self._read_char() + token += ch + for ch in fiter: + if ch not in self.whitespace: + if ch == "\\": + ch = self._read_char() + token += ch + continue + return token + return token + + def push_token(self, token): + self._stack.append(token) + class netrc: def __init__(self, file=None): default_netrc = file is None @@ -33,31 +78,31 @@ self._parse(file, fp, default_netrc) def _parse(self, file, fp, default_netrc): - lexer = shlex.shlex(fp) - lexer.wordchars += r"""!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~""" - lexer.commenters = lexer.commenters.replace('#', '') + lexer = _netrclex(fp) while 1: # Look for a machine, default, or macdef top-level keyword - saved_lineno = lexer.lineno - toplevel = tt = lexer.get_token() + prev_lineno = lexer.lineno + tt = lexer.get_token() if not tt: break elif tt[0] == '#': - if lexer.lineno == saved_lineno and len(tt) == 1: + if prev_lineno == lexer.lineno: lexer.instream.readline() continue elif tt == 'machine': entryname = lexer.get_token() elif tt == 'default': entryname = 'default' - elif tt == 'macdef': # Just skip to end of macdefs + elif tt == 'macdef': entryname = lexer.get_token() self.macros[entryname] = [] - lexer.whitespace = ' \t' while 1: line = lexer.instream.readline() - if not line or line == '\012': - lexer.whitespace = ' \t\r\n' + if not line: + raise NetrcParseError( + "Macro definition missing null line terminator.", + file, lexer.lineno) + if line == '\n': break self.macros[entryname].append(line) continue @@ -65,29 +110,29 @@ raise NetrcParseError( "bad toplevel token %r" % tt, file, lexer.lineno) + if not entryname: + raise NetrcParseError("missing %r name" % tt, file, lexer.lineno) + # We're looking at start of an entry for a named machine or default. - login = '' - account = password = None + login = account = password = '' self.hosts[entryname] = {} while 1: + prev_lineno = lexer.lineno tt = lexer.get_token() - if (tt.startswith('#') or - tt in {'', 'machine', 'default', 'macdef'}): - if password: - self.hosts[entryname] = (login, account, password) - lexer.push_token(tt) - break - else: - raise NetrcParseError( - "malformed %s entry %s terminated by %s" - % (toplevel, entryname, repr(tt)), - file, lexer.lineno) + if tt.startswith('#'): + if lexer.lineno == prev_lineno: + lexer.instream.readline() + continue + if tt in {'', 'machine', 'default', 'macdef'}: + self.hosts[entryname] = (login, account, password) + lexer.push_token(tt) + break elif tt == 'login' or tt == 'user': login = lexer.get_token() elif tt == 'account': account = lexer.get_token() elif tt == 'password': - if os.name == 'posix' and default_netrc: + if os.name == 'posix' and default_netrc and login != "anonymous": prop = os.fstat(fp.fileno()) if prop.st_uid != os.getuid(): import pwd @@ -127,10 +172,10 @@ rep = "" for host in self.hosts.keys(): attrs = self.hosts[host] - rep = rep + "machine "+ host + "\n\tlogin " + repr(attrs[0]) + "\n" + rep = rep + "machine "+ host + "\n\tlogin " + attrs[0] + "\n" if attrs[1]: - rep = rep + "account " + repr(attrs[1]) - rep = rep + "\tpassword " + repr(attrs[2]) + "\n" + rep = rep + "account " + attrs[1] + rep = rep + "\tpassword " + attrs[2] + "\n" for macro in self.macros.keys(): rep = rep + "macdef " + macro + "\n" for line in self.macros[macro]: diff -r f16f0500b49b Lib/test/test_netrc.py --- a/Lib/test/test_netrc.py Fri Nov 25 17:38:13 2016 +0300 +++ b/Lib/test/test_netrc.py Sat Nov 26 19:01:42 2016 +0800 @@ -12,20 +12,22 @@ mode += 't' with open(temp_filename, mode) as fp: fp.write(test_data) - self.addCleanup(os.unlink, temp_filename) - return netrc.netrc(temp_filename) + try: + nrc = netrc.netrc(temp_filename) + finally: + os.unlink(temp_filename) + return nrc - def test_default(self): + def test_toplevel_tokens(self): nrc = self.make_nrc("""\ - machine host1.domain.com login log1 password pass1 account acct1 - default login log2 password pass2 + machine host.domain.com login log1 password pass1 account acct1 + default login log2 password pass2 account acct2 """) - self.assertEqual(nrc.hosts['host1.domain.com'], - ('log1', 'acct1', 'pass1')) - self.assertEqual(nrc.hosts['default'], ('log2', None, 'pass2')) + self.assertEqual(nrc.hosts['host.domain.com'], ('log1', 'acct1', 'pass1')) + self.assertEqual(nrc.hosts['default'], ('log2', 'acct2', 'pass2')) def test_macros(self): - nrc = self.make_nrc("""\ + data = """\ macdef macro1 line1 line2 @@ -33,33 +35,151 @@ macdef macro2 line3 line4 - """) + + """ + nrc = self.make_nrc(data) self.assertEqual(nrc.macros, {'macro1': ['line1\n', 'line2\n'], 'macro2': ['line3\n', 'line4\n']}) + # strip the last \n + self.assertRaises(netrc.NetrcParseError, self.make_nrc, + data.rstrip(' ')[:-1]) - def _test_passwords(self, nrc, passwd): + def test_optional_tokens(self): + data = ( + "machine host.domain.com", + "machine host.domain.com login", + "machine host.domain.com account", + "machine host.domain.com password", + "machine host.domain.com login \"\" account", + "machine host.domain.com login \"\" password", + "machine host.domain.com account \"\" password" + ) + for item in data: + nrc = self.make_nrc(item) + self.assertEqual(nrc.hosts['host.domain.com'], ('', '', '')) + data = ( + "default", + "default login", + "default account", + "default password", + "default login \"\" account", + "default login \"\" password", + "default account \"\" password" + ) + for item in data: + nrc = self.make_nrc(item) + self.assertEqual(nrc.hosts['default'], ('', '', '')) + + def test_invalid_tokens(self): + data = ( + "invalid host.domain.com", + "machine host.domain.com invalid", + "machine host.domain.com login log password pass account acct invalid", + "default host.domain.com invalid", + "default host.domain.com login log password pass account acct invalid" + ) + for item in data: + self.assertRaises(netrc.NetrcParseError, self.make_nrc, item) + + def _test_token_x(self, nrc, token, value): nrc = self.make_nrc(nrc) - self.assertEqual(nrc.hosts['host.domain.com'], ('log', 'acct', passwd)) + if token == 'login': + self.assertEqual(nrc.hosts['host.domain.com'], (value, 'acct', 'pass')) + elif token == 'account': + self.assertEqual(nrc.hosts['host.domain.com'], ('log', value, 'pass')) + elif token == 'password': + self.assertEqual(nrc.hosts['host.domain.com'], ('log', 'acct', value)) - def test_password_with_leading_hash(self): - self._test_passwords("""\ + def test_token_value_quotes(self): + self._test_token_x("""\ + machine host.domain.com login "log" password pass account acct + """, 'login', 'log') + self._test_token_x("""\ + machine host.domain.com login log password pass account "acct" + """, 'account', 'acct') + self._test_token_x("""\ + machine host.domain.com login log password "pass" account acct + """, 'password', 'pass') + + def test_token_value_escape(self): + self._test_token_x("""\ + machine host.domain.com login \\"log password pass account acct + """, 'login', '"log') + self._test_token_x("""\ + machine host.domain.com login "\\"log" password pass account acct + """, 'login', '"log') + self._test_token_x("""\ + machine host.domain.com login log password pass account \\"acct + """, 'account', '"acct') + self._test_token_x("""\ + machine host.domain.com login log password pass account "\\"acct" + """, 'account', '"acct') + self._test_token_x("""\ + machine host.domain.com login log password \\"pass account acct + """, 'password', '"pass') + self._test_token_x("""\ + machine host.domain.com login log password "\\"pass" account acct + """, 'password', '"pass') + + def test_token_value_whitespace(self): + self._test_token_x("""\ + machine host.domain.com login "lo g" password pass account acct + """, 'login', 'lo g') + self._test_token_x("""\ + machine host.domain.com login log password "pas s" account acct + """, 'password', 'pas s') + self._test_token_x("""\ + machine host.domain.com login log password pass account "acc t" + """, 'account', 'acc t') + + def test_token_value_non_ascii(self): + self._test_token_x("""\ + machine host.domain.com login \xa1\xa2 password pass account acct + """, 'login', '\xa1\xa2') + self._test_token_x("""\ + machine host.domain.com login log password pass account \xa1\xa2 + """, 'account', '\xa1\xa2') + self._test_token_x("""\ + machine host.domain.com login log password \xa1\xa2 account acct + """, 'password', '\xa1\xa2') + + def test_token_value_leading_hash(self): + self._test_token_x("""\ + machine host.domain.com login #log password pass account acct + """, 'login', '#log') + self._test_token_x("""\ + machine host.domain.com login log password pass account #acct + """, 'account', '#acct') + self._test_token_x("""\ machine host.domain.com login log password #pass account acct - """, '#pass') + """, 'password', '#pass') - def test_password_with_trailing_hash(self): - self._test_passwords("""\ + def test_token_value_trailing_hash(self): + self._test_token_x("""\ + machine host.domain.com login log# password pass account acct + """, 'login', 'log#') + self._test_token_x("""\ + machine host.domain.com login log password pass account acct# + """, 'account', 'acct#') + self._test_token_x("""\ machine host.domain.com login log password pass# account acct - """, 'pass#') + """, 'password', 'pass#') - def test_password_with_internal_hash(self): - self._test_passwords("""\ + def test_token_value_internal_hash(self): + self._test_token_x("""\ + machine host.domain.com login lo#g password pass account acct + """, 'login', 'lo#g') + self._test_token_x("""\ + machine host.domain.com login log password pass account ac#ct + """, 'account', 'ac#ct') + self._test_token_x("""\ machine host.domain.com login log password pa#ss account acct - """, 'pa#ss') + """, 'password', 'pa#ss') def _test_comment(self, nrc, passwd='pass'): nrc = self.make_nrc(nrc) - self.assertEqual(nrc.hosts['foo.domain.com'], ('bar', None, passwd)) - self.assertEqual(nrc.hosts['bar.domain.com'], ('foo', None, 'pass')) + self.assertEqual(nrc.hosts['foo.domain.com'], ('bar', '', passwd)) + self.assertEqual(nrc.hosts['bar.domain.com'], ('foo', '', 'pass')) def test_comment_before_machine_line(self): self._test_comment("""\ @@ -82,6 +202,42 @@ machine bar.domain.com login foo password pass """) + def test_comment_after_machine_line(self): + self._test_comment("""\ + machine foo.domain.com login bar password pass + # comment + machine bar.domain.com login foo password pass + """) + self._test_comment("""\ + machine foo.domain.com login bar password pass + machine bar.domain.com login foo password pass + # comment + """) + + def test_comment_after_machine_line_no_space(self): + self._test_comment("""\ + machine foo.domain.com login bar password pass + #comment + machine bar.domain.com login foo password pass + """) + self._test_comment("""\ + machine foo.domain.com login bar password pass + machine bar.domain.com login foo password pass + #comment + """) + + def test_comment_after_machine_line_hash_only(self): + self._test_comment("""\ + machine foo.domain.com login bar password pass + # + machine bar.domain.com login foo password pass + """) + self._test_comment("""\ + machine foo.domain.com login bar password pass + machine bar.domain.com login foo password pass + # + """) + def test_comment_at_end_of_machine_line(self): self._test_comment("""\ machine foo.domain.com login bar password pass # comment @@ -119,9 +275,23 @@ os.chmod(fn, 0o600) nrc = netrc.netrc() self.assertEqual(nrc.hosts['foo.domain.com'], - ('bar', None, 'pass')) + ('bar', '', 'pass')) os.chmod(fn, 0o622) self.assertRaises(netrc.NetrcParseError, netrc.netrc) + with open(fn, 'wt') as f: + f.write("""\ + machine foo.domain.com login anonymous password pass + default login foo password pass + """) + with support.EnvironmentVarGuard() as environ: + environ.set('HOME', d) + os.chmod(fn, 0o600) + nrc = netrc.netrc() + self.assertEqual(nrc.hosts['foo.domain.com'], + ('anonymous', '', 'pass')) + os.chmod(fn, 0o622) + self.assertEqual(nrc.hosts['foo.domain.com'], + ('anonymous', '', 'pass')) def test_main(): support.run_unittest(NetrcTestCase)