#! /usr/bin/env python """ Protocol definitions for python based lib9p server/client. The sub-namespace td has type definitions (qid, stat) and values that are "#define" constants in C code (e.g., DMDIR, QTFILE, etc). This also contains the byte values for protocol codes like Tversion, Rversion, Rerror, and so on. >>> td.Tversion 100 >>> td.Rlerror 7 The qid and stat types are PFOD classes and generate instances that are a cross between namedtuple and OrderedDictionary (see pfod.py for details): >>> td.qid(type=td.QTFILE, path=2, version=1) qid(type=0, version=1, path=2) The td.stat() type output is pretty long, since it has all the dotu-specific members (used only when packing for dotu/dotl and set only when unpacking those), so here's just one field: >>> td.stat(*(15 * [0])).mode 0 >>> import pprint; pprint.pprint(td.stat()._fields) ('type', 'dev', 'qid', 'mode', 'atime', 'mtime', 'length', 'name', 'uid', 'gid', 'muid', 'extension', 'n_uid', 'n_gid', 'n_muid') Stat objects sent across the protocol must first be encoded into wirestat objects, which are basically size-counted pre-sequenced stat objects. The pre-sequencing uses: >>> td.stat_seq Sequencer('stat') For parsing bytes returned in a Tread on a directory, td.wirestat_seq is the sequencer. However, most users should rely on the packers and unpackers in each protocol (see {pack,unpack}_wirestat below). >>> td.wirestat_seq Sequencer('wirestat') There is a dictionary fcall_to_name that maps from byte value to protocol code. Names map to themselves as well: >>> fcall_names[101] 'Rversion' >>> fcall_names['Tversion'] 'Tversion' The sub-namespace rrd has request (Tversion, Topen, etc) and response (Rversion, Ropen, etc) data definitions. Each of these is a PFOD class: >>> rrd.Tversion(1000, 'hello', tag=0) Tversion(tag=0, msize=1000, version='hello') The function p9_version() looks up the instance of each supported protocol, or raises a KeyError when given an invalid protocol. The names may be spelled in any mixture of cases. The names plain, dotu, and dotl are predefined as the three supported protocols: >>> p9_version('invalid') Traceback (most recent call last): ... KeyError: 'invalid' >>> p9_version('9p2000') == plain True >>> p9_version('9P2000') == plain True >>> p9_version('9P2000.u') == dotu True >>> p9_version('9p2000.L') == dotl True Protocol instances have a pack() method that encodes a set of arguments into a packet. To know what to encode, pack() must receive an fcall value and a dictionary containing argument values, or something equivalent. The required argument values depend on the fcall. For instance, a Tversion fcall needs three arguments: the version name, the tag, and the msize (these of course are the pre-filled fields in a Tversion PFOD instance). >>> args = {'version': '!', 'tag': 1, 'msize': 1000} >>> pkt = dotu.pack(fcall='Tversion', args=args) >>> len(pkt) 14 The length of string '!' is 1, and the packet (or wire) format of a Tversion request is: size[4] fcall[1] tag[2] msize[4] version[s] which corresponds to a struct's IBHIH (for the fixed size parts) followed by 1 B (for the string). The overall packet is 14 bytes long, so we have size=9, fcall=100, tag=1, msize=1000, and the version string is length=1, value=33 (ord('!')). >>> import struct >>> struct.unpack('>> dotu.version '9P2000.u' >>> args = {'tag': 99, 'msize': 1000} >>> pkt = dotu.pack(fcall='Tversion', args=args) >>> len(pkt) 21 The fcall can be supplied numerically: >>> pkt2 = dotu.pack(fcall=td.Tversion, args=args) >>> pkt == pkt2 True Instead of providing an fcall you can provide an instance of the appropriate PFOD. In this case pack() finds the type from the PFOD instance. As usual, the version parameter is filled in for you: >>> pkt2 = dotu.pack(rrd.Tversion(tag=99, msize=1000)) >>> pkt == pkt2 True Note that it's up to you to check the other end's version and switch to a "lower" protocol as needed. Each instance does provide a downgrade_to() method that gets you a possibly-downgraded instance. This will fail if you are actually trying to upgrade, and also if you provide a bogus version: >>> dotu.downgrade_to('9P2000.L') Traceback (most recent call last): ... KeyError: '9P2000.L' >>> dotu.downgrade_to('we never heard of this protocol') Traceback (most recent call last): ... KeyError: 'we never heard of this protocol' Hence you might use: try: proto = protocol.dotl.downgrade(vstr) except KeyError: pkt = protocol.plain.pack(fcall='Rerror', args={'tag': tag, 'errstr': 'unknown protocol version ' '{0!r}'.format(vstr)}) else: pkt = proto.pack(fcall='Rversion', args={'tag': tag, 'msize': msize}) When using a PFOD instance, it is slightly more efficient to use pack_from(): try: proto = protocol.dotl.downgrade(vstr) reply = protocol.rrd.Rversion(tag=tag, msize=msize) except KeyError: proto = protocol.plain reply = protocol.rrd.Rerror(tag=tag, errstr='unknown protocol version {0!r}'.format(vstr)) pkt = proto.pack_from(reply) does the equivalent of the try/except/else variant. Note that the protocol.rrd.Rversion() instance has version=None. Like proto.pack, the pack_from will detect this "missing" value and fill it in. Because errors vary (one should use Rlerror for dotl and Rerror for dotu and plain), and it's convenient to use an Exception instance for an error, all protocols provide .error(). This builds the appropriate kind of error response, extracting and converting errno's and error messages as appropriate. If is an instance of Exception, err.errno provides the errnum or ecode value (if used, for dotu and dotl) and err.strerror as the errstr value (if used, for plain 9p2000). Otherwise err should be an integer, and we'll use os.strerror() to get a message. When using plain 9P2000 this sends error *messages*: >>> import errno, os >>> utf8 = os.strerror(errno.ENOENT).encode('utf-8') >>> pkt = None >>> try: ... os.open('presumably this file does not exist here', 0) ... except OSError as err: ... pkt = plain.error(1, err) ... >>> pkt[-len(utf8):] == utf8 True >>> pkt2 = plain.error(1, errno.ENOENT) >>> pkt == pkt2 True When using 9P2000.u it sends the error code as well, and when using 9P2000.L it sends only the error code (and more error codes can pass through): >>> len(pkt) 34 >>> len(dotu.error(1, errno.ENOENT)) 38 >>> len(dotl.error(1, errno.ENOENT)) 11 For even more convenience (and another slight speed hack), the protocol has member functions for each valid pfod, which effectively do a pack_from of a pfod built from the arguments. In the above example this is not very useful (because we want two different replies), but for Rlink, for instance, which has only a tag, a server might implement Tlink() as: def do_Tlink(proto, data): # data will be a protocol.rrd.Tlink(...) tag = data.tag dfid = data.dfid fid = data.fid name = data.name ... some code to set up for doing the link link ... try: os.link(path1, path2) except OSError as err: return proto.error(tag, err) else: return proto.Rlink(tag) >>> pkt = dotl.Rlink(12345) >>> struct.unpack('>> vpkt = dotl.Tversion(tag=0, msize=12345) To see that this is a valid version packet, let's unpack its bytes. The overall length is 21 bytes: 4 bytes of size, 1 byte of code 100 for Tversion, 2 bytes of tag, 4 bytes of msize, 2 bytes of string length, and 8 bytes of string '9P2000.L'. >>> tup = struct.unpack('>> tup[0:5] (21, 100, 0, 12345, 8) >>> ''.join(chr(i) for i in tup[5:]) '9P2000.L' Of course, since you can *pack*, you can also *unpack*. It's possible that the incoming packet is malformed. If so, this raises various errors (see below). Unpack is actually a two step process: first we unpack a header (where the size is already removed and is implied by len(data)), then we unpack the data within the packet. You can invoke the first step separately. Furthermore, there's a noerror argument that leaves some fields set to None or empty strings, if the packet is too short. (Note that we need a hack for py2k vs py3k strings here, for doctests. Also, encoding 12345 into a byte string produces '90', by ASCII luck!) >>> pkt = pkt[4:] # strip generated size >>> import sys >>> py3k = sys.version_info[0] >= 3 >>> b2s = lambda x: x.decode('utf-8') if py3k else x >>> d = plain.unpack_header(pkt[0:1], noerror=True) >>> d.data = b2s(d.data) >>> d Header(size=5, dsize=0, fcall=71, data='') >>> d = plain.unpack_header(pkt[0:2], noerror=True) >>> d.data = b2s(d.data) >>> d Header(size=6, dsize=1, fcall=71, data='9') Without noerror=True a short packet raises a SequenceError: >>> plain.unpack_header(pkt[0:0]) # doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): ... SequenceError: out of data while unpacking 'fcall' Of course, a normal packet decodes fine: >>> d = plain.unpack_header(pkt) >>> d.data = b2s(d.data) >>> d Header(size=7, dsize=2, fcall=71, data='90') but one that is too *long* potentially raises a SequencError. (This is impossible for a header, though, since the size and data size are both implied: either there is an fcall code, and the rest of the bytes are "data", or there isn't and the packet is too short. So we can only demonstrate this for regular unpack; see below.) Note that all along, this has been decoding Rlink (fcall=71), which is not valid for plain 9P2000 protocol. It's up to the caller to check: >>> plain.supports(71) False >>> plain.unpack(pkt) # doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): ... SequenceError: invalid fcall 'Rlink' for 9P2000 >>> dotl.unpack(pkt) Rlink(tag=12345) However, the unpack() method DOES check that the fcall type is valid, even if you supply noerror=True. This is because we can only really decode the header, not the data, if the fcall is invalid: >>> plain.unpack(pkt, noerror=True) # doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): ... SequenceError: invalid fcall 'Rlink' for 9P2000 The same applies to much-too-short packets even if noerror is set. Specifically, if the (post-"size") header shortens down to the empty string, the fcall will be None: >>> dotl.unpack(b'', noerror=True) # doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): ... SequenceError: invalid fcall None for 9P2000.L If there is at least a full header, though, noerror will do the obvious: >>> dotl.unpack(pkt[0:1], noerror=True) Rlink(tag=None) >>> dotl.unpack(pkt[0:2], noerror=True) Rlink(tag=None) If the packet is too long, noerror suppresses the SequenceError: >>> dotl.unpack(pkt + b'x') # doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): ... SequenceError: 1 byte(s) unconsumed >>> dotl.unpack(pkt + b'x', noerror=True) Rlink(tag=12345) To pack a stat object when producing data for reading a directory, use pack_wirestat. This puts a size in front of the packed stat data (they're represented this way in read()-of-directory data, but not elsewhere). To unpack the result of a Tstat or a read() on a directory, use unpack_wirestat. The stat values are variable length so this works with offsets. If the packet is truncated, you'll get a SequenceError, but just as for header unpacking, you can use noerror to suppress this. (First, we'll need to build some valid packet data.) >>> statobj = td.stat(type=0,dev=0,qid=td.qid(0,0,0),mode=0, ... atime=0,mtime=0,length=0,name=b'foo',uid=b'0',gid=b'0',muid=b'0') >>> data = plain.pack_wirestat(statobj) >>> len(data) 55 Now we can unpack it: >>> newobj, offset = plain.unpack_wirestat(data, 0) >>> newobj == statobj True >>> offset 55 Since the packed data do not include the dotu extensions, we get a SequenceError if we try to unpack with dotu or dotl: >>> dotu.unpack_wirestat(data, 0) # doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): ... SequenceError: out of data while unpacking 'extension' When using noerror, the returned new offset will be greater than the length of the packet, after a failed unpack, and some elements may be None: >>> newobj, offset = plain.unpack_wirestat(data[0:10], 0, noerror=True) >>> offset 55 >>> newobj.length is None True Similarly, use unpack_dirent to unpack the result of a dot-L readdir(), using offsets. (Build them with pack_dirent.) >>> dirent = td.dirent(qid=td.qid(1,2,3),offset=0, ... type=td.DT_REG,name=b'foo') >>> pkt = dotl.pack_dirent(dirent) >>> len(pkt) 27 and then: >>> newde, offset = dotl.unpack_dirent(pkt, 0) >>> newde == dirent True >>> offset 27 """ from __future__ import print_function import collections import os import re import sys import p9err import pfod import sequencer SequenceError = sequencer.SequenceError fcall_names = {} # begin ??? # to interfere with (eg) the size part of the packet: # pkt = proto.pack(fcall=protocol.td.Tversion, # size=123, # wrong # args={ 'tag': 1, msize: 1000, version: '9p2000.u' }) # a standard Twrite: # pkt = proto.pack(fcall=protocol.td.Twrite, # args={ 'tag': 1, 'fid': 2, 'offset': 0, 'data': b'rawdata' }) # or: # pkt = proto.pack(fcall=protocol.td.Twrite, # data=proto.Twrite(tag=1, fid=2, offset=0, data=b'rawdata' }) # a broken Twrite: # pkt = proto.pack(fcall=protocol.td.Twrite, # args={ 'tag': 1, 'fid': 2, 'offset': 0, 'count': 99, # 'data': b'rawdata' }) -- XXX won't work (yet?) # # build a QID: (td => typedefs and defines) # qid = protocol.td.qid(type=protocol.td.QTFILE, version=1, path=2) # build the Twrite data as a data structure: # wrdata = protocol.td.Twrite(tag=1, fid=2, offset=0, data=b'rawdata') # # turn incoming byte stream data into a Header and remaining data: # foo = proto.pack(data) class _PackInfo(object): """ Essentially just a Sequencer, except that we remember if there are any :auto annotations on any of the coders, and we check for coders that are string coders ('data[size]'). This could in theory be a recursive check, but in practice all the automatics are at the top level, and we have no mechanism to pass down inner automatics. """ def __init__(self, seq): self.seq = seq self.autos = None for pair in seq: # (cond, code) pair sub = pair[1] if sub.aux is None: continue assert sub.aux == 'auto' or sub.aux == 'len' if self.autos is None: self.autos = [] self.autos.append(pair) def __repr__(self): return '{0}({1!r})'.format(self.__class__.__name__, self.seq) def pack(self, auto_vars, conditions, data, rodata): """ Pack data. Insert automatic and/or counted variables automatically, if they are not already set in the data. If rodata ("read-only data") is True we make sure not to modify the caller's data. Since data is a PFOD rather than a normal ordered dictionary, we use _copy(). """ if self.autos: for cond, sub in self.autos: # False conditionals don't need to be filled-in. if cond is not None and not conditions[cond]: continue if sub.aux == 'auto': # Automatic variable, e.g., version. The # sub-coder's name ('version') is the test item. if data.get(sub.name) is None: if rodata: data = data._copy() rodata = False data[sub.name] = auto_vars[sub.name] else: # Automatic length, e.g., data[count]. The # sub-coders's repeat item ('count') is the # test item. Of course, it's possible that # the counted item is missing as well. If so # we just leave both None and take the # encoding error. assert sub.aux == 'len' if data.get(sub.repeat) is not None: continue item = data.get(sub.name) if item is not None: if rodata: data = data._copy() rodata = False data[sub.repeat] = len(item) return self.seq.pack(data, conditions) class _P9Proto(object): def __init__(self, auto_vars, conditions, p9_data, pfods, index): self.auto_vars = auto_vars # currently, just version self.conditions = conditions # '.u' self.pfods = pfods # dictionary, maps pfod to packinfo self.index = index # for comparison: plain < dotu < dotl self.use_rlerror = rrd.Rlerror in pfods for dtype in pfods: name = dtype.__name__ # For each Txxx/Rxxx, define a self.() to # call self.pack_from(). # # The packinfo is from _Packinfo(seq); the fcall and # seq come from p9_data.protocol[]. proto_tuple = p9_data.protocol[name] assert dtype == proto_tuple[0] packinfo = pfods[dtype] # in theory we can do this with no names using nested # lambdas, but that's just too confusing, so let's # do it with nested functions instead. def builder(constructor=dtype, packinfo=packinfo): "return function that calls _pack_from with built PFOD" def invoker(self, *args, **kwargs): "build PFOD and call _pack_from" return self._pack_from(constructor(*args, **kwargs), rodata=False, caller=None, packinfo=packinfo) return invoker func = builder() func.__name__ = name func.__doc__ = 'pack from {0}'.format(name) setattr(self.__class__, name, func) def __repr__(self): return '{0}({1!r})'.format(self.__class__.__name__, self.version) def __str__(self): return self.version # define rich-comparison operators, so we can, e.g., test vers > plain def __lt__(self, other): return self.index < other.index def __le__(self, other): return self.index <= other.index def __eq__(self, other): return self.index == other.index def __ne__(self, other): return self.index != other.index def __gt__(self, other): return self.index > other.index def __ge__(self, other): return self.index >= other.index def downgrade_to(self, other_name): """ Downgrade from this protocol to a not-greater one. Raises KeyError if other_name is not a valid protocol, or this is not a downgrade (with setting back to self considered a valid "downgrade", i.e., we're doing subseteq rather than subset). """ if not isinstance(other_name, str) and isinstance(other_name, bytes): other_name = other_name.decode('utf-8', 'surrogateescape') other = p9_version(other_name) if other > self: raise KeyError(other_name) return other def error(self, tag, err): "produce Rerror or Rlerror, whichever is appropriate" if isinstance(err, Exception): errnum = err.errno errmsg = err.strerror else: errnum = err errmsg = os.strerror(errnum) if self.use_rlerror: return self.Rlerror(tag=tag, ecode=p9err.to_dotl(errnum)) return self.Rerror(tag=tag, errstr=errmsg, errnum=p9err.to_dotu(errnum)) def pack(self, *args, **kwargs): "pack up a pfod or fcall-and-arguments" fcall = kwargs.pop('fcall', None) if fcall is None: # Called without fcall=... # This requires that args have one argument that # is the PFOD; kwargs should be empty (but we'll take # data=pfod as well). The size is implied, and # fcall comes from the pfod. data = kwargs.pop('data', None) if data is None: if len(args) != 1: raise TypeError('pack() with no fcall requires 1 argument') data = args[0] if len(kwargs): raise TypeError('pack() got an unexpected keyword argument ' '{0}'.format(kwargs.popitem()[0])) return self._pack_from(data, True, 'pack', None) # Called as pack(fcall=whatever, data={...}). # The data argument must be a dictionary since we're going to # apply ** to it in the call to build the PFOD. Note that # it could already be a PFOD, which is OK, but we're going to # copy it to a new one regardless (callers that have a PFOD # should use pack_from instead). if len(args): raise TypeError('pack() got unexpected arguments ' '{0!r}'.format(args)) data = kwargs.pop('args', None) if len(kwargs): raise TypeError('pack() got an unexpected keyword argument ' '{0}'.format(kwargs.popitem()[0])) if not isinstance(data, dict): raise TypeError('pack() with fcall and data ' 'requires data to be a dictionary') try: name = fcall_names[fcall] except KeyError: raise TypeError('pack(): {0} is not a valid ' 'fcall value'.format(fcall)) cls = getattr(rrd, name) data = cls(**data) return self._pack_from(data, False, 'pack', None) def pack_from(self, data): "pack from pfod data, using its type to determine fcall" return self._pack_from(data, True, 'pack_from', None) def _pack_from(self, data, rodata, caller, packinfo): """ Internal pack(): called from both invokers (self.Tversion, self.Rwalk, etc.) and from pack and pack_from methods. "caller" says which. If rodata is True we're not supposed to modify the incoming data, as it may belong to someone else. Some calls to pack() build a PFOD and hence pass in False. The predefined invokers pass in a preconstructed PFOD, *and* set rodata=False, *and* provide a packinfo, so that we never have to copy, nor look up the packinfo. """ if caller is not None: assert caller in ('pack', 'pack_from') and packinfo is None # Indirect call from pack_from(), or from pack() after # pack() built a PFOD. We make sure this kind of PFOD # is allowed for this protocol. packinfo = self.pfods.get(data.__class__, None) if packinfo is None: raise TypeError('{0}({1!r}): invalid ' 'input'.format(caller, data)) # Pack the data pkt = packinfo.pack(self.auto_vars, self.conditions, data, rodata) fcall = data.__class__.__name__ fcall_code = getattr(td, fcall) # That's the inner data; now we must add the header, # with fcall (translated back to byte code value) and # outer data. The size is implied by len(pkt). There # are no other auto variables, and no conditions. # # NB: the size includes the size of the header itself # and the fcall code byte, plus the size of the data. data = _9p_data.header_pfod(size=4 + 1 + len(pkt), dsize=len(pkt), fcall=fcall_code, data=pkt) empty = None # logically should be {}, but not actually used below pkt = _9p_data.header_pack_seq.pack(data, empty) return pkt @staticmethod def unpack_header(bstring, noerror=False): """ Unpack header. We know that our caller has already stripped off the overall size field (4 bytes), leaving us with the fcall (1 byte) and data (len(bstring)-1 bytes). If len(bstring) is 0, this is an invalid header: set dsize to 0 and let fcall become None, if noerror is set. """ vdict = _9p_data.header_pfod() vdict['size'] = len(bstring) + 4 vdict['dsize'] = max(0, len(bstring) - 1) _9p_data.header_unpack_seq.unpack(vdict, None, bstring, noerror) return vdict def unpack(self, bstring, noerror=False): "produce filled PFOD from fcall in packet" vdict = self.unpack_header(bstring, noerror) # NB: vdict['dsize'] is used internally during unpack, to # find out how many bytes to copy to vdict['data'], but by # the time unpack is done, we no longer need it. # # size = vdict['size'] # dsize = vdict['dsize'] fcall = vdict['fcall'] data = vdict['data'] # Note: it's possible for size and/or fcall to be None, # when noerror is true. However, if we support fcall, then # clearly fcall is not None; and since fcall follows size, # we can always proceed if we support fcall. if self.supports(fcall): fcall = fcall_names[fcall] cls = getattr(rrd, fcall) seq = self.pfods[cls].seq elif fcall == td.Rlerror: # As a special case for diod, we accept Rlerror even # if it's not formally part of the protocol. cls = rrd.Rlerror seq = dotl.pfods[rrd.Rlerror].seq else: fcall = fcall_names.get(fcall, fcall) raise SequenceError('invalid fcall {0!r} for ' '{1}'.format(fcall, self)) vdict = cls() seq.unpack(vdict, self.conditions, data, noerror) return vdict def pack_wirestat(self, statobj): """ Pack a stat object to appear as data returned by read() on a directory. Essentially, we prefix the data with a size. """ data = td.stat_seq.pack(statobj, self.conditions) return td.wirestat_seq.pack({'size': len(data), 'data': data}, {}) def unpack_wirestat(self, bstring, offset, noerror=False): """ Produce the next td.stat object from byte-string, returning it and new offset. """ statobj = td.stat() d = { 'size': None } newoff = td.wirestat_seq.unpack_from(d, self.conditions, bstring, offset, noerror) size = d['size'] if size is None: # implies noerror; newoff==offset+2 return statobj, newoff # We now have size and data. If noerror, data might be # too short, in which case we'll unpack a partial statobj. # Or (with or without noeror), data might be too long, so # that while len(data) == size, not all the data get used. # That may be allowed by the protocol: it's not clear. data = d['data'] used = td.stat_seq.unpack_from(statobj, self.conditions, data, 0, noerror) # if size != used ... then what? return statobj, newoff def pack_dirent(self, dirent): """ Dirents (dot-L only) are easy to pack, but we provide this function for symmetry. (Should we raise an error if called on plain or dotu?) """ return td.dirent_seq.pack(dirent, self.conditions) def unpack_dirent(self, bstring, offset, noerror=False): """ Produces the next td.dirent object from byte-string, returning it and new offset. """ deobj = td.dirent() offset = td.dirent_seq.unpack_from(deobj, self.conditions, bstring, offset, noerror) return deobj, offset def supports(self, fcall): """ Return True if and only if this protocol supports the given fcall. >>> plain.supports(100) True >>> plain.supports('Tversion') True >>> plain.supports('Rlink') False """ fcall = fcall_names.get(fcall, None) if fcall is None: return False cls = getattr(rrd, fcall) return cls in self.pfods def get_version(self, as_bytes=True): "get Plan 9 protocol version, as string or (default) as bytes" ret = self.auto_vars['version'] if as_bytes and not isinstance(ret, bytes): ret = ret.encode('utf-8') return ret @property def version(self): "Plan 9 protocol version" return self.get_version(as_bytes=False) DEBUG = False # This defines a special en/decoder named "s" using a magic # builtin. This and stat are the only variable-length # decoders, and this is the only recursively-variable-length # one (i.e., stat decoding is effectively fixed size once we # handle strings). So this magic avoids the need for recursion. # # Note that _string_ is, in effect, size[2] orig_var[size]. _STRING_MAGIC = '_string_' SDesc = "typedef s: " + _STRING_MAGIC # This defines an en/decoder for type "qid", # which en/decodes 1 byte called type, 4 called version, and # 8 called path (for a total of 13 bytes). # # It also defines QTDIR, QTAPPEND, etc. (These are not used # for en/decode, or at least not yet.) QIDDesc = """\ typedef qid: type[1] version[4] path[8] #define QTDIR 0x80 #define QTAPPEND 0x40 #define QTEXCL 0x20 #define QTMOUNT 0x10 #define QTAUTH 0x08 #define QTTMP 0x04 #define QTSYMLINK 0x02 #define QTFILE 0x00 """ # This defines a stat decoder, which has a 9p2000 standard front, # followed by an optional additional portion. # # The constants are named DMDIR etc. STATDesc = """ typedef stat: type[2] dev[4] qid[qid] mode[4] atime[4] mtime[4] \ length[8] name[s] uid[s] gid[s] muid[s] \ {.u: extension[s] n_uid[4] n_gid[4] n_muid[4] } #define DMDIR 0x80000000 #define DMAPPEND 0x40000000 #define DMMOUNT 0x10000000 #define DMAUTH 0x08000000 #define DMTMP 0x04000000 #define DMSYMLINK 0x02000000 /* 9P2000.u extensions */ #define DMDEVICE 0x00800000 #define DMNAMEDPIPE 0x00200000 #define DMSOCKET 0x00100000 #define DMSETUID 0x00080000 #define DMSETGID 0x00040000 """ # This defines a wirestat decoder. A wirestat is a size and then # a (previously encoded, or future-decoded) stat. WirestatDesc = """ typedef wirestat: size[2] data[size] """ # This defines a dirent decoder, which has a dot-L specific format. # # The dirent type fields are defined as DT_* (same as BSD and Linux). DirentDesc = """ typedef dirent: qid[qid] offset[8] type[1] name[s] #define DT_UNKNOWN 0 #define DT_FIFO 1 #define DT_CHR 2 #define DT_DIR 4 #define DT_BLK 6 #define DT_REG 8 #define DT_LNK 10 #define DT_SOCK 12 #define DT_WHT 14 """ # N.B.: this is largely a slightly more rigidly formatted variant of # the contents of: # https://github.com/chaos/diod/blob/master/protocol.md # # Note that = : ... assigns names for the fcall # fcall (function call) table. Names without "= value" are # assumed to be the previous value +1 (and the two names are # also checked to make sure they are Tfoo,Rfoo). ProtocolDesc = """\ Rlerror.L = 7: tag[2] ecode[4] ecode is a numerical Linux errno Tstatfs.L = 8: tag[2] fid[4] Rstatfs.L: tag[2] type[4] bsize[4] blocks[8] bfree[8] bavail[8] \ files[8] ffree[8] fsid[8] namelen[4] Rstatfs corresponds to Linux statfs structure: struct statfs { long f_type; /* type of file system */ long f_bsize; /* optimal transfer block size */ long f_blocks; /* total data blocks in file system */ long f_bfree; /* free blocks in fs */ long f_bavail; /* free blocks avail to non-superuser */ long f_files; /* total file nodes in file system */ long f_ffree; /* free file nodes in fs */ fsid_t f_fsid; /* file system id */ long f_namelen; /* maximum length of filenames */ }; This comes from nowhere obvious... #define FSTYPE 0x01021997 Tlopen.L = 12: tag[2] fid[4] flags[4] Rlopen.L: tag[2] qid[qid] iounit[4] lopen prepares fid for file (or directory) I/O. flags contains Linux open(2) flag bits, e.g., O_RDONLY, O_RDWR, O_WRONLY. #define L_O_CREAT 000000100 #define L_O_EXCL 000000200 #define L_O_NOCTTY 000000400 #define L_O_TRUNC 000001000 #define L_O_APPEND 000002000 #define L_O_NONBLOCK 000004000 #define L_O_DSYNC 000010000 #define L_O_FASYNC 000020000 #define L_O_DIRECT 000040000 #define L_O_LARGEFILE 000100000 #define L_O_DIRECTORY 000200000 #define L_O_NOFOLLOW 000400000 #define L_O_NOATIME 001000000 #define L_O_CLOEXEC 002000000 #define L_O_SYNC 004000000 #define L_O_PATH 010000000 #define L_O_TMPFILE 020000000 Tlcreate.L = 14: tag[2] fid[4] name[s] flags[4] mode[4] gid[4] Rlcreate.L: tag[2] qid[qid] iounit[4] lcreate creates a regular file name in directory fid and prepares it for I/O. fid initially represents the parent directory of the new file. After the call it represents the new file. flags contains Linux open(2) flag bits (including O_CREAT). mode contains Linux creat(2) mode (permissions) bits. gid is the effective gid of the caller. Tsymlink.L = 16: tag[2] dfid[4] name[s] symtgt[s] gid[4] Rsymlink.L: tag[2] qid[qid] symlink creates a symbolic link name in directory dfid. The link will point to symtgt. gid is the effective group id of the caller. The qid for the new symbolic link is returned in the reply. Tmknod.L = 18: tag[2] dfid[4] name[s] mode[4] major[4] minor[4] gid[4] Rmknod.L: tag[2] qid[qid] mknod creates a device node name in directory dfid with major and minor numbers. mode contains Linux mknod(2) mode bits. (Note that these include the S_IFMT bits which may be S_IFBLK, S_IFCHR, or S_IFSOCK.) gid is the effective group id of the caller. The qid for the new device node is returned in the reply. Trename.L = 20: tag[2] fid[4] dfid[4] name[s] Rrename.L: tag[2] rename renames a file system object referenced by fid, to name in the directory referenced by dfid. This operation will eventually be replaced by renameat. Treadlink.L = 22: tag[2] fid[4] Rreadlink.L: tag[2] target[s] readlink returns the contents of teh symbolic link referenced by fid. Tgetattr.L = 24: tag[2] fid[4] request_mask[8] Rgetattr.L: tag[2] valid[8] qid[qid] mode[4] uid[4] gid[4] nlink[8] \ rdev[8] size[8] blksize[8] blocks[8] \ atime_sec[8] atime_nsec[8] mtime_sec[8] mtime_nsec[8] \ ctime_sec[8] ctime_nsec[8] btime_sec[8] btime_nsec[8] \ gen[8] data_version[8] getattr gets attributes of a file system object referenced by fid. The response is intended to follow pretty closely the fields returned by the stat(2) system call: struct stat { dev_t st_dev; /* ID of device containing file */ ino_t st_ino; /* inode number */ mode_t st_mode; /* protection */ nlink_t st_nlink; /* number of hard links */ uid_t st_uid; /* user ID of owner */ gid_t st_gid; /* group ID of owner */ dev_t st_rdev; /* device ID (if special file) */ off_t st_size; /* total size, in bytes */ blksize_t st_blksize; /* blocksize for file system I/O */ blkcnt_t st_blocks; /* number of 512B blocks allocated */ time_t st_atime; /* time of last access */ time_t st_mtime; /* time of last modification */ time_t st_ctime; /* time of last status change */ }; The differences are: * st_dev is omitted * st_ino is contained in the path component of qid * times are nanosecond resolution * btime, gen and data_version fields are reserved for future use Not all fields are valid in every call. request_mask is a bitmask indicating which fields are requested. valid is a bitmask indicating which fields are valid in the response. The mask values are as follows: #define GETATTR_MODE 0x00000001 #define GETATTR_NLINK 0x00000002 #define GETATTR_UID 0x00000004 #define GETATTR_GID 0x00000008 #define GETATTR_RDEV 0x00000010 #define GETATTR_ATIME 0x00000020 #define GETATTR_MTIME 0x00000040 #define GETATTR_CTIME 0x00000080 #define GETATTR_INO 0x00000100 #define GETATTR_SIZE 0x00000200 #define GETATTR_BLOCKS 0x00000400 #define GETATTR_BTIME 0x00000800 #define GETATTR_GEN 0x00001000 #define GETATTR_DATA_VERSION 0x00002000 #define GETATTR_BASIC 0x000007ff /* Mask for fields up to BLOCKS */ #define GETATTR_ALL 0x00003fff /* Mask for All fields above */ Tsetattr.L = 26: tag[2] fid[4] valid[4] mode[4] uid[4] gid[4] size[8] \ atime_sec[8] atime_nsec[8] mtime_sec[8] mtime_nsec[8] Rsetattr.L: tag[2] setattr sets attributes of a file system object referenced by fid. As with getattr, valid is a bitmask selecting which fields to set, which can be any combination of: mode - Linux chmod(2) mode bits. uid, gid - New owner, group of the file as described in Linux chown(2). size - New file size as handled by Linux truncate(2). atime_sec, atime_nsec - Time of last file access. mtime_sec, mtime_nsec - Time of last file modification. The valid bits are defined as follows: #define SETATTR_MODE 0x00000001 #define SETATTR_UID 0x00000002 #define SETATTR_GID 0x00000004 #define SETATTR_SIZE 0x00000008 #define SETATTR_ATIME 0x00000010 #define SETATTR_MTIME 0x00000020 #define SETATTR_CTIME 0x00000040 #define SETATTR_ATIME_SET 0x00000080 #define SETATTR_MTIME_SET 0x00000100 If a time bit is set without the corresponding SET bit, the current system time on the server is used instead of the value sent in the request. Txattrwalk.L = 30: tag[2] fid[4] newfid[4] name[s] Rxattrwalk.L: tag[2] size[8] xattrwalk gets a newfid pointing to xattr name. This fid can later be used to read the xattr value. If name is NULL newfid can be used to get the list of extended attributes associated with the file system object. Txattrcreate.L = 32: tag[2] fid[4] name[s] attr_size[8] flags[4] Rxattrcreate.L: tag[2] xattrcreate gets a fid pointing to the xattr name. This fid can later be used to set the xattr value. flag is derived from set Linux setxattr. The manpage says The flags parameter can be used to refine the semantics of the operation. XATTR_CREATE specifies a pure create, which fails if the named attribute exists already. XATTR_REPLACE specifies a pure replace operation, which fails if the named attribute does not already exist. By default (no flags), the extended attribute will be created if need be, or will simply replace the value if the attribute exists. The actual setxattr operation happens when the fid is clunked. At that point the written byte count and the attr_size specified in TXATTRCREATE should be same otherwise an error will be returned. Treaddir.L = 40: tag[2] fid[4] offset[8] count[4] Rreaddir.L: tag[2] count[4] data[count] readdir requests that the server return directory entries from the directory represented by fid, previously opened with lopen. offset is zero on the first call. Directory entries are represented as variable-length records: qid[qid] offset[8] type[1] name[s] At most count bytes will be returned in data. If count is not zero in the response, more data is available. On subsequent calls, offset is the offset returned in the last directory entry of the previous call. Tfsync.L = 50: tag[2] fid[4] Rfsync.L: tag[2] fsync tells the server to flush any cached data associated with fid, previously opened with lopen. Tlock.L = 52: tag[2] fid[4] type[1] flags[4] start[8] length[8] \ proc_id[4] client_id[s] Rlock.L: tag[2] status[1] lock is used to acquire or release a POSIX record lock on fid and has semantics similar to Linux fcntl(F_SETLK). type has one of the values: #define LOCK_TYPE_RDLCK 0 #define LOCK_TYPE_WRLCK 1 #define LOCK_TYPE_UNLCK 2 start, length, and proc_id correspond to the analagous fields passed to Linux fcntl(F_SETLK): struct flock { short l_type; /* Type of lock: F_RDLCK, F_WRLCK, F_UNLCK */ short l_whence;/* How to intrprt l_start: SEEK_SET,SEEK_CUR,SEEK_END */ off_t l_start; /* Starting offset for lock */ off_t l_len; /* Number of bytes to lock */ pid_t l_pid; /* PID of process blocking our lock (F_GETLK only) */ }; flags bits are: #define LOCK_SUCCESS 0 #define LOCK_BLOCKED 1 #define LOCK_ERROR 2 #define LOCK_GRACE 3 The Linux v9fs client implements the fcntl(F_SETLKW) (blocking) lock request by calling lock with LOCK_FLAGS_BLOCK set. If the response is LOCK_BLOCKED, it retries the lock request in an interruptible loop until status is no longer LOCK_BLOCKED. The Linux v9fs client translates BSD advisory locks (flock) to whole-file POSIX record locks. v9fs does not implement mandatory locks and will return ENOLCK if use is attempted. Because of POSIX record lock inheritance and upgrade properties, pass-through servers must be implemented carefully. Tgetlock.L = 54: tag[2] fid[4] type[1] start[8] length[8] proc_id[4] \ client_id[s] Rgetlock.L: tag[2] type[1] start[8] length[8] proc_id[4] client_id[s] getlock tests for the existence of a POSIX record lock and has semantics similar to Linux fcntl(F_GETLK). As with lock, type has one of the values defined above, and start, length, and proc_id correspond to the analagous fields in struct flock passed to Linux fcntl(F_GETLK), and client_Id is an additional mechanism for uniquely identifying the lock requester and is set to the nodename by the Linux v9fs client. Tlink.L = 70: tag[2] dfid[4] fid[4] name[s] Rlink.L: tag[2] link creates a hard link name in directory dfid. The link target is referenced by fid. Tmkdir.L = 72: tag[2] dfid[4] name[s] mode[4] gid[4] Rmkdir.L: tag[2] qid[qid] mkdir creates a new directory name in parent directory dfid. mode contains Linux mkdir(2) mode bits. gid is the effective group ID of the caller. The qid of the new directory is returned in the response. Trenameat.L = 74: tag[2] olddirfid[4] oldname[s] newdirfid[4] newname[s] Rrenameat.L: tag[2] Change the name of a file from oldname to newname, possible moving it from old directory represented by olddirfid to new directory represented by newdirfid. If the server returns ENOTSUPP, the client should fall back to the rename operation. Tunlinkat.L = 76: tag[2] dirfd[4] name[s] flags[4] Runlinkat.L: tag[2] Unlink name from directory represented by dirfd. If the file is represented by a fid, that fid is not clunked. If the server returns ENOTSUPP, the client should fall back to the remove operation. There seems to be only one defined flag: #define AT_REMOVEDIR 0x200 Tversion = 100: tag[2] msize[4] version[s]:auto Rversion: tag[2] msize[4] version[s] negotiate protocol version version establishes the msize, which is the maximum message size inclusive of the size value that can be handled by both client and server. It also establishes the protocol version. For 9P2000.L version must be the string 9P2000.L. Tauth = 102: tag[2] afid[4] uname[s] aname[s] n_uname[4] Rauth: tag[2] aqid[qid] auth initiates an authentication handshake for n_uname. Rlerror is returned if authentication is not required. If successful, afid is used to read/write the authentication handshake (protocol does not specify what is read/written), and afid is presented in the attach. Tattach = 104: tag[2] fid[4] afid[4] uname[s] aname[s] {.u: n_uname[4] } Rattach: tag[2] qid[qid] attach introduces a new user to the server, and establishes fid as the root for that user on the file tree selected by aname. afid can be NOFID (~0) or the fid from a previous auth handshake. The afid can be clunked immediately after the attach. #define NOFID 0xffffffff n_uname, if not set to NONUNAME (~0), is the uid of the user and is used in preference to uname. Note that it appears in both .u and .L (unlike most .u-specific features). #define NONUNAME 0xffffffff v9fs has several modes of access which determine how it uses attach. In the default access=user, an initial attach is sent for the user provided in the uname=name mount option, and for each user that accesses the file system thereafter. For access=, only the initial attach is sent for and all other users are denied access by the client. Rerror = 107: tag[2] errstr[s] {.u: errnum[4] } Tflush = 108: tag[2] oldtag[2] Rflush: tag[2] flush aborts an in-flight request referenced by oldtag, if any. Twalk = 110: tag[2] fid[4] newfid[4] nwname[2] nwname*(wname[s]) Rwalk: tag[2] nwqid[2] nwqid*(wqid[qid]) walk is used to descend a directory represented by fid using successive path elements provided in the wname array. If succesful, newfid represents the new path. fid can be cloned to newfid by calling walk with nwname set to zero. if nwname==0, fid need not represent a directory. Topen = 112: tag[2] fid[4] mode[1] Ropen: tag[2] qid[qid] iounit[4] open prepares fid for file (or directory) I/O. mode is: #define OREAD 0 /* open for read */ #define OWRITE 1 /* open for write */ #define ORDWR 2 /* open for read and write */ #define OEXEC 3 /* open for execute */ #define OTRUNC 16 /* truncate (illegal if OEXEC) */ #define OCEXEC 32 /* close on exec (nonsensical) */ #define ORCLOSE 64 /* remove on close */ #define ODIRECT 128 /* direct access (.u extension?) */ Tcreate = 114: tag[2] fid[4] name[s] perm[4] mode[1] {.u: extension[s] } Rcreate: tag[2] qid[qid] iounit[4] create is similar to open; however, the incoming fid is the diretory in which the file is to be created, and on success, return, the fid refers to the then-created file. Tread = 116: tag[2] fid[4] offset[8] count[4] Rread: tag[2] count[4] data[count] perform a read on the file represented by fid. Note that in v9fs, a read(2) or write(2) system call for a chunk of the file that won't fit in a single request is broken up into multiple requests. Under 9P2000.L, read cannot be used on directories. See readdir. Twrite = 118: tag[2] fid[4] offset[8] count[4] data[count] Rwrite: tag[2] count[4] perform a write on the file represented by fid. Note that in v9fs, a read(2) or write(2) system call for a chunk of the file that won't fit in a single request is broken up into multiple requests. write cannot be used on directories. Tclunk = 120: tag[2] fid[4] Rclunk: tag[2] clunk signifies that fid is no longer needed by the client. Tremove = 122: tag[2] fid[4] Rremove: tag[2] remove removes the file system object represented by fid. The fid is always clunked (even on error). Tstat = 124: tag[2] fid[4] Rstat: tag[2] size[2] data[size] Twstat = 126: tag[2] fid[4] size[2] data[size] Rwstat: tag[2] """ class _Token(object): r""" A scanned token. Tokens have a type (tok.ttype) and value (tok.value). The value is generally the token itself, although sometimes a prefix and/or suffix has been removed (for 'label', 'word*', ':aux', and '[type]' tokens). If prefix and/or suffix are removed, the full original token is in its .orig. Tokens are: - 'word', 'word*', or 'label': '[.\w]+' followed by optional '*' or ':': - 'aux': ':' followed by '\w+' (used for :auto annotation) - 'type': open bracket '[', followed by '\w+' or '\d+' (only one of these), followed by close bracket ']' - '(', ')', '{', '}': themeselves Each token can have arbitrary leading white space (which is discarded). (Probably should return ':' as a char and handle it in parser, but oh well.) """ def __init__(self, ttype, value, orig=None): self.ttype = ttype self.value = value self.orig = value if orig is None else orig if self.ttype == 'type' and self.value.isdigit(): self.ival = int(self.value) else: self.ival = None def __str__(self): return self.orig _Token.tok_expr = re.compile(r'\s*([.\w]+(?:\*|:)?' r'|:\w+' r'|\[(?:\w+|\d+)\]' r'|[(){}])') def _scan(string): """ Tokenize a string. Note: This raises a ValueError with the position of any unmatched character in the string. """ tlist = [] # make sure entire string is tokenized properly pos = 0 for item in _Token.tok_expr.finditer(string): span = item.span() if span[0] != pos: print('error: unmatched character(s) in input\n{0}\n{1}^'.format( string, ' ' * pos)) raise ValueError('unmatched lexeme', pos) pos = span[1] tlist.append(item.group(1)) if pos != len(string): print('error: unmatched character(s) in input\n{0}\n{1}^'.format( string, ' ' * pos)) raise ValueError('unmatched lexeme', pos) # classify each token, stripping decorations result = [] for item in tlist: if item in ('(', ')', '{', '}'): tok = _Token(item, item) elif item[0] == ':': tok = _Token('aux', item[1:], item) elif item.endswith(':'): tok = _Token('label', item[0:-1], item) elif item.endswith('*'): tok = _Token('word*', item[0:-1], item) elif item[0] == '[': # integer or named type if item[-1] != ']': raise ValueError('internal error: "{0}" is not [...]'.format( item)) tok = _Token('type', item[1:-1], item) else: tok = _Token('word', item) result.append(tok) return result def _debug_print_sequencer(seq): """for debugging""" print('sequencer is {0!r}'.format(seq), file=sys.stderr) for i, enc in enumerate(seq): print(' [{0:d}] = {1}'.format(i, enc), file=sys.stderr) def _parse_expr(seq, string, typedefs): """ Parse "expression-ish" items, which is a list of: name[type] name*(subexpr) (a literal asterisk) { label ... } The "type" may be an integer or a second name. In the case of a second name it must be something from . The meaning of name[integer] is that we are going to encode or decode a fixed-size field of bytes, using the given name. For name[name2], we can look up name2 in our typedefs table. The only real typedefs's used here are "stat" and "s"; each of these expands to a variable-size encode/decode. See the special case below, though. The meaning of name*(...) is: the earlier name will have been defined by an earlier _parse_expr for this same line. That earlier name provides a repeat-count. Inside the parens we get a name[type] sub-expressino. This may not recurse further, so we can use a pretty cheesy parser. As a special case, given name[name2], we first check whether name2 is an earlier name a la name*(...). Here the meaning is much like name2*(name[1]), except that the result is a simple byte string, rather than an array. The meaning of "{ label ... " is that everything following up to "}" is optional and used only with 9P2000.u and/or 9P2000.L. Inside the {...} pair is the usual set of tokens, but again {...} cannot recurse. The parse fills in a Sequencer instance, and returns a list of the parsed names. """ names = [] cond = None tokens = collections.deque(_scan(string)) def get_subscripted(tokens): """ Allows name[integer] and name1[name2] only; returns tuple after stripping off both tokens, or returns None and does not strip tokens. """ if len(tokens) == 0 or tokens[0].ttype != 'word': return None if len(tokens) > 1 and tokens[1].ttype == 'type': word = tokens.popleft() return word, tokens.popleft() return None def lookup(name, typeinfo, aux=None): """ Convert cond (if not None) to its .value, so that instead of (x, '.u') we get '.u'. Convert typeinfo to an encdec. Typeinfo may be 1/2/4/8, or one of our typedef names. If it's a typedef name it will normally correspond to an EncDecTyped, but we have one special case for string types, and another for using an earlier-defined variable. """ condval = None if cond is None else cond.value if typeinfo.ival is None: try: cls, sub = typedefs[typeinfo.value] except KeyError: raise ValueError('unknown type name {0}'.format(typeinfo)) # the type name is typeinfo.value; the corresponding # pfod class is cls; the *variable* name is name; # and the sub-sequence is sub. But if cls is None # then it's our string type. if cls is None: encdec = sequencer.EncDecSimple(name, _STRING_MAGIC, aux) else: encdec = sequencer.EncDecTyped(cls, name, sub, aux) else: if typeinfo.ival not in (1, 2, 4, 8): raise ValueError('bad integer code in {0}'.format(typeinfo)) encdec = sequencer.EncDecSimple(name, typeinfo.ival, aux) return condval, encdec def emit_simple(name, typeinfo, aux=None): """ Emit name[type]. We may be inside a conditional; if so cond is not None. """ condval, encdec = lookup(name, typeinfo, aux) seq.append_encdec(condval, encdec) names.append(name) def emit_repeat(name1, name2, typeinfo): """ Emit name1*(name2[type]). Note that the conditional is buried in the sub-coder for name2. It must be passed through anyway in case the sub- coder is only partly conditional. If the sub-coder is fully conditional, each sub-coding uses or produces no bytes and hence the array itself is effectively conditional as well (it becomes name1 * [None]). We don't (currently) have any auxiliary data for arrays. """ if name1 not in names: raise ValueError('{0}*({1}[{2}]): ' '{0} undefined'.format(name1, name2, typeinfo.value)) condval, encdec = lookup(name2, typeinfo) encdec = sequencer.EncDecA(name1, name2, encdec) seq.append_encdec(condval, encdec) names.append(name2) def emit_bytes_repeat(name1, name2): """ Emit name1[name2], e.g., data[count]. """ condval = None if cond is None else cond.value # Note that the two names are reversed when compared to # count*(data[type]). The "sub-coder" is handled directly # by EncDecA, hence is None. # # As a peculiar side effect, all bytes-repeats cause the # count itself to become automatic (to have an aux of 'len'). encdec = sequencer.EncDecA(name2, name1, None, 'len') seq.append_encdec(condval, encdec) names.append(name1) supported_conditions = ('.u') while tokens: token = tokens.popleft() if token.ttype == 'label': raise ValueError('misplaced label') if token.ttype == 'aux': raise ValueError('misplaced auxiliary') if token.ttype == '{': if cond is not None: raise ValueError('nested "{"') if len(tokens) == 0: raise ValueError('unclosed "{"') cond = tokens.popleft() if cond.ttype != 'label': raise ValueError('"{" not followed by cond label') if cond.value not in supported_conditions: raise ValueError('unsupported condition "{0}"'.format( cond.value)) continue if token.ttype == '}': if cond is None: raise ValueError('closing "}" w/o opening "{"') cond = None continue if token.ttype == 'word*': if len(tokens) == 0 or tokens[0].ttype != '(': raise ValueError('{0} not followed by (...)'.format(token)) tokens.popleft() repeat = get_subscripted(tokens) if repeat is None: raise ValueError('parse error after {0}('.format(token)) if len(tokens) == 0 or tokens[0].ttype != ')': raise ValueError('missing ")" after {0}({1}{2}'.format( token, repeat[0], repeat[1])) tokens.popleft() # N.B.: a repeat cannot have an auxiliary info (yet?). emit_repeat(token.value, repeat[0].value, repeat[1]) continue if token.ttype == 'word': # Special case: _STRING_MAGIC turns into a string # sequencer. This should be used with just one # typedef (typedef s: _string_). if token.value == _STRING_MAGIC: names.append(_STRING_MAGIC) # XXX temporary continue if len(tokens) == 0 or tokens[0].ttype != 'type': raise ValueError('parse error after {0}'.format(token)) type_or_size = tokens.popleft() # Check for name[name2] where name2 is a word (not a # number) that is in the names[] array. if type_or_size.value in names: # NB: this cannot have auxiliary info. emit_bytes_repeat(token.value, type_or_size.value) continue if len(tokens) > 0 and tokens[0].ttype == 'aux': aux = tokens.popleft() if aux.value != 'auto': raise ValueError('{0}{1}: only know "auto", not ' '{2}'.format(token, type_or_size, aux.value)) emit_simple(token.value, type_or_size, aux.value) else: emit_simple(token.value, type_or_size) continue raise ValueError('"{0}" not valid here"'.format(token)) if cond is not None: raise ValueError('unclosed "}"') return names class _ProtoDefs(object): def __init__(self): # Scan our typedefs. This may execute '#define's as well. self.typedefs = {} self.defines = {} typedef_re = re.compile(r'\s*typedef\s+(\w+)\s*:\s*(.*)') self.parse_lines('SDesc', SDesc, typedef_re, self.handle_typedef) self.parse_lines('QIDDesc', QIDDesc, typedef_re, self.handle_typedef) self.parse_lines('STATDesc', STATDesc, typedef_re, self.handle_typedef) self.parse_lines('WirestatDesc', WirestatDesc, typedef_re, self.handle_typedef) self.parse_lines('DirentDesc', DirentDesc, typedef_re, self.handle_typedef) # Scan protocol (the bulk of the work). This, too, may # execute '#define's. self.protocol = {} proto_re = re.compile(r'(\*?\w+)(\.\w+)?\s*(?:=\s*(\d+))?\s*:\s*(.*)') self.prev_proto_value = None self.parse_lines('ProtocolDesc', ProtocolDesc, proto_re, self.handle_proto_def) self.setup_header() # set these up for export() self.plain = {} self.dotu = {} self.dotl = {} def parse_lines(self, name, text, regexp, match_handler): """ Parse a sequence of lines. Match each line using the given regexp, or (first) as a #define line. Note that indented lines are either #defines or are commentary! If hnadling raises a ValueError, we complain and include the appropriate line offset. Then we sys.exit(1) (!). """ define = re.compile(r'\s*#define\s+(\w+)\s+([^/]*)' r'(\s*/\*.*\*/)?\s*$') for lineoff, line in enumerate(text.splitlines()): try: match = define.match(line) if match: self.handle_define(*match.groups()) continue match = regexp.match(line) if match: match_handler(*match.groups()) continue if len(line) and not line[0].isspace(): raise ValueError('unhandled line: {0}'.format(line)) except ValueError as err: print('Internal error while parsing {0}:\n' ' {1}\n' '(at line offset +{2}, discounting \\-newline)\n' 'The original line in question reads:\n' '{3}'.format(name, err.args[0], lineoff, line), file=sys.stderr) sys.exit(1) def handle_define(self, name, value, comment): """ Handle #define match. The regexp has three fields, matching the name, value, and possibly-empty comment; these are our arguments. """ # Obnoxious: int(,0) requires new 0o syntax in py3k; # work around by trying twice, once with base 0, then again # with explicit base 8 if the first attempt fails. try: value = int(value, 0) except ValueError: value = int(value, 8) if DEBUG: print('define: defining {0} as {1:x}'.format(name, value), file=sys.stderr) if name in self.defines: raise ValueError('redefining {0}'.format(name)) self.defines[name] = (value, comment) def handle_typedef(self, name, expr): """ Handle typedef match. The regexp has just two fields, the name and the expression to parse (note that the expression must fit all on one line, using backslach-newline if needed). Typedefs may refer back to existing typedefs, so we pass self.typedefs to _parse_expr(). """ seq = sequencer.Sequencer(name) fields = _parse_expr(seq, expr, self.typedefs) # Check for special string magic typedef. (The name # probably should be just 's' but we won't check that # here.) if len(fields) == 1 and fields[0] == _STRING_MAGIC: cls = None else: cls = pfod.pfod(name, fields) if DEBUG: print('typedef: {0} = {1!r}; '.format(name, fields), end='', file=sys.stderr) _debug_print_sequencer(seq) if name in self.typedefs: raise ValueError('redefining {0}'.format(name)) self.typedefs[name] = cls, seq def handle_proto_def(self, name, proto_version, value, expr): """ Handle protocol definition. The regexp matched: - The name of the protocol option such as Tversion, Rversion, Rlerror, etc. - The protocol version, if any (.u or .L). - The value, if specified. If no value is specified we use "the next value". - The expression to parse. As with typedefs, the expression must fit all on one line. """ if value: value = int(value) elif self.prev_proto_value is not None: value = self.prev_proto_value + 1 else: raise ValueError('{0}: missing protocol value'.format(name)) if value < 0 or value > 255: raise ValueError('{0}: protocol value {1} out of ' 'range'.format(name, value)) self.prev_proto_value = value seq = sequencer.Sequencer(name) fields = _parse_expr(seq, expr, self.typedefs) cls = pfod.pfod(name, fields) if DEBUG: print('proto: {0} = {1}; '.format(name, value), end='', file=sys.stderr) _debug_print_sequencer(seq) if name in self.protocol: raise ValueError('redefining {0}'.format(name)) self.protocol[name] = cls, value, proto_version, seq def setup_header(self): """ Handle header definition. This is a bit gimmicky and uses some special cases, because data is sized to dsize which is effectively just size - 5. We can't express this in our mini language, so we just hard-code the sequencer and pfod. In addition, the unpacker never gets the original packet's size field, only the fcall and the data. """ self.header_pfod = pfod.pfod('Header', 'size dsize fcall data') seq = sequencer.Sequencer('Header-pack') # size: 4 bytes seq.append_encdec(None, sequencer.EncDecSimple('size', 4, None)) # fcall: 1 byte seq.append_encdec(None, sequencer.EncDecSimple('fcall', 1, None)) # data: string of length dsize seq.append_encdec(None, sequencer.EncDecA('dsize', 'data', None)) if DEBUG: print('Header-pack:', file=sys.stderr) _debug_print_sequencer(seq) self.header_pack_seq = seq seq = sequencer.Sequencer('Header-unpack') seq.append_encdec(None, sequencer.EncDecSimple('fcall', 1, None)) seq.append_encdec(None, sequencer.EncDecA('dsize', 'data', None)) if DEBUG: print('Header-unpack:', file=sys.stderr) _debug_print_sequencer(seq) self.header_unpack_seq = seq def export(self, mod): """ Dump results of internal parsing process into our module namespace. Note that we do not export the 's' typedef, which did not define a data structure. Check for name collisions while we're at it. """ namespace = type('td', (object,), {}) # Export the typedefs (qid, stat). setattr(mod, 'td', namespace) for key in self.typedefs: cls = self.typedefs[key][0] if cls is None: continue setattr(namespace, key, cls) # Export two sequencers for en/decoding stat fields # (needed for reading directories and doing Twstat). setattr(namespace, 'stat_seq', self.typedefs['stat'][1]) setattr(namespace, 'wirestat_seq', self.typedefs['wirestat'][1]) # Export the similar dirent decoder. setattr(namespace, 'dirent_seq', self.typedefs['dirent'][1]) # Export the #define values for key, val in self.defines.items(): if hasattr(namespace, key): print('{0!r} is both a #define and a typedef'.format(key)) raise AssertionError('bad internal names') setattr(namespace, key, val[0]) # Export Tattach, Rattach, Twrite, Rversion, etc values. # Set up fcall_names[] table to map from value back to name. # We also map fcall names to themselves, so given either a # name or a byte code we can find out whether it's a valid # fcall. for key, val in self.protocol.items(): if hasattr(namespace, key): prev_def = '#define' if key in self.defines else 'typedef' print('{0!r} is both a {1} and a protocol ' 'value'.format(key, prev_def)) raise AssertionError('bad internal names') setattr(namespace, key, val[1]) fcall_names[key] = key fcall_names[val[1]] = key # Hook up PFOD's for each protocol object -- for # Tversion/Rversion, Twrite/Rwrite, Tlopen/Rlopen, etc. # They go in the rrd name-space, and also in dictionaries # per-protocol here, with the lookup pointing to a _PackInfo # for the corresponding sequencer. # # Note that each protocol PFOD is optionally annotated with # its specific version. We know that .L > .u > plain; but # all the "lesser" PFODs are available to all "greater" # protocols at all times. # # (This is sort-of-wrong for Rerror vs Rlerror, but we # don't bother to exclude Rerror from .L.) # # The PFODs themselves were already created, at parse time. namespace = type('rrd', (object,), {}) setattr(mod, 'rrd', namespace) for key, val in self.protocol.items(): cls = val[0] proto_version = val[2] seq = val[3] packinfo = _PackInfo(seq) if proto_version is None: # all three protocols have it self.plain[cls] = packinfo self.dotu[cls] = packinfo self.dotl[cls] = packinfo elif proto_version == '.u': # only .u and .L have it self.dotu[cls] = packinfo self.dotl[cls] = packinfo elif proto_version == '.L': # only .L has it self.dotl[cls] = packinfo else: raise AssertionError('unknown protocol {1} for ' '{0}'.format(key, proto_version)) setattr(namespace, key, cls) _9p_data = _ProtoDefs() _9p_data.export(sys.modules[__name__]) # Currently we look up by text-string, in lowercase. _9p_versions = { '9p2000': _P9Proto({'version': '9P2000'}, {'.u': False}, _9p_data, _9p_data.plain, 0), '9p2000.u': _P9Proto({'version': '9P2000.u'}, {'.u': True}, _9p_data, _9p_data.dotu, 1), '9p2000.l': _P9Proto({'version': '9P2000.L'}, {'.u': True}, _9p_data, _9p_data.dotl, 2), } def p9_version(vers_string): """ Return protocol implementation of given version. Raises KeyError if the version is invalid. Note that the KeyError will be on a string-ified, lower-cased version of the vers_string argument, even if it comes in as a bytes instance in py3k. """ if not isinstance(vers_string, str) and isinstance(vers_string, bytes): vers_string = vers_string.decode('utf-8', 'surrogateescape') return _9p_versions[vers_string.lower()] plain = p9_version('9p2000') dotu = p9_version('9p2000.u') dotl = p9_version('9p2000.L') def qid_type2name(qidtype): """ Convert qid type field to printable string. >>> qid_type2name(td.QTDIR) 'dir' >>> qid_type2name(td.QTAPPEND) 'append-only' >>> qid_type2name(0xff) 'invalid(0xff)' """ try: # Is it ever OK to have multiple bits set, # e.g., both QTAPPEND and QTEXCL? return { td.QTDIR: 'dir', td.QTAPPEND: 'append-only', td.QTEXCL: 'exclusive', td.QTMOUNT: 'mount', td.QTAUTH: 'auth', td.QTTMP: 'tmp', td.QTSYMLINK: 'symlink', td.QTFILE: 'file', }[qidtype] except KeyError: pass return 'invalid({0:#x})'.format(qidtype) if __name__ == '__main__': import doctest doctest.testmod()