The syntax for "token" is taken from RFC 1945 [2], reproduced here for
    convenience:

     token        = 1*<any CHAR except CTLs or tspecials>

     tspecials    = "(" | ")" | "<" | ">" | "@"
                  | "," | ";" | ":" | "\" | <">
                  | "/" | "[" | "]" | "?" | "="
                  | "{" | "}" | SP | HT

   The syntax for "path" is defined in RFC 1808 [6], reproduced here for
   convenience:

     path        = fsegment *( "/" segment )
     fsegment    = 1*pchar
     segment     =  *pchar

     pchar       = uchar | ":" | "@" | "&" | "="
     uchar       = unreserved | escape
     unreserved  = alpha | digit | safe | extra

     escape      = "%" hex hex
     hex         = digit | "A" | "B" | "C" | "D" | "E" | "F" |
                          "a" | "b" | "c" | "d" | "e" | "f"

     alpha       = lowalpha | hialpha




Koster                draft-koster-robots-00.txt                [Page 7]

INTERNET DRAFT        A Method for Robots Control       December 4, 1996

     lowalpha    = "a" | "b" | "c" | "d" | "e" | "f" | "g" | "h" | "i" |
                   "j" | "k" | "l" | "m" | "n" | "o" | "p" | "q" | "r" |
                   "s" | "t" | "u" | "v" | "w" | "x" | "y" | "z"
     hialpha     = "A" | "B" | "C" | "D" | "E" | "F" | "G" | "H" | "I" |
                   "J" | "K" | "L" | "M" | "N" | "O" | "P" | "Q" | "R" |
                   "S" | "T" | "U" | "V" | "W" | "X" | "Y" | "Z"

     digit       = "0" | "1" | "2" | "3" | "4" | "5" | "6" | "7" |
                   "8" | "9"

     safe        = "$" | "-" | "_" | "." | "+"
     extra       = "!" | "*" | "'" | "(" | ")" | ","