- Timestamp:
- Jul 11, 2012 4:20:16 PM (10 years ago)
- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
specdomain/trunk/src/specdomain/sphinxcontrib/specmacrofileparser.py
r1002 r1004 43 43 variable_name_match = r'(@?' + macro_name + r'\[?\]?)' 44 44 45 46 47 extended_comment_block_sig_re = re.compile( 48 string_start 49 + non_greedy_whitespace 50 + extended_comment_marker 51 + r'(' + non_greedy_filler + r')' 52 + extended_comment_marker 53 + non_greedy_filler 54 + string_end, 55 re.IGNORECASE|re.DOTALL|re.MULTILINE) 56 57 variable_description_re = re.compile( 58 string_start 59 + non_greedy_filler 60 + r'#:' 61 + non_greedy_whitespace 62 + r'(' + non_greedy_filler + r')' 63 + non_greedy_whitespace 64 + string_end, 65 re.IGNORECASE|re.DOTALL|re.MULTILINE) 66 67 68 lgc_variable_sig_re = re.compile( 69 r'' 70 + string_start 71 + non_greedy_whitespace 72 + r'(local|global|constant)' # 1: object type 73 + non_greedy_whitespace 74 + r'(' + non_greedy_filler + r')' # 2: too complicated to parse all at once 75 + string_end 76 , 77 re.DOTALL 78 |re.MULTILINE 79 ) 80 81 variable_name_re = re.compile( 82 variable_name_match, 83 re.IGNORECASE|re.DOTALL|re.MULTILINE 84 ) 85 86 spec_macro_declaration_match_re = re.compile( 87 string_start 88 + r'\s*?' # optional blank space 89 + r'(r?def)' # 1: def_type (rdef | def) 90 + non_greedy_whitespace 91 + macro_name_match # 2: macro_name 92 + non_greedy_filler_match # 3: optional arguments 93 + r'\'\{?' # start body section 94 + non_greedy_filler_match # 4: body 95 + r'\}?\'' # end body section 96 + r'(#.*?)?' # 5: optional comment 97 + string_end, 98 re.IGNORECASE|re.DOTALL|re.MULTILINE) 99 100 args_match = re.compile( 101 r'\(' 102 + arglist_match # 1: argument list 103 + r'\)', 104 re.DOTALL) 105 45 106 class SpecMacrofileParser: 46 107 ''' … … 77 138 def read(self, macrofile): 78 139 """ 79 load the SPEC macro source code file into an internal buffer .80 Also remember the start and end position of each line .140 load the SPEC macro source code file into an internal buffer (self.buf). 141 Also remember the start and end position of each line (self.line_positions). 81 142 82 143 :param str filename: name (with optional path) of SPEC macro file … … 109 170 110 171 def parse_macro_file(self): 111 ext_com = self.find_extended_comments() 112 desc_com = self.find_descriptive_comments() 113 def_macro = self.find_def_macro() 114 cdef_macro = self.find_cdef_macro() 115 vars = self.find_variables() 116 117 for linenumber in range(len(self.line_positions)): 118 # TODO: decide the parent for each item, expect all def are at global scope 119 # TODO: decide which macros and variables should not be documented 120 # walk through the line numbers in the file 121 # if a def_macro starts, note its name and set the parent field 122 # of all comments, variables, var_desc, rdef, and cdef within 123 # its start_line and end_line range 124 # How to handle descriptive comments? 125 pass 126 172 """ 173 Figure out what can be documented in the file's contents (in self.buf) 174 175 each of the list_...() methods returns a 176 list of dictionaries where each dictionary 177 has the keys: objtype, start_line, end_line, and others 178 """ 179 db = {} 180 # first, the file parsing 181 for func in (self.list_def_macros, 182 self.list_cdef_macros, 183 self.list_variables, 184 self.list_extended_comments, 185 self.list_descriptive_comments, 186 ): 187 for item in func(): 188 s = item['start_line'] 189 if s not in db.keys(): 190 db[s] = [] 191 db[s].append(item) 192 193 # then, the analysis of what was found 127 194 self.findings = [] 128 for item in (ext_com, desc_com, def_macro, cdef_macro, vars,): 129 if len(item)>0: 130 self.findings.extend(item) 131 132 extended_comment_block_sig_re = re.compile( 133 string_start 134 + non_greedy_whitespace 135 + extended_comment_marker 136 + r'(' + non_greedy_filler + r')' 137 + extended_comment_marker 138 + non_greedy_filler 139 + string_end, 140 re.IGNORECASE|re.DOTALL|re.MULTILINE) 141 142 def find_extended_comments(self): 195 description = '' 196 clear_description = False 197 found_first_global_extended_comment = False 198 for linenumber in sorted(db.keys()): 199 #print linenumber, ':', ', '.join([d['objtype'] for d in db[linenumber]]) 200 201 line = db[linenumber] 202 item = line[-1] 203 if item['objtype'] in ('def', 'function def'): 204 # identify all the children of this item 205 parent = item['name'] 206 found_first_local_extended_comment = False 207 for row in xrange(item['start_line']+1, item['end_line']-1): 208 if row in db.keys(): 209 for thing in db[row]: 210 thing['parent'] = parent 211 if thing['objtype'] == 'extended comment': 212 if not found_first_local_extended_comment: 213 # TODO: could override this rule with an option 214 item['description'] = thing['text'] 215 found_first_local_extended_comment = False 216 if not item['name'].startswith('_'): 217 # TODO: could override this rule with an option 218 self.findings.append(item) 219 220 if item['objtype'] == 'extended comment': 221 start = item['start_line'] 222 if item['parent'] == None: 223 if not found_first_global_extended_comment: 224 # TODO: could override this rule with an option 225 self.findings.append(item) 226 found_first_global_extended_comment = True 227 228 if item['objtype'] == 'descriptive comment': 229 description = item['text'] 230 231 for item in line: 232 if item['objtype'] in ('local', 'global', 'constant', 'rdef', 'cdef'): 233 if len(description)>0: 234 item['description'] = description 235 clear_description = True 236 if not item['name'].startswith('_'): 237 # TODO: could override this rule with an option 238 self.findings.append(item) 239 240 if clear_description: 241 description, clear_description = '', False 242 243 def list_extended_comments(self): 143 244 """ 144 245 parse the internal buffer for triple-quoted strings, possibly multiline 246 247 Usually, an extended comment is used at the top of a macro file 248 to describe the file's contents. It is also used at the top 249 of a macro definition to describe the macro. The first line 250 of an extended comment for a macro should be a short summary, 251 followed by a blank line, then either a parameter list or 252 more extensive documentation, as needed. 145 253 """ 146 254 items = [] 147 for mo in self.extended_comment_block_sig_re.finditer(self.buf):255 for mo in extended_comment_block_sig_re.finditer(self.buf): 148 256 start = self.find_pos_in_line_number(mo.start(1)) 149 257 end = self.find_pos_in_line_number(mo.end(1)) … … 157 265 }) 158 266 return items 159 160 variable_description_re = re.compile( 161 string_start 162 + non_greedy_filler 163 + r'#:' 164 + non_greedy_whitespace 165 + r'(' + non_greedy_filler + r')' 166 + non_greedy_whitespace 167 + string_end, 168 re.IGNORECASE|re.DOTALL|re.MULTILINE) 169 170 def find_descriptive_comments(self): 267 268 def list_descriptive_comments(self): 171 269 """ 172 270 Descriptive comments are used to document items that cannot contain … … 185 283 """ 186 284 items = [] 187 for mo in self.variable_description_re.finditer(self.buf):285 for mo in variable_description_re.finditer(self.buf): 188 286 start = self.find_pos_in_line_number(mo.start(1)) 189 287 end = self.find_pos_in_line_number(mo.end(1)) … … 191 289 'start_line': start, 192 290 'end_line': end, 193 'objtype': ' variable description',291 'objtype': 'descriptive comment', 194 292 'text': mo.group(1), 195 293 'parent': None, 196 294 }) 197 295 return items 198 199 lgc_variable_sig_re = re.compile( 200 r'' 201 + string_start 202 + non_greedy_whitespace 203 + r'(local|global|constant)' # 1: object type 204 + non_greedy_whitespace 205 + r'(' + non_greedy_filler + r')' # 2: too complicated to parse all at once 206 + string_end 207 , 208 re.DOTALL 209 |re.MULTILINE 210 ) 211 212 variable_name_re = re.compile( 213 variable_name_match, 214 re.IGNORECASE|re.DOTALL|re.MULTILINE 215 ) 216 217 def find_variables(self): 296 297 def list_variables(self): 218 298 """ 219 299 parse the internal buffer for local, global, and constant variable declarations 220 300 """ 221 301 items = [] 222 for mo in self.lgc_variable_sig_re.finditer(self.buf):302 for mo in lgc_variable_sig_re.finditer(self.buf): 223 303 start = self.find_pos_in_line_number(mo.start(1)) 224 304 end = self.find_pos_in_line_number(mo.end(1)) … … 226 306 content = mo.group(2) 227 307 p = content.find('#') 228 if p >= 0: # strip off any comment308 if p >= 0: # strip off any comment 229 309 content = content[:p] 230 content = re.sub('[,;]', ' ', content) # replace , or ; with blank space310 content = re.sub('[,;]', ' ', content) # replace , or ; with blank space 231 311 if content.find('[') >= 0: 232 content = re.sub('\s*?\[', '[', content) # remove blank space before [233 for var in self.variable_name_re.finditer(content):312 content = re.sub('\s*?\[', '[', content) # remove blank space before [ 313 for var in variable_name_re.finditer(content): 234 314 name = var.group(1) 235 315 if len(name) > 0: … … 240 320 'name': name, 241 321 'parent': None, 242 'text': 'FIX in find_variables(self):',243 322 }) 244 323 return items 245 324 246 spec_macro_declaration_match_re = re.compile( 247 string_start 248 + r'\s*?' # optional blank space 249 + r'(r?def)' # 1: def_type (rdef | def) 250 + non_greedy_whitespace 251 + macro_name_match # 2: macro_name 252 + non_greedy_filler_match # 3: optional arguments 253 + r'\'\{?' # start body section 254 + non_greedy_filler_match # 4: body 255 + r'\}?\'' # end body section 256 + r'(#.*?)?' # 5: optional comment 257 + string_end, 258 re.IGNORECASE|re.DOTALL|re.MULTILINE) 259 260 args_match = re.compile( 261 r'\(' 262 + arglist_match # 1: argument list 263 + r'\)', 264 re.DOTALL) 265 266 def find_def_macro(self): 325 def list_def_macros(self): 267 326 """ 268 327 parse the internal buffer for def and rdef macro declarations 269 328 """ 270 329 items = [] 271 for mo in s elf.spec_macro_declaration_match_re.finditer(self.buf):330 for mo in spec_macro_declaration_match_re.finditer(self.buf): 272 331 objtype = mo.group(1) 273 332 start = self.find_pos_in_line_number(mo.start(1)) … … 275 334 args = mo.group(3) 276 335 if len(args)>2: 277 m = self.args_match.search(args)336 m = args_match.search(args) 278 337 if m is not None: 279 338 objtype = 'function ' + objtype … … 292 351 return items 293 352 294 def find_cdef_macro(self):353 def list_cdef_macros(self): 295 354 """ 296 355 parse the internal buffer for def and rdef macro declarations 297 356 """ 298 299 # note: It is not possible to find properly all variations 300 # of the argument list in a cdef declaration using a regular expression, 301 # especially across multiple lines. 302 357 # too complicated for a regular expression, just look for the initial part 303 358 items = [] 304 for mo in re.finditer('cdef\ (', self.buf):359 for mo in re.finditer('cdef\s*?\(', self.buf): 305 360 # look at each potential cdef declaration 306 361 objtype = 'cdef' 307 s = mo.start() 308 start = self.find_pos_in_line_number(s) 309 p = mo.end() 362 start = self.find_pos_in_line_number(mo.start()) 363 s = p = mo.end() # offset s for start of args 310 364 nesting = 1 # number of nested parentheses 311 365 sign = {'(': 1, ')': -1} # increment or decrement … … 315 369 p += 1 316 370 e = p 317 text = self.buf[s +5:e-1] # carve it out, and remove cdef( ... ) wrapping371 text = self.buf[s:e-1] # carve it out, and remove cdef( ... ) wrapping 318 372 end = self.find_pos_in_line_number(e) 319 373 p = text.find(',') … … 344 398 :param int pos: position in the file 345 399 """ 346 # straight search 347 # TODO: optimize using search by bisection 400 # TODO: optimize this straight search using a search by bisection 348 401 linenumber = None 349 402 for linenumber, start, end in self.line_positions: … … 356 409 def ReST(self): 357 410 """create the ReStructured Text from what has been found""" 358 # if not self.state == 'parsed':359 # raise RuntimeWarning, "state = %s, should be 'parsed'" % self.filename360 411 return self._simple_ReST_renderer() 361 412 362 413 def _simple_ReST_renderer(self): 363 414 """create a simple ReStructured Text rendition of the findings""" 364 # if not self.state == 'parsed':365 # raise RuntimeWarning, "state = %s, should be 'parsed'" % self.filename366 367 415 declarations = [] # variables and constants 368 416 macros = [] # def, cdef, and rdef macros … … 388 436 r['start_line'], 389 437 r['end_line']) ) 390 s.append( '' )391 # TODO: make this next be part of the signature display (in specdomain)392 #s.append( '.. rubric:: %s macro declaration' % r['objtype'] )393 s.append( '' )394 438 s.append( '.. spec:%s:: %s' % ( r['objtype'], r['name'],) ) 439 desc = r.get('description', '') 440 if len(desc) > 0: 441 s.append('') 442 for line in desc.splitlines(): 443 s.append(' '*4 + line) 395 444 elif r['objtype'] in ('function def', 'function rdef',): 396 445 # TODO: apply rules to suppress reporting under certain circumstances 397 446 functions.append(r) 447 objtype = r['objtype'].split()[1] 398 448 s.append( '' ) 399 449 s.append( '.. %s %s %s %d %d' % (self.filename, 400 r['objtype'],450 objtype, 401 451 r['name'], 402 452 r['start_line'], 403 453 r['end_line']) ) 404 s.append( '' ) 405 #s.append( '.. rubric:: %s macro function declaration' % r['objtype'] ) 406 s.append( '' ) 407 s.append( '.. spec:%s:: %s(%s)' % ( r['objtype'], r['name'], r['args']) ) 454 s.append( '.. spec:%s:: %s(%s)' % ( objtype, r['name'], r['args']) ) 455 desc = r.get('description', '') 456 if len(desc) > 0: 457 s.append('') 458 for line in desc.splitlines(): 459 s.append(' '*4 + line) 408 460 elif r['objtype'] in ('local', 'global', 'constant'): 409 461 # TODO: apply rules to suppress reporting under certain circumstances 410 del r['text']411 462 declarations.append(r) 412 413 s += report_table('Variable Declarations (%s)' % self.filename, declarations, ('objtype', 'name', 'start_line', )) 414 s += report_table('Macro Declarations (%s)' % self.filename, macros, ('objtype', 'name', 'start_line', 'end_line')) 415 s += report_table('Function Macro Declarations (%s)' % self.filename, functions, ('objtype', 'name', 'start_line', 'end_line', 'args')) 416 #s += report_table('Findings from .mac File', self.findings, ('start_line', 'objtype', 'line',)) 463 s.append( '.. spec:%s:: %s' % ( r['objtype'], r['name']) ) 464 desc = r.get('description', '') 465 if len(desc) > 0: 466 s.append('') 467 for line in desc.splitlines(): 468 s.append(' '*4 + line) 469 470 s += _report_table('Variable Declarations (%s)' % self.filename, declarations, 471 ('objtype', 'name', 'start_line',)) 472 s += _report_table('Macro Declarations (%s)' % self.filename, macros, 473 ('objtype', 'name', 'start_line', 'end_line')) 474 s += _report_table('Function Macro Declarations (%s)' % self.filename, functions, 475 ('objtype', 'name', 'start_line', 'end_line', 'args')) 476 #s += _report_table('Findings from .mac File', self.findings, ('start_line', 'objtype', 'line',)) 417 477 418 478 return '\n'.join(s) 419 479 420 480 421 def report_table(title, itemlist, col_keys = ('objtype', 'start_line', 'end_line', )):481 def _report_table(title, itemlist, col_keys = ('objtype', 'start_line', 'end_line', )): 422 482 """ 423 483 return the itemlist as a reST table … … 436 496 rows.append( tuple([str(d[key]).strip() for key in col_keys]) ) 437 497 last_line = d['start_line'] 438 return make_ table(title, col_keys, rows, '=')439 440 441 def make_ table(title, labels, rows, titlechar = '='):498 return make_rest_table(title, col_keys, rows, '=') 499 500 501 def make_rest_table(title, labels, rows, titlechar = '='): 442 502 """ 443 build a reST table (internal routine)503 build a reST table 444 504 445 505 :param str title: placed in a section heading above the table … … 448 508 :param str titlechar: character to use when underlining title as reST section heading 449 509 :returns [str]: each list item is reST 510 511 Example:: 512 513 title = 'This is a reST table' 514 labels = ('name', 'phone', 'email') 515 rows = [ 516 ['Snoopy', '12345', 'dog@house'], 517 ['Red Baron', '65432', 'fokker@triplane'], 518 ['Charlie Brown', '12345', 'main@house'], 519 ] 520 print '\n'.join(make_rest_table(title, labels, rows)) 521 522 This results in this reST:: 523 524 This is a reST table 525 ==================== 526 527 ============= ===== =============== 528 name phone email 529 ============= ===== =============== 530 Snoopy 12345 dog@house 531 Red Baron 65432 fokker@triplane 532 Charlie Brown 12345 main@house 533 ============= ===== =============== 450 534 """ 451 535 s = [] … … 467 551 s.append( fmt % labels ) 468 552 s.append( separator ) 469 s.extend( fmt % row for row in rows)553 s.extend( [fmt % tuple(row) for row in rows] ) 470 554 s.append( separator ) 471 555 return s … … 482 566 p = SpecMacrofileParser(filename) 483 567 print p.ReST() 484 pprint (p.findings)568 #pprint (p.findings)
Note: See TracChangeset
for help on using the changeset viewer.
