Changeset 350

Timestamp:

Mar 12, 2011 12:48:04 AM (14 years ago)

Author:

jemian

Message:

move the next() method into TokenLog?

Location:

topdoc/src/TopDoc

Files:

• EpicsDatabase.py (modified) (2 diffs)
• EpicsTemplate.py (modified) (10 diffs)
• TokenLog.py (modified) (1 diff)
• utilities.py (modified) (2 diffs)

topdoc/src/TopDoc/EpicsDatabase.py ¶

@@ -50 +50 @@
             self.filename = dbFilename
             self.absolute_filename = os.path.abspath(dbFilename)
-            self.buf = utilities.tokenizeTextFile(dbFilename)
+            self.tokenLog, self.buf = utilities.tokenizeTextFile(dbFilename)
             self._parse()
 
@@ -124 +124 @@
         into the internal memory structure.
         '''
+        # p_tkn, tkn = self.tokenLog.next()
+        # p_tkn, tkn = self.tokenLog.next(p_tkn)
         if self.buf is not None:
             pvDict = {}

topdoc/src/TopDoc/EpicsTemplate.py ¶

@@ -37 +37 @@
             self.filename = templateFilename
             self.absolute_filename = os.path.abspath(templateFilename)
-            self.buf = utilities.tokenizeTextFile(templateFilename)
+            self.tokenLog, self.buf = utilities.tokenizeTextFile(templateFilename)
             self._parse_()
 
@@ -54 +54 @@
         return db.getFull(macros)
 
-    def nextUsefulToken(self, ptr = -1):
-        '''
-        walk through the tokens and find the next actionable token
-        @param ptr: current buffer pointer, integer [0 .. len(self.buf)-1]
-        @return: tuple (ptr, self.buf[ptr]), where ptr points to an actionable token or (None, None)
-        '''
-        # TODO Should this be part of the TokenLog module?
-        ptr += 1
-        while ptr < len(self.buf):
-            tkn = self.buf[ptr]
-            if tkn['tokName'] not in ('COMMENT', 'NEWLINE', 'ENDMARKER'):
-                return ptr, tkn
-            ptr += 1
-        return None, None
-
     def _parse_(self):
         '''
         walk through the tokens and find the databases, then macros
         '''
-        p_tkn, tkn = self.nextUsefulToken()
+        p_tkn, tkn = self.tokenLog.next()
         while p_tkn is not None:
             if tkn['tokName'] == 'NAME' and tkn['tokStr'] == 'file':
-                p_tkn, tkn = self.nextUsefulToken(p_tkn)
+                p_tkn, tkn = self.tokenLog.next(p_tkn)
                 dbFile = utilities.strip_quotes( tkn['tokStr'] )
                 #
@@ -83 +68 @@
                 # TODO macro expansion of dbFile
                 #
-                p_tkn, tkn = self.nextUsefulToken(p_tkn)
+                p_tkn, tkn = self.tokenLog.next(p_tkn)
                 if tkn['tokStr'] != '{':
                     raise Exception, "problem token: " + str(tkn)
                 else:
-                    p_tkn, tkn = self.nextUsefulToken(p_tkn)
+                    p_tkn, tkn = self.tokenLog.next(p_tkn)
                     if tkn['tokStr'] == 'pattern':
                         # get the macro names, then
@@ -98 +83 @@
                         raise Exception, "problem token: " + str(tkn)
                 # start processing the macro set(s)
-            p_tkn, tkn = self.nextUsefulToken(p_tkn)
+            p_tkn, tkn = self.tokenLog.next(p_tkn)
 
     def _gather_inline_macros_(self, dbFile, p_tkn):
@@ -113 +98 @@
             buf = ""
             while tkn['tokStr'] != '}':
-                p_tkn, tkn = self.nextUsefulToken(p_tkn)
+                p_tkn, tkn = self.tokenLog.next(p_tkn)
                 s = tkn['tokStr']
                 if s == '=':
@@ -126 +111 @@
             # TODO do something with db
             print dbFile, macros
-            p_tkn, tkn = self.nextUsefulToken(p_tkn)
+            p_tkn, tkn = self.tokenLog.next(p_tkn)
         return p_tkn, tkn
 
@@ -139 +124 @@
         tkn = self.buf[p_tkn]
         keyList = []
-        p_tkn, tkn = self.nextUsefulToken(p_tkn)
+        p_tkn, tkn = self.tokenLog.next(p_tkn)
         if tkn['tokStr'] != '{':
             raise Exception, "problem token: " + str(tkn)
         else:
-            p_tkn, tkn = self.nextUsefulToken(p_tkn)
+            p_tkn, tkn = self.tokenLog.next(p_tkn)
             # gather the macro keys in a list
             while tkn['tokStr'] != '}':
@@ -152 +137 @@
                 if key != ',':
                     keyList.append( key )
-                p_tkn, tkn = self.nextUsefulToken(p_tkn)
-            p_tkn, tkn = self.nextUsefulToken(p_tkn)
+                p_tkn, tkn = self.tokenLog.next(p_tkn)
+            p_tkn, tkn = self.tokenLog.next(p_tkn)
             # loop through each set of macro definitions
             while tkn['tokStr'] == '{':
-                p_tkn, tkn = self.nextUsefulToken(p_tkn)
+                p_tkn, tkn = self.tokenLog.next(p_tkn)
                 buf = ""
                 liszt = []
@@ -169 +154 @@
                         buf += tkn['tokStr']
                         lastend = tkn['end'][1]
-                    p_tkn, tkn = self.nextUsefulToken(p_tkn)
+                    p_tkn, tkn = self.tokenLog.next(p_tkn)
                 liszt.append( buf )
                 if len(keyList) != len(liszt):
@@ -184 +169 @@
                 # TODO do something with db
                 print dbFile, macros
-                p_tkn, tkn = self.nextUsefulToken(p_tkn)
+                p_tkn, tkn = self.tokenLog.next(p_tkn)
         return p_tkn, tkn
 

topdoc/src/TopDoc/TokenLog.py ¶

@@ -169 +169 @@
         return lines
 
+    def next(self, ptr = -1):
+        '''
+        walk through the tokens and find the next actionable token
+        @param ptr: current buffer pointer, integer [0 .. len(self.tokenList)-1]
+        @return: tuple (ptr, self.tokenList[ptr]), where ptr points to an actionable token or (None, None)
+        '''
+        ptr += 1
+        while ptr < len(self.tokenList):
+            tkn = self.tokenList[ptr]
+            if tkn['tokName'] not in ('COMMENT', 'NEWLINE', 'ENDMARKER'):
+                return ptr, tkn
+            ptr += 1
+        return None, None
+
 
 ######################################################################

topdoc/src/TopDoc/utilities.py ¶

@@ -16 +16 @@
 import re
 import TokenLog
-
 
 
@@ -128 +127 @@
     '''
     @param filename: string name of file to be read
-    @return: structure from TokenLog.lineAnalysis()
+    @return: tuple of (TokenLog object, structure from TokenLog.lineAnalysis())
     '''
     obj = TokenLog.TokenLog()
     obj.processFile(filename)
-    return obj.getTokenList()
+    return obj, obj.getTokenList()