This commit is contained in:
2018-08-26 22:07:34 +03:00
parent 5338449ca2
commit b227c23ce0
3 changed files with 44 additions and 45 deletions

View File

@@ -2,46 +2,9 @@ unit AG.PascalTokeniser;
interface interface
implementation type
TPasTokenizer=class
(* (*class PasTokenizer():
#
# PyPascalTokenizer
# Author: Artem Gavrilov (@Artem3213212)
# License: MPL 2.0
#
import queue, threading
SYMS1 = ['(',')','[',']','/','|','\\','@','#','=','>','<',':',';',',','.','$','+','-','*']
SYMS2 = ['>=','<=','<>',':=','..','-=','+=','/=','*=']
SPACES = ['\f','\n','\r','\t','\v',' ']
NO_NAME_SYMS = SYMS1 + SPACES + ['{','}']
CHARS_ID0 = '&abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_'
CHARS_ID = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_'
def is_comment(s):
if type(s) is list:
return True
else:
return s.startswith('{') or s.startswith('(*') or s.startswith('//')
def is_name(s):
if len(s)<=0:
return False
if s=='&':
return False
if not (s[0] in CHARS_ID0):
return False
for i in s[1:]:
if not (i in CHARS_ID):
return False
return True
def is_string(s):
return s.startswith("'")
class PasTokenizer():
def __init__(self, s): def __init__(self, s):
self.s, self.y, self.x, self.ended = s, 0, 0, False self.s, self.y, self.x, self.ended = s, 0, 0, False
self._skip_spaces() self._skip_spaces()
@@ -169,7 +132,43 @@ class PasTokenizer():
return z return z
def is_ended(self): def is_ended(self):
return self.ended return self.ended*)
end;
implementation
(*
import queue, threading
SYMS1 = ['(',')','[',']','/','|','\\','@','#','=','>','<',':',';',',','.','$','+','-','*']
SYMS2 = ['>=','<=','<>',':=','..','-=','+=','/=','*=']
SPACES = ['\f','\n','\r','\t','\v',' ']
NO_NAME_SYMS = SYMS1 + SPACES + ['{','}']
CHARS_ID0 = '&abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_'
CHARS_ID = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_'
def is_comment(s):
if type(s) is list:
return True
else:
return s.startswith('{') or s.startswith('(*') or s.startswith('//')
def is_name(s):
if len(s)<=0:
return False
if s=='&':
return False
if not (s[0] in CHARS_ID0):
return False
for i in s[1:]:
if not (i in CHARS_ID):
return False
return True
def is_string(s):
return s.startswith("'")
class PasTokenizerStack(): class PasTokenizerStack():
def __init__(self, s, comments=True): def __init__(self, s, comments=True):

View File

@@ -12,7 +12,7 @@ uses
DUnitX.Loggers.Xml.NUnit, DUnitX.Loggers.Xml.NUnit,
DUnitX.TestFramework, DUnitX.TestFramework,
MainTest in 'MainTest.pas', MainTest in 'MainTest.pas',
AGPascalTokeniser in '..\AGPascalTokeniser.pas'; AG.PascalTokeniser in '..\AG.PascalTokeniser.pas';
var var
runner : ITestRunner; runner : ITestRunner;

View File

@@ -93,7 +93,7 @@
<MainSource>MainSource</MainSource> <MainSource>MainSource</MainSource>
</DelphiCompile> </DelphiCompile>
<DCCReference Include="MainTest.pas"/> <DCCReference Include="MainTest.pas"/>
<DCCReference Include="..\AGPascalTokeniser.pas"/> <DCCReference Include="..\AG.PascalTokeniser.pas"/>
<BuildConfiguration Include="Release"> <BuildConfiguration Include="Release">
<Key>Cfg_2</Key> <Key>Cfg_2</Key>
<CfgParent>Base</CfgParent> <CfgParent>Base</CfgParent>
@@ -455,13 +455,13 @@
<Operation>1</Operation> <Operation>1</Operation>
</Platform> </Platform>
</DeployClass> </DeployClass>
<ProjectRoot Platform="Win64" Name="$(PROJECTNAME)"/> <ProjectRoot Platform="iOSDevice32" Name="$(PROJECTNAME).app"/>
<ProjectRoot Platform="Android" Name="$(PROJECTNAME)"/> <ProjectRoot Platform="Android" Name="$(PROJECTNAME)"/>
<ProjectRoot Platform="Win32" Name="$(PROJECTNAME)"/> <ProjectRoot Platform="Win32" Name="$(PROJECTNAME)"/>
<ProjectRoot Platform="iOSDevice64" Name="$(PROJECTNAME).app"/> <ProjectRoot Platform="iOSDevice64" Name="$(PROJECTNAME).app"/>
<ProjectRoot Platform="Win64" Name="$(PROJECTNAME)"/>
<ProjectRoot Platform="OSX32" Name="$(PROJECTNAME)"/> <ProjectRoot Platform="OSX32" Name="$(PROJECTNAME)"/>
<ProjectRoot Platform="iOSSimulator" Name="$(PROJECTNAME).app"/> <ProjectRoot Platform="iOSSimulator" Name="$(PROJECTNAME).app"/>
<ProjectRoot Platform="iOSDevice32" Name="$(PROJECTNAME).app"/>
</Deployment> </Deployment>
<Platforms> <Platforms>
<Platform value="OSX32">False</Platform> <Platform value="OSX32">False</Platform>