artem3213212/AGPascalTokenizer
1
0
Fork 0
Code Issues 3 Pull Requests Releases Wiki Activity

Refactoring

Browse Source
This commit is contained in:
Artem3213212
2018-10-29 14:46:33 +02:00
parent dfd4a5e4bd
commit 886cbb2974
5 changed files with 108 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files

100
AG.PascalTokenizer.pas
View File

@@ -12,34 +12,34 @@ uses
type type
{$IFDEF FPC}TFpcList=specialize TFPGList<String>;{$ENDIF} {$IFDEF FPC}TFpcList=specialize TFPGList<String>;{$ENDIF}
TTokenizerPos = record TAGTokenizerPos = record
x, y: integer; x, y: integer;
end; end;
TToken = record TAGToken = record
Text: string; Text: string;
&begin, &end: TTokenizerPos; &begin, &end: TAGTokenizerPos;
ended: boolean; ended: boolean;
{$IFNDEF FPC}constructor Create(Text: string; &begin, &end: TTokenizerPos;ended:boolean);{$ENDIF} {$IFNDEF FPC}constructor Create(Text: string; &begin, &end: TAGTokenizerPos;ended:boolean);{$ENDIF}
end; end;
TPasTokenizer = class TAGPasTokenizer = class
private strict protected
s: TStrings; s: TStrings;
y: integer; y: integer;
x: integer; x: integer;
ended: boolean; function DoReadable():boolean;
function _do_readable(): boolean; function IsReadable():boolean;
function _is_readable(): boolean; function NextReadable():boolean;
function _next_readable(): boolean; procedure SkipSpaces();
procedure _skip_spaces(); function GetPos():TAGTokenizerPos;
function _get_pos(): TTokenizerPos; procedure SetPos(pos:TAGTokenizerPos);
procedure _set_pos(i0: integer; i1: integer);
public public
function get_next(): TToken; ended: boolean;
function GetNext(): TAGToken;
// procedure read_next(); // procedure read_next();
// procedure is_ended();
constructor Create(input:TStrings); constructor Create(input:TStrings);
property Pos:TAGTokenizerPos read GetPos write SetPos;
end; end;
{PasTokenizerStack = class {PasTokenizerStack = class
@@ -55,14 +55,14 @@ type
procedure is_ended(); procedure is_ended();
end;} end;}
function is_comment(s: string): boolean; function IsComment(s: string): boolean;
function is_name(s: string): boolean; function IsName(s: string): boolean;
function is_string(s: string): boolean; function IsString(s: string): boolean;
implementation implementation
{$IFNDEF FPC} {$IFNDEF FPC}
constructor TToken.Create(Text: string; &begin, &end: TTokenizerPos; constructor TAGToken.Create(Text: string; &begin, &end: TAGTokenizerPos;
ended: boolean); ended: boolean);
begin begin
Self.Text := Text; Self.Text := Text;
@@ -83,12 +83,12 @@ const
var var
SYMS2:{$IFDEF FPC}TFpcList{$ELSE}TList<string>{$ENDIF}; // array[0..8]of string=(); SYMS2:{$IFDEF FPC}TFpcList{$ELSE}TList<string>{$ENDIF}; // array[0..8]of string=();
function is_comment(s: string): boolean; function IsComment(s:string):boolean;
begin begin
Result:=(s.startswith('{') or s.startswith('(*') or s.startswith('//')); Result:=(s.startswith('{') or s.startswith('(*') or s.startswith('//'));
end; end;
function is_name(s: string): boolean; function IsName(s:string):boolean;
var var
i: integer; i: integer;
begin begin
@@ -103,16 +103,17 @@ begin
if not CHARS_ID.Contains(s[i]) then if not CHARS_ID.Contains(s[i]) then
Exit(False); Exit(False);
end; end;
Result:=True;
end; end;
function is_string(s: string): boolean; function IsString(s: string):boolean;
begin begin
Result:=s.StartsWith(#39); Result:=s.StartsWith(#39);
end; end;
function TPasTokenizer._do_readable(): boolean; function TAGPasTokenizer.DoReadable(): boolean;
begin begin
if not _is_readable() then if not IsReadable() then
begin begin
if (y + 1 = s.Count) then if (y + 1 = s.Count) then
ended := True ended := True
@@ -136,54 +137,53 @@ begin
Exit(False); Exit(False);
end; end;
function TPasTokenizer._is_readable(): boolean; function TAGPasTokenizer.IsReadable(): boolean;
begin begin
Exit(length(s[y])+1+Fix > x); Exit(length(s[y])+1+Fix > x);
end; end;
function TPasTokenizer._next_readable(): boolean; function TAGPasTokenizer.NextReadable(): boolean;
begin begin
inc(x); inc(x);
Result := _do_readable(); Result := DoReadable();
end; end;
procedure TPasTokenizer._skip_spaces(); procedure TAGPasTokenizer.SkipSpaces();
begin begin
_do_readable(); DoReadable();
if not ended then if not ended then
begin begin
while SPACES.Contains(s[y][x]) do while SPACES.Contains(s[y][x]) do
_next_readable(); NextReadable();
end; end;
end; end;
function TPasTokenizer._get_pos(): TTokenizerPos; function TAGPasTokenizer.GetPos(): TAGTokenizerPos;
begin begin
Result.x := x; Result.x := x;
Result.y := y; Result.y := y;
end; end;
procedure TPasTokenizer._set_pos(i0: integer; i1: integer); procedure TAGPasTokenizer.SetPos(pos:TAGTokenizerPos);
begin begin
y := i0; y:=Pos.x;
x := i1; x:=Pos.y;
ended := False; ended:=False;
_do_readable(); DoReadable();
end; end;
function TPasTokenizer.get_next(): TToken; function TAGPasTokenizer.GetNext(): TAGToken;
var var
l,i,last_i0:integer; l,i,last_i0:integer;
ml,ss,line:string; ml,ss,line:string;
now_sym,next_sym:char; now_sym,next_sym:char;
f,{$IFDEF FPC}ff,{$ENDIF}str_changed:boolean; f{$IFDEF FPC},ff{$ENDIF}:boolean;
begin_pos:TTokenizerPos; begin_pos:TAGTokenizerPos;
begin begin
ml := ''; ml := '';
ss := ''; ss := '';
f := True; f := True;
str_changed := True; begin_pos := GetPos();
begin_pos := _get_pos();
while f and not ended do while f and not ended do
begin begin
line := s[y]; line := s[y];
@@ -261,7 +261,7 @@ begin
while line[x] <> #39 do while line[x] <> #39 do
begin begin
inc(x); inc(x);
if not _is_readable() then if not IsReadable() then
begin begin
dec(x); dec(x);
break; break;
@@ -278,7 +278,7 @@ begin
begin begin
ss := ss + line[x]; ss := ss + line[x];
inc(x); inc(x);
if not _is_readable() then if not IsReadable() then
break; break;
end; end;
break; break;
@@ -305,26 +305,26 @@ begin
break; break;
end; end;
end; end;
_next_readable(); NextReadable();
end; end;
{$IFDEF FPC} {$IFDEF FPC}
Result.Text:=ss; Result.Text:=ss;
Result.&begin:=begin_pos; Result.&begin:=begin_pos;
Result.&end:=_get_pos; Result.&end:=GetPos;
Result.ended:=ended; Result.ended:=ended;
{$ELSE} {$ELSE}
Result := TToken.Create(ss, begin_pos, _get_pos, ended); Result := TAGToken.Create(ss, begin_pos, GetPos, ended);
{$ENDIF} {$ENDIF}
_skip_spaces; SkipSpaces;
end; end;
constructor TPasTokenizer.Create(input:TStrings); constructor TAGPasTokenizer.Create(input:TStrings);
begin begin
s:=input; s:=input;
y:=0; y:=0;
x:=1+fix; x:=1+fix;
ended:=False; ended:=False;
_skip_spaces; SkipSpaces;
end; end;
initialization initialization

18
DelphiTests/MainTest.pas
View File

@@ -28,16 +28,16 @@ implementation
procedure TMyTestObject.Test1; procedure TMyTestObject.Test1;
var var
input:TStrings; input:TStrings;
tokenizer:TPasTokenizer; tokenizer:TAGPasTokenizer;
token:TToken; token:TAGToken;
begin begin
input:= TStringList.Create(); input:= TStringList.Create();
input.LoadFromFile('..\..\MainTest.pas'); input.LoadFromFile('..\..\MainTest.pas');
tokenizer:=TPasTokenizer.Create(input); tokenizer:=TAGPasTokenizer.Create(input);
token.ended:=False; token.ended:=False;
while not token.ended do while not token.ended do
begin begin
token:=tokenizer.get_next; token:=tokenizer.GetNext;
TDUnitX.CurrentRunner.Log(TLogLevel.Information, token.Text); TDUnitX.CurrentRunner.Log(TLogLevel.Information, token.Text);
end; end;
//sleep(10000); //sleep(10000);
@@ -48,10 +48,10 @@ var
s:string; s:string;
begin begin
s:=#39'kek'#39; s:=#39'kek'#39;
if not is_string(s) then if not IsString(s) then
raise Exception.Create('Is string error 1'); raise Exception.Create('Is string error 1');
s:='s:=12334;'; s:='s:=12334;';
if is_string(s) then if IsString(s) then
raise Exception.Create('Is string error 2'); raise Exception.Create('Is string error 2');
end; end;
@@ -60,13 +60,13 @@ var
s:string; s:string;
begin begin
s:='{ asdasdasd }'; s:='{ asdasdasd }';
if not is_comment(s) then if not IsComment(s) then
raise Exception.Create('Is comment error 1'); raise Exception.Create('Is comment error 1');
s:='(* s:=12334;*)'; s:='(* s:=12334;*)';
if not is_comment(s) then if not IsComment(s) then
raise Exception.Create('Is comment error 2'); raise Exception.Create('Is comment error 2');
s:='// s:=12334;*)'; s:='// s:=12334;*)';
if not is_comment(s) then if not IsComment(s) then
raise Exception.Create('Is comment error 3'); raise Exception.Create('Is comment error 3');
end; end;

10
FPCTests/fpcunitproject1.lpi
View File

@@ -9,11 +9,7 @@
<Title Value="fpcunitproject1"/> <Title Value="fpcunitproject1"/>
<ResourceType Value="res"/> <ResourceType Value="res"/>
<UseXPManifest Value="True"/> <UseXPManifest Value="True"/>
<Icon Value="0"/>
</General> </General>
<VersionInfo>
<StringTable ProductVersion=""/>
</VersionInfo>
<BuildModes Count="1"> <BuildModes Count="1">
<Item1 Name="Default" Default="True"/> <Item1 Name="Default" Default="True"/>
</BuildModes> </BuildModes>
@@ -36,7 +32,7 @@
<PackageName Value="FCL"/> <PackageName Value="FCL"/>
</Item3> </Item3>
</RequiredPackages> </RequiredPackages>
<Units Count="2"> <Units Count="3">
<Unit0> <Unit0>
<Filename Value="fpcunitproject1.lpr"/> <Filename Value="fpcunitproject1.lpr"/>
<IsPartOfProject Value="True"/> <IsPartOfProject Value="True"/>
@@ -46,6 +42,10 @@
<IsPartOfProject Value="True"/> <IsPartOfProject Value="True"/>
<UnitName Value="TestCase1"/> <UnitName Value="TestCase1"/>
</Unit1> </Unit1>
<Unit2>
<Filename Value="..\AG.PascalTokenizer.pas"/>
<IsPartOfProject Value="True"/>
</Unit2>
</Units> </Units>
</ProjectOptions> </ProjectOptions>
<CompilerOptions> <CompilerOptions>

79
FPCTests/fpcunitproject1.lps
View File

@@ -16,9 +16,10 @@
<Filename Value="testcase1.pas"/> <Filename Value="testcase1.pas"/>
<IsPartOfProject Value="True"/> <IsPartOfProject Value="True"/>
<UnitName Value="TestCase1"/> <UnitName Value="TestCase1"/>
<IsVisibleTab Value="True"/>
<EditorIndex Value="1"/> <EditorIndex Value="1"/>
<TopLine Value="7"/> <TopLine Value="13"/>
<CursorPos X="68" Y="18"/> <CursorPos X="23" Y="31"/>
<UsageCount Value="20"/> <UsageCount Value="20"/>
<Loaded Value="True"/> <Loaded Value="True"/>
</Unit1> </Unit1>
@@ -32,11 +33,11 @@
</Unit2> </Unit2>
<Unit3> <Unit3>
<Filename Value="..\AG.PascalTokenizer.pas"/> <Filename Value="..\AG.PascalTokenizer.pas"/>
<IsVisibleTab Value="True"/> <IsPartOfProject Value="True"/>
<EditorIndex Value="2"/> <EditorIndex Value="2"/>
<TopLine Value="278"/> <TopLine Value="299"/>
<CursorPos X="9" Y="295"/> <CursorPos X="20" Y="313"/>
<UsageCount Value="10"/> <UsageCount Value="20"/>
<Loaded Value="True"/> <Loaded Value="True"/>
</Unit3> </Unit3>
<Unit4> <Unit4>
@@ -51,123 +52,123 @@
<JumpHistory Count="30" HistoryIndex="29"> <JumpHistory Count="30" HistoryIndex="29">
<Position1> <Position1>
<Filename Value="..\AG.PascalTokenizer.pas"/> <Filename Value="..\AG.PascalTokenizer.pas"/>
<Caret Line="211" TopLine="191"/> <Caret Line="189" TopLine="171"/>
</Position1> </Position1>
<Position2> <Position2>
<Filename Value="..\AG.PascalTokenizer.pas"/> <Filename Value="..\AG.PascalTokenizer.pas"/>
<Caret Line="212" TopLine="191"/> <Caret Line="190" TopLine="171"/>
</Position2> </Position2>
<Position3> <Position3>
<Filename Value="..\AG.PascalTokenizer.pas"/> <Filename Value="..\AG.PascalTokenizer.pas"/>
<Caret Line="312" TopLine="295"/> <Caret Line="191" TopLine="171"/>
</Position3> </Position3>
<Position4> <Position4>
<Filename Value="..\AG.PascalTokenizer.pas"/> <Filename Value="..\AG.PascalTokenizer.pas"/>
<Caret Line="189" TopLine="171"/> <Caret Line="192" TopLine="171"/>
</Position4> </Position4>
<Position5> <Position5>
<Filename Value="..\AG.PascalTokenizer.pas"/> <Filename Value="..\AG.PascalTokenizer.pas"/>
<Caret Line="190" TopLine="171"/> <Caret Line="193" TopLine="171"/>
</Position5> </Position5>
<Position6> <Position6>
<Filename Value="..\AG.PascalTokenizer.pas"/> <Filename Value="..\AG.PascalTokenizer.pas"/>
<Caret Line="191" TopLine="171"/> <Caret Line="196" TopLine="171"/>
</Position6> </Position6>
<Position7> <Position7>
<Filename Value="..\AG.PascalTokenizer.pas"/> <Filename Value="..\AG.PascalTokenizer.pas"/>
<Caret Line="192" TopLine="171"/> <Caret Line="295" TopLine="278"/>
</Position7> </Position7>
<Position8> <Position8>
<Filename Value="..\AG.PascalTokenizer.pas"/> <Filename Value="..\AG.PascalTokenizer.pas"/>
<Caret Line="193" TopLine="171"/> <Caret Line="300" TopLine="278"/>
</Position8> </Position8>
<Position9> <Position9>
<Filename Value="..\AG.PascalTokenizer.pas"/> <Filename Value="..\AG.PascalTokenizer.pas"/>
<Caret Line="196" TopLine="171"/> <Caret Line="312" TopLine="284"/>
</Position9> </Position9>
<Position10> <Position10>
<Filename Value="..\AG.PascalTokenizer.pas"/> <Filename Value="..\AG.PascalTokenizer.pas"/>
<Caret Line="295" TopLine="278"/> <Caret Line="189" TopLine="171"/>
</Position10> </Position10>
<Position11> <Position11>
<Filename Value="..\AG.PascalTokenizer.pas"/> <Filename Value="..\AG.PascalTokenizer.pas"/>
<Caret Line="300" TopLine="278"/> <Caret Line="190" TopLine="171"/>
</Position11> </Position11>
<Position12> <Position12>
<Filename Value="..\AG.PascalTokenizer.pas"/> <Filename Value="..\AG.PascalTokenizer.pas"/>
<Caret Line="312" TopLine="284"/> <Caret Line="191" TopLine="171"/>
</Position12> </Position12>
<Position13> <Position13>
<Filename Value="..\AG.PascalTokenizer.pas"/> <Filename Value="..\AG.PascalTokenizer.pas"/>
<Caret Line="189" TopLine="171"/> <Caret Line="192" TopLine="171"/>
</Position13> </Position13>
<Position14> <Position14>
<Filename Value="..\AG.PascalTokenizer.pas"/> <Filename Value="..\AG.PascalTokenizer.pas"/>
<Caret Line="190" TopLine="171"/> <Caret Line="193" TopLine="171"/>
</Position14> </Position14>
<Position15> <Position15>
<Filename Value="..\AG.PascalTokenizer.pas"/> <Filename Value="..\AG.PascalTokenizer.pas"/>
<Caret Line="191" TopLine="171"/> <Caret Line="196" TopLine="171"/>
</Position15> </Position15>
<Position16> <Position16>
<Filename Value="..\AG.PascalTokenizer.pas"/> <Filename Value="..\AG.PascalTokenizer.pas"/>
<Caret Line="192" TopLine="171"/> <Caret Line="295" TopLine="278"/>
</Position16> </Position16>
<Position17> <Position17>
<Filename Value="..\AG.PascalTokenizer.pas"/> <Filename Value="..\AG.PascalTokenizer.pas"/>
<Caret Line="193" TopLine="171"/> <Caret Line="300" TopLine="278"/>
</Position17> </Position17>
<Position18> <Position18>
<Filename Value="..\AG.PascalTokenizer.pas"/> <Filename Value="..\AG.PascalTokenizer.pas"/>
<Caret Line="196" TopLine="171"/> <Caret Line="312" TopLine="284"/>
</Position18> </Position18>
<Position19> <Position19>
<Filename Value="..\AG.PascalTokenizer.pas"/> <Filename Value="..\AG.PascalTokenizer.pas"/>
<Caret Line="295" TopLine="278"/> <Caret Line="189" TopLine="171"/>
</Position19> </Position19>
<Position20> <Position20>
<Filename Value="..\AG.PascalTokenizer.pas"/> <Filename Value="..\AG.PascalTokenizer.pas"/>
<Caret Line="300" TopLine="278"/> <Caret Line="190" TopLine="171"/>
</Position20> </Position20>
<Position21> <Position21>
<Filename Value="..\AG.PascalTokenizer.pas"/> <Filename Value="..\AG.PascalTokenizer.pas"/>
<Caret Line="312" TopLine="284"/> <Caret Line="191" TopLine="171"/>
</Position21> </Position21>
<Position22> <Position22>
<Filename Value="..\AG.PascalTokenizer.pas"/> <Filename Value="..\AG.PascalTokenizer.pas"/>
<Caret Line="189" TopLine="171"/> <Caret Line="192" TopLine="171"/>
</Position22> </Position22>
<Position23> <Position23>
<Filename Value="..\AG.PascalTokenizer.pas"/> <Filename Value="..\AG.PascalTokenizer.pas"/>
<Caret Line="190" TopLine="171"/> <Caret Line="193" TopLine="171"/>
</Position23> </Position23>
<Position24> <Position24>
<Filename Value="..\AG.PascalTokenizer.pas"/> <Filename Value="..\AG.PascalTokenizer.pas"/>
<Caret Line="191" TopLine="171"/> <Caret Line="196" TopLine="171"/>
</Position24> </Position24>
<Position25> <Position25>
<Filename Value="..\AG.PascalTokenizer.pas"/> <Filename Value="..\AG.PascalTokenizer.pas"/>
<Caret Line="192" TopLine="171"/> <Caret Line="295" Column="7" TopLine="278"/>
</Position25> </Position25>
<Position26> <Position26>
<Filename Value="..\AG.PascalTokenizer.pas"/> <Filename Value="testcase1.pas"/>
<Caret Line="193" TopLine="171"/> <Caret Line="28" Column="11" TopLine="4"/>
</Position26> </Position26>
<Position27> <Position27>
<Filename Value="..\AG.PascalTokenizer.pas"/> <Filename Value="testcase1.pas"/>
<Caret Line="196" TopLine="171"/> <Caret Line="29" Column="15" TopLine="4"/>
</Position27> </Position27>
<Position28> <Position28>
<Filename Value="..\AG.PascalTokenizer.pas"/> <Filename Value="..\AG.PascalTokenizer.pas"/>
<Caret Line="295" Column="7" TopLine="278"/> <Caret Line="295" Column="9"/>
</Position28> </Position28>
<Position29> <Position29>
<Filename Value="testcase1.pas"/> <Filename Value="..\AG.PascalTokenizer.pas"/>
<Caret Line="28" Column="11" TopLine="4"/> <Caret Line="313" Column="20" TopLine="299"/>
</Position29> </Position29>
<Position30> <Position30>
<Filename Value="testcase1.pas"/> <Filename Value="testcase1.pas"/>
<Caret Line="29" Column="15" TopLine="4"/> <Caret Line="23" Column="12" TopLine="8"/>
</Position30> </Position30>
</JumpHistory> </JumpHistory>
</ProjectSession> </ProjectSession>

8
FPCTests/testcase1.pas
View File

@@ -19,16 +19,16 @@ implementation
procedure TTestCase1.TestHookUp; procedure TTestCase1.TestHookUp;
var var
input:TStrings; input:TStrings;
tokenizer:TPasTokenizer; tokenizer:TAGPasTokenizer;
token:TToken; token:TAGToken;
begin begin
input:= TStringList.Create(); input:= TStringList.Create();
input.LoadFromFile('testcase1.pas'); input.LoadFromFile('testcase1.pas');
tokenizer:=TPasTokenizer.Create(input); tokenizer:=TAGPasTokenizer.Create(input);
token.ended:=False; token.ended:=False;
while not token.ended do while not token.ended do
begin begin
token:=tokenizer.get_next; token:=tokenizer.GetNext;
TestRunner.MemoLog.Append(token.Text); TestRunner.MemoLog.Append(token.Text);
end; end;
end; end;