artem3213212/AGPascalTokenizer
1
0
Fork 0
Code Issues 3 Pull Requests Releases Wiki Activity

Add stack

Browse Source
This commit is contained in:
Artem3213212
2018-11-22 12:28:37 +02:00
parent 2e07350571
commit b837d46736
1 changed files with 168 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files

273
AG.PascalTokenizer.pas
View File

@@ -44,18 +44,25 @@ type
property Pos:TAGTokenizerPos read GetPos write SetPos; property Pos:TAGTokenizerPos read GetPos write SetPos;
end; end;
{PasTokenizerStack = class {$IFNDEF FPC}
private TAGPasTokenizerStack=class
stack: TStack<integer>; strict protected
// _pop type
procedure _get_with_comments(); GetCall=reference to function(Tokenizer:TAGPasTokenizer):TAGToken;
procedure _get_without_comments(); var
Stack:TStack<TAGToken>;
Tokenizer:TAGPasTokenizer;
Get:GetCall;
function GetLast():TAGToken;
function IsEnded():Boolean;
public public
procedure push(s: string); constructor Create(input:TStrings;GetComments:boolean=True);
procedure pop(); procedure Push(t:TAGToken);
procedure read_last(); function Pop():TAGToken;
procedure is_ended(); property Last:TAGToken read GetLast write Push;
end;} property Ended:Boolean read IsEnded;
end;
{$ENDIF}
function IsComment(s: string): boolean; function IsComment(s: string): boolean;
function IsName(s: string): boolean; function IsName(s: string): boolean;
@@ -182,123 +189,124 @@ var
f{$IFDEF FPC},ff{$ENDIF}:boolean; f{$IFDEF FPC},ff{$ENDIF}:boolean;
begin_pos:TAGTokenizerPos; begin_pos:TAGTokenizerPos;
begin begin
ml := ''; ml:='';
ss := ''; ss:='';
f := True; f:=True;
begin_pos := GetPos(); begin_pos:=GetPos();
while f and not ended do while f and not ended do
begin begin
line := s[y]; line:=s[y];
now_sym := line[x]; now_sym:=line[x];
l := length(line); l:=length(line);
if x<>l+Fix then if x<>l+Fix then
next_sym := line[x + 1] next_sym:=line[x+1]
else else
next_sym := #0; next_sym:=#0;
if ml='' then if ml='' then
begin begin
if now_sym = '/' then if now_sym='/' then
begin begin
if next_sym = '/' then if next_sym='/' then
begin begin
for i:=x to l+Fix do for i:=x to l+Fix do
ss:=ss+line[i]; ss:=ss+line[i];
x := l+Fix; x:=l+Fix;
break; break;
end; end;
end end
else if now_sym = '{' then else if now_sym='{' then
begin begin
ml := '}'; ml:='}';
ss := now_sym; ss:=now_sym;
last_i0 := y; last_i0:=y;
end end
else if now_sym = '(' then else if now_sym='(' then
begin begin
if next_sym = '*' then if next_sym='*' then
begin begin
ml := ')'; ml:=')';
inc(x); inc(x);
last_i0 := y; last_i0:=y;
ss := now_sym + next_sym; ss:=now_sym+next_sym;
end end
else else
begin begin
ss := '('; ss:='(';
inc(x); inc(x);
break; break;
end; end;
end end
else else
begin begin
if SYMS1.Contains(now_sym) then if SYMS1.Contains(now_sym)then
begin begin
ss := now_sym; ss:=now_sym;
inc(x); inc(x);
if SYMS2.{$IFDEF FPC} if SYMS2.
{$IFDEF FPC}
IndexOf(now_sym+next_sym)<>-1 IndexOf(now_sym+next_sym)<>-1
{$ELSE} {$ELSE}
Contains(now_sym+next_sym) Contains(now_sym+next_sym)
{$ENDIF}then {$ENDIF}then
begin begin
inc(x); inc(x);
ss := ss + next_sym; ss:=ss+next_sym;
end; end;
break; break;
end end
else if now_sym = #39 then else if now_sym=#39 then
begin begin
ss := #39; ss:=#39;
inc(x); inc(x);
if next_sym <> '' then if next_sym<>'' then
begin begin
ss := ss + next_sym; ss:=ss+next_sym;
while line[x] <> #39 do while line[x] <> #39 do
begin begin
inc(x); inc(x);
if not IsReadable() then if not IsReadable() then
begin begin
dec(x); dec(x);
break; break;
end; end;
ss := ss + line[x]; ss:=ss+line[x];
end; end;
inc(x); inc(x);
end; end;
break; break;
end end
else else
begin begin
while not NO_NAME_SYMS.Contains(line[x]) do while not NO_NAME_SYMS.Contains(line[x]) do
begin begin
ss := ss + line[x]; ss:=ss+line[x];
inc(x); inc(x);
if not IsReadable() then if not IsReadable() then
break; break;
end; end;
break; break;
end; end;
end; end;
end end
else else
begin begin
while last_i0 <> y do while last_i0<>y do
begin begin
ss := ss + #10; ss:=ss+#10;
inc(last_i0); inc(last_i0);
end; end;
ss:=ss+now_sym; ss:=ss+now_sym;
if now_sym = ml then if now_sym=ml then
if ml = '}' then if ml='}' then
begin begin
inc(x); inc(x);
break; break;
end end
else if (x <> 0) and (line[x - 1] = '*') then else if(x<>0)and(line[x-1]='*')then
begin begin
inc(x); inc(x);
break; break;
end; end;
end; end;
NextReadable(); NextReadable();
end; end;
@@ -308,7 +316,7 @@ begin
Result.&end:=GetPos; Result.&end:=GetPos;
Result.ended:=ended; Result.ended:=ended;
{$ELSE} {$ELSE}
Result := TAGToken.Create(ss, begin_pos, GetPos, ended); Result:=TAGToken.Create(ss,begin_pos,GetPos,ended);
{$ENDIF} {$ENDIF}
SkipSpaces; SkipSpaces;
end; end;
@@ -322,6 +330,61 @@ begin
SkipSpaces; SkipSpaces;
end; end;
{$IFNDEF FPC}
{TAGPasTokenizerStack}
function TAGPasTokenizerStack.GetLast():TAGToken;
begin
if Stack.Count<>0 then
Result:=Stack.Peek
else
begin
Result:=Get;
Stack.Push(Result);
end;
end;
function TAGPasTokenizerStack.IsEnded():Boolean;
begin
end;
constructor TAGPasTokenizerStack.Create(input:TStrings;GetComments:boolean=True);
begin
Stack:=TStack<TAGToken>.Create();
Tokenizer:=TAGPasTokenizer.Create(input);
if GetComments then
Get:=function(Tokenizer:TAGPasTokenizer):TAGToken
begin
Result:=Tokenizer.GetNext;
end
else
Get:=function(Tokenizer:TAGPasTokenizer):TAGToken
begin
while True do
begin
Result:=Tokenizer.GetNext;
if Result.ended or not IsComment(Result.Text) then
break;
end;
end;
end;
procedure TAGPasTokenizerStack.Push(t:TAGToken);
begin
Stack.Push(t);
end;
function TAGPasTokenizerStack.Pop():TAGToken;
begin
if Stack.Count<>0 then
Result:=Stack.Pop
else
Result:=Get;
end;
{$ENDIF}
initialization initialization
SYMS2 := {$IFDEF FPC}TStringList{$ELSE}TList<string>{$ENDIF}.Create(); SYMS2 := {$IFDEF FPC}TStringList{$ELSE}TList<string>{$ENDIF}.Create();
SYMS2.Add('>='); SYMS2.Add('>=');