diff --git a/AG.PascalTokenizer.pas b/AG.PascalTokenizer.pas index 432276e..fb4b7b3 100644 --- a/AG.PascalTokenizer.pas +++ b/AG.PascalTokenizer.pas @@ -5,545 +5,520 @@ unit AG.PascalTokenizer; interface - uses - SysUtils, Classes, IniFiles, SyncObjs, Generics.Collections; +uses + SysUtils,Classes,IniFiles,SyncObjs,Generics.Collections; +type + TAGTokenizerPos=record + x,y:integer; + end; - type - TAGTokenizerPos = record - x, y: integer; - end; + TAGToken = record + Text:string; + &begin,&end:TAGTokenizerPos; + ended:boolean; + constructor Create(const Text:string;&begin,&end:TAGTokenizerPos;ended:boolean); + end; - TAGToken = record - Text:string; - &begin, &end: TAGTokenizerPos; - ended: boolean; - procedure Make(const Text:string;&begin, &end: TAGTokenizerPos;ended: boolean); - end; + TAGPasTokenizer=class + strict protected + FStrings:TStrings; + FLineIx,x:integer; + ended:boolean; + procedure DoReadable(); + function IsReadable():boolean; + procedure NextReadable(); + procedure SkipSpaces(); + function GetPos():TAGTokenizerPos; + procedure SetPos(pos:TAGTokenizerPos); + public + constructor Create(input:TStrings); + function GetNext():TAGToken; + property pos:TAGTokenizerPos read GetPos write SetPos; + property is_ended:boolean read ended; + end; - TAGPasTokenizer = class - strict protected - FStrings: TStrings; - FLineIx: integer; - x: integer; - function DoReadable(): boolean; - function IsReadable(): boolean; - function NextReadable(): boolean; - procedure SkipSpaces(); - function GetPos(): TAGTokenizerPos; - procedure SetPos(pos: TAGTokenizerPos); - public - ended: boolean; - function GetNext(): TAGToken; - // procedure read_next(); - constructor Create(input: TStrings); - property pos: TAGTokenizerPos read GetPos write SetPos; - end; - - TAGPasTokenizerStack = class - strict protected + TAGPasTokenizerStack = class + strict protected type - GetCall = function(Tokenizer: TAGPasTokenizer): TAGToken of object; - + GetCall=function(Tokenizer:TAGPasTokenizer):TAGToken of object; var - Stack: TQueue; - Tokenizer: TAGPasTokenizer; - Get: GetCall; - IsEnd: boolean; - function GetLast(): TAGToken;virtual; - function GetWithComments(Tokenizer: TAGPasTokenizer): TAGToken; - function GetWithoutComments(Tokenizer: TAGPasTokenizer): TAGToken; - destructor Destroy;override; - protected - function GetCachedCount: integer;inline; - public - constructor Create(input: TStrings;GetComments: boolean = True); - procedure Push(const t: TAGToken);virtual; - function Pop(): TAGToken;virtual; - property Last: TAGToken read GetLast write Push; - property ended: boolean read IsEnd; - end; + Stack:TQueue; + Tokenizer:TAGPasTokenizer; + Get:GetCall; + IsEnd:boolean; + function GetLast():TAGToken;virtual; + function GetWithComments(Tokenizer:TAGPasTokenizer):TAGToken; + function GetWithoutComments(Tokenizer:TAGPasTokenizer):TAGToken; + protected + function GetCachedCount:integer;inline; + public + constructor Create(input:TStrings;GetComments:boolean=True); + procedure Push(const t:TAGToken);virtual; + function Pop():TAGToken;virtual; + destructor Destroy;override; + property Last:TAGToken read GetLast write Push; + property ended:boolean read IsEnd; + end; - TAGPasTokenizerParallelStack = class(TAGPasTokenizerStack) - strict protected + TAGPasTokenizerParallelStack=class(TAGPasTokenizerStack) + strict protected type - TWorkerThread = class(TThread) + TWorkerThread=class(TThread) strict protected - FStack: TAGPasTokenizerParallelStack; + FStack:TAGPasTokenizerParallelStack; procedure Execute;override; public - Idling: boolean; - constructor Create(const Stack: TAGPasTokenizerParallelStack); + Idling:boolean; + constructor Create(const Stack:TAGPasTokenizerParallelStack); end; + var + FWorker:TWorkerThread; + FStackLock:TCriticalSection; + function AddTokenToStack:boolean; + function GetLast():TAGToken;override; + procedure EnsureThreadDone(); + protected + FStackHalfMax:integer; + FSignal:TEvent; + public + constructor Create(const input:TStrings;GetComments:boolean=True;stackMax:integer=1000); + procedure Push(const t:TAGToken);override; + function Pop():TAGToken;override; + destructor Destroy;override; + end; - var - FWorker: TWorkerThread; - FStackLock: TCriticalSection; - function AddTokenToStack: boolean; - function GetLast(): TAGToken;override; - procedure EnsureThreadDone(); - destructor Destroy;override; - protected - FStackHalfMax: integer; - FSignal: TEvent; - procedure Push(const t: TAGToken);override; - public - constructor Create(const input: TStrings;GetComments: boolean = True; - stackMax: integer = 1000); - function Pop(): TAGToken;override; - end; - - function IsComment(s:string): boolean; - function IsName(s:string): boolean; - function IsString(const s:string): boolean; +function IsComment(s:string):boolean; +function IsName(s:string): boolean; +function IsString(const s:string): boolean; implementation - procedure TAGToken.Make(const Text:string;&begin, &end: TAGTokenizerPos;ended: - boolean); +const + SYMS1='()[]/|\@#=><:;,.$+-*^'; + SPACES=#12#10#13#9#11' '; + NO_NAME_SYMS=SYMS1+SPACES+'{}'; + CHARS_ID0='&abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_'; + CHARS_ID='abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_'; + fix={$IFDEF NEXTGEN}-1{$ELSE}0{$ENDIF}; + +var + SYMS2:THashedStringList; + +function IsComment(s:string):boolean; +begin +Result:=s.startswith('{')or s.startswith('(*')or s.startswith('//'); +end; + +function IsName(s:string):boolean; +var + i:integer; +begin +if(length(s)=0)or(s='&')or(not CHARS_ID0.Contains(s[1+fix]))then + Exit(False); +for i:=2+fix to length(s)+fix do + if not CHARS_ID.Contains(s[i])then + Exit(False); +Result:=True; +end; + +function IsString(const s:string):boolean; +begin +Result:=s.startswith(#39); +end; + +{TAGToken} + +constructor TAGToken.Create(const Text:string;&begin,&end:TAGTokenizerPos;ended:boolean); +begin +Self.Text:=Text; +Self.&begin:=&begin; +Self.&end:=&end; +Self.ended:=ended; +end; + +{TAGPasTokenizer} + +procedure TAGPasTokenizer.DoReadable(); +begin +if not IsReadable()then + if FLineIx+1=FStrings.Count then + ended:=True + else begin - Self.Text := Text; - Self.&begin := &begin; - Self.&end := &end; - Self.ended := ended; - end; - - const - SYMS1 = '()[]/|\@#=><:;,.$+-*^'; - SPACES = #12#10#13#9#11' '; - NO_NAME_SYMS = SYMS1 + SPACES + '{}'; - CHARS_ID0 = '&abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_'; - CHARS_ID = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_'; - fix = {$IFDEF NEXTGEN}-1{$ELSE}0{$ENDIF}; - - var - SYMS2: THashedStringList; - - function IsComment(s:string): boolean; - begin - Result :=(s.startswith('{')or s.startswith('(*')or s.startswith('//')); - end; - - function IsName(s:string): boolean; - var - i: integer; - begin - if length(s)<= 0 then - Exit(False); - if s = '&' then - Exit(False); - if not CHARS_ID0.Contains(s[1 + fix])then - Exit(False); - for i := 1 to length(s)do + inc(FLineIx); + x:=1+fix; + while FStrings[FLineIx]<='' do begin - if not CHARS_ID.Contains(s[i])then - Exit(False); - end; - Result := True; - end; - - function IsString(const s:string): boolean; - begin - Result := s.startswith(#39); - end; - - function TAGPasTokenizer.DoReadable(): boolean; - begin - if not IsReadable()then - begin - if(FLineIx + 1 = FStrings.Count)then - ended := True - else + if FLineIx+1=FStrings.Count then begin - inc(FLineIx); - x := 1 + fix; - while FStrings[FLineIx]<= '' do - begin - if FLineIx + 1 = FStrings.Count then - begin - ended := True; - break; - end; - inc(FLineIx); - end; + ended:=True; + break; end; - Exit(True); - end - else - Exit(False); - end; - - function TAGPasTokenizer.IsReadable(): boolean; - begin - Result := x <= length(FStrings[FLineIx])+ fix; - end; - - function TAGPasTokenizer.NextReadable(): boolean; - begin - inc(x); - Result := DoReadable(); - end; - - procedure TAGPasTokenizer.SkipSpaces(); - begin - DoReadable(); - if not ended then - begin - while SPACES.Contains(FStrings[FLineIx][x])do - NextReadable(); + inc(FLineIx); end; end; +end; - function TAGPasTokenizer.GetPos(): TAGTokenizerPos; - begin - Result.x := x; - Result.y := FLineIx; - end; +function TAGPasTokenizer.IsReadable():boolean; +begin +Result:=x<=length(FStrings[FLineIx])+fix; +end; - procedure TAGPasTokenizer.SetPos(pos: TAGTokenizerPos); - begin - FLineIx := pos.x; - x := pos.y; - ended := False; - DoReadable(); - end; +procedure TAGPasTokenizer.NextReadable(); +begin +inc(x); +DoReadable(); +end; - function TAGPasTokenizer.GetNext(): TAGToken; - var - l, last_i0: integer; - ml, ss, line:string; - now_sym, next_sym: char; - f: boolean; - begin_pos: TAGTokenizerPos; +procedure TAGPasTokenizer.SkipSpaces(); +begin +DoReadable(); +if not ended then + while SPACES.Contains(FStrings[FLineIx][x])do + NextReadable(); +end; + +function TAGPasTokenizer.GetPos():TAGTokenizerPos; +begin +Result.x:=x; +Result.y:=FLineIx; +end; + +procedure TAGPasTokenizer.SetPos(pos:TAGTokenizerPos); +begin +FLineIx:=pos.x; +x:=pos.y; +ended:=False; +DoReadable(); +end; + +constructor TAGPasTokenizer.Create(input: TStrings); +begin +FStrings:=input; +FLineIx:=0; +x:=1+fix; +ended:=False; +SkipSpaces; +end; + +{$WARN USE_BEFORE_DEF OFF} +function TAGPasTokenizer.GetNext():TAGToken; +var + l,last_i0:integer; + ml,ss,line:string; + now_sym,next_sym:char; + f:boolean; + begin_pos:TAGTokenizerPos; +begin + ml:=''; + ss:=''; + f:=True; + begin_pos:=GetPos(); + while f and not ended do begin - ml := ''; - ss := ''; - f := True; - begin_pos := GetPos(); - while f and not ended do + line:=FStrings[FLineIx]; + now_sym:=line[x]; + l:=length(line); + if x-1 then begin - inc(x); - ss := ss + next_sym; - end; - break; - end - else if now_sym = #39 then - begin - ss := #39; - inc(x); - if next_sym <> '' then - begin - ss := ss + next_sym; - while line[x]<> #39 do - begin - inc(x); - if not IsReadable()then - begin - dec(x); - break; - end; - ss := ss + line[x]; - end; - inc(x); - end; - break; - end - else - begin - while not NO_NAME_SYMS.Contains(line[x])do - begin - ss := ss + line[x]; - inc(x); - if not IsReadable()then - break; - end; - break; - end; + ss:='('; + inc(x); + break; end; end else begin - while last_i0 <> FLineIx do + if SYMS1.Contains(now_sym)then begin - ss := ss + #10; - inc(last_i0); - end; - ss := ss + now_sym; - if now_sym = ml then - if ml = '}' then - begin + ss := now_sym; + inc(x); + if SYMS2.IndexOf(now_sym + next_sym)<>-1 then begin inc(x); - break; - end - else if(x <> 0)and(line[x - 1]= '*')then - begin - inc(x); - break; + ss := ss + next_sym; end; + break; + end + else if now_sym = #39 then + begin + ss := #39; + inc(x); + if next_sym <> '' then + begin + ss := ss + next_sym; + while line[x]<> #39 do + begin + inc(x); + if not IsReadable()then + begin + dec(x); + break; + end; + ss := ss + line[x]; + end; + inc(x); + end; + break; + end + else + begin + while not NO_NAME_SYMS.Contains(line[x])do + begin + ss:=ss+line[x]; + inc(x); + if not IsReadable()then + break; + end; + break; + end; end; - NextReadable(); - end; - Result.Make(ss, begin_pos, GetPos, ended); - SkipSpaces; - end; - - constructor TAGPasTokenizer.Create(input: TStrings); - begin - FStrings := input; - FLineIx := 0; - x := 1 + fix; - ended := False; - SkipSpaces; - end; - - { TAGPasTokenizerStack } - - destructor TAGPasTokenizerStack.Destroy; - begin - FreeAndNil(Stack); - FreeAndNil(Tokenizer); - inherited; - end; - - function TAGPasTokenizerStack.GetCachedCount: integer; - begin - Result := Stack.Count - end; - - function TAGPasTokenizerStack.GetLast(): TAGToken; - begin - if Stack.Count <> 0 then - Result := Stack.Peek + end else begin - Result := Get(Tokenizer); - Stack.Enqueue(Result); - end; - end; - - function TAGPasTokenizerStack.GetWithComments( - Tokenizer: TAGPasTokenizer): TAGToken; - begin - Result := Tokenizer.GetNext; - end; - - function TAGPasTokenizerStack.GetWithoutComments( - Tokenizer: TAGPasTokenizer): TAGToken; - var done: boolean; - begin - repeat - Result := Tokenizer.GetNext; - until Result.ended or not IsComment(Result.Text); - IsEnd := Result.ended; - end; - - constructor TAGPasTokenizerStack.Create(input: TStrings; - GetComments: boolean = True); - begin - Stack := TQueue.Create(); - Tokenizer := TAGPasTokenizer.Create(input); - - if GetComments then - Get := GetWithComments - else - Get := GetWithoutComments; - end; - - procedure TAGPasTokenizerStack.Push(const t: TAGToken); - begin - Stack.Enqueue(t); - end; - - function TAGPasTokenizerStack.Pop(): TAGToken; - begin - if Stack.Count > 0 then - Result := Stack.Dequeue - else - Result := Get(Tokenizer); - IsEnd := Result.ended; - end; - - { TAGPasTokenizerParallelStack } - - function TAGPasTokenizerParallelStack.AddTokenToStack(): boolean; - var tkn: TAGToken; - begin - FStackLock.Enter; - tkn := Get(Tokenizer); - Result := tkn.ended; - Stack.Enqueue(tkn); - FStackLock.Leave; - end; - - constructor TAGPasTokenizerParallelStack.Create(const input: TStrings; - GetComments: - boolean = True;stackMax: integer = 1000); - begin - inherited Create(input, GetComments); - FStackHalfMax := stackMax shr 1; - FStackLock := TCriticalSection.Create(); - FSignal := TEvent.Create(nil, False, True, 'ag_tkn_work_signal'); - FWorker := TWorkerThread.Create(Self); - end; - - destructor TAGPasTokenizerParallelStack.Destroy; - begin - EnsureThreadDone(); - FreeAndNil(FStackLock); - inherited; - end; - - procedure TAGPasTokenizerParallelStack.EnsureThreadDone; - begin - if not FWorker.Terminated then begin - FWorker.Terminate; - FSignal.SetEvent();// stop idling - FWorker.WaitFor(); - end; - FreeAndNil(FWorker); - FreeAndNil(FSignal); - end; - - function TAGPasTokenizerParallelStack.GetLast: TAGToken; - begin - FStackLock.Enter; - try - Result := inherited GetLast(); - finally - FStackLock.Leave; - end; - end; - - function TAGPasTokenizerParallelStack.Pop: TAGToken; - var - doReplentishStack, wasRead: boolean; - begin - - FStackLock.Enter; - - wasRead := Stack.Count > 0; - if wasRead then - Result := Stack.Dequeue - else - Result := Get(Tokenizer); - - doReplentishStack := FWorker.Idling and(Stack.Count < FStackHalfMax); - FStackLock.Leave; - - IsEnd := Result.ended; - if doReplentishStack then - FSignal.SetEvent; - - end; - - procedure TAGPasTokenizerParallelStack.Push(const t: TAGToken); - begin - FStackLock.Enter; - try - inherited Push(t); - finally - FStackLock.Leave; - end; - end; - - { TAGPasTokenizerParallelStack.TWorkerThread } - - constructor TAGPasTokenizerParallelStack.TWorkerThread.Create(const Stack: - TAGPasTokenizerParallelStack); - begin - FStack := Stack; - inherited Create(False, 4 * 4096); - end; - - procedure TAGPasTokenizerParallelStack.TWorkerThread.Execute; - var Count: integer; - isDone: boolean; - max: integer; - begin - Count := FStack.GetCachedCount; - max := FStack.FStackHalfMax * 2; - repeat - isDone := FStack.AddTokenToStack(); - inc(Count); - while not Terminated and(Count >= max)do begin - Count := FStack.GetCachedCount();// fetch real count - if Count < max then - break; - Idling := True; - FStack.FSignal.WaitFor(1000); - Count := FStack.GetCachedCount(); + while last_i0<>FLineIx do + begin + ss:=ss+#10; + inc(last_i0); end; - Idling := False; - until Terminated or isDone or FStack.ended; + ss:=ss+now_sym; + if now_sym=ml then + if ml='}' then + begin + inc(x); + break; + end + else if(x<>0)and(line[x - 1]='*')then + begin + inc(x); + break; + end; + end; + NextReadable(); end; + Result:=TAGToken.Create(ss,begin_pos,GetPos,ended); + SkipSpaces(); +end; +{$WARN USE_BEFORE_DEF ON} + +{TAGPasTokenizerStack} + +function TAGPasTokenizerStack.GetLast(): TAGToken; +begin +if Stack.Count<>0 then + Result:=Stack.Peek +else +begin + Result:=Get(Tokenizer); + Stack.Enqueue(Result); +end; +end; + +function TAGPasTokenizerStack.GetWithComments(Tokenizer:TAGPasTokenizer):TAGToken; +begin +Result:=Tokenizer.GetNext; +end; + +function TAGPasTokenizerStack.GetWithoutComments(Tokenizer:TAGPasTokenizer):TAGToken; +begin +repeat + Result:=Tokenizer.GetNext; +until Result.ended or not IsComment(Result.Text); +IsEnd:=Result.ended; +end; + +function TAGPasTokenizerStack.GetCachedCount:integer; +begin +Result:=Stack.Count +end; + +constructor TAGPasTokenizerStack.Create(input:TStrings;GetComments:boolean=True); +begin +Stack:=TQueue.Create(); +Tokenizer:=TAGPasTokenizer.Create(input); +if GetComments then + Get:=GetWithComments +else + Get:=GetWithoutComments; +end; + +procedure TAGPasTokenizerStack.Push(const t: TAGToken); +begin +Stack.Enqueue(t); +end; + +function TAGPasTokenizerStack.Pop():TAGToken; +begin +if Stack.Count>0 then + Result:=Stack.Dequeue +else + Result:=Get(Tokenizer); +IsEnd:=Result.ended; +end; + +destructor TAGPasTokenizerStack.Destroy; +begin +FreeAndNil(Stack); +FreeAndNil(Tokenizer); +inherited; +end; + +{TAGPasTokenizerParallelStack} + +{TAGPasTokenizerParallelStack.TWorkerThread} + +procedure TAGPasTokenizerParallelStack.TWorkerThread.Execute; +var + Count,max:integer; + isDone:boolean; +begin +Count:=FStack.GetCachedCount; +max:=FStack.FStackHalfMax*2; +repeat + isDone:=FStack.AddTokenToStack(); + inc(Count); + while not Terminated and(Count>=max)do + begin + Count:=FStack.GetCachedCount(); + if Count 0 then + Result:=Stack.Dequeue +else + Result:=Get(Tokenizer); + +doReplentishStack:=FWorker.Idling and(Stack.Count='); - SYMS2.Add('<='); - SYMS2.Add('<>'); - SYMS2.Add(':='); - SYMS2.Add('..'); - SYMS2.Add('-='); - SYMS2.Add('+='); - SYMS2.Add('/='); - SYMS2.Add('*='); - SYMS2.Add('**'); - SYMS2.Add('><'); - SYMS2.Add('(.'); - SYMS2.Add('.)'); - SYMS2.Add('<<'); - SYMS2.Add('>>'); - finalization - FreeAndNil(SYMS2); - +SYMS2:=THashedStringList.Create(); +SYMS2.Add('>='); +SYMS2.Add('<='); +SYMS2.Add('<>'); +SYMS2.Add(':='); +SYMS2.Add('..'); +SYMS2.Add('-='); +SYMS2.Add('+='); +SYMS2.Add('/='); +SYMS2.Add('*='); +SYMS2.Add('**'); +SYMS2.Add('><'); +SYMS2.Add('(.'); +SYMS2.Add('.)'); +SYMS2.Add('<<'); +SYMS2.Add('>>'); +finalization +FreeAndNil(SYMS2); end. diff --git a/DOC_EN.MD b/DOC_EN.MD index c983fb0..f30ad02 100644 --- a/DOC_EN.MD +++ b/DOC_EN.MD @@ -1,54 +1,67 @@ -# AGPascalTokenizer - -Tokenizer Library for Delphi/FPC. Based on [PyPascalTokenizer](https://github.com/Artem3213212/PyPascalTokenizer) work by [Artem Gavrilov](https://github.com/Artem3213212). - -## API - -### TToken struct - -TToken is a `record` type of: - -* Text: `string` with token text. -* &begin: `TTokenizerPos` with start position of token. -* &end: `TTokenizerPos` with end position of token. -* ended: `boolean` flag, true if it was the last token. - -Has Create constructor with all values. - -### TTokenizerPos - -`record`, saves position of token if format: - -* x: `integer` platform-depend-base (`NEXTGEN`) character index in line. -* y: `integer` 0-based line index. - -### `Class` PasTokenizer - -Main class of tokenizer. - -* s: `TStrings` private, use constructor. List of source code strings. -* get_next(): function, returns `TToken`. Get next token and change end pos -* read_next(): function, not implemented, get next token, but don't change end pos. -* is_ended(): function, not implemented, check if text ended. - -Has constructor Create with `s` param. - -### `Class` PasTokenizerStack - -Not implemented. - -### `Class` PasTokenizerParallelStack - -Not implemented. - -## Utils - -Helper functions to analyze token text. - -* is_name(s: `string`): `boolean` Check for valid identifier (can be reserved word too). -* is_comment(s: `string`): `boolean` Check for valid comment. -* is_string(s: `string`): `boolean` Check for string constant. - -## Author - -[Artem Gavrilov](https://github.com/Artem3213212) and [Vladislav Utkin](https://github.com/viad00). +# AGPascalTokenizer + +Tokenizer Library for Delphi/FPC. Based on [PyPascalTokenizer](https://github.com/Artem3213212/PyPascalTokenizer) work by [Artem Gavrilov](https://github.com/Artem3213212). + +## API + +### TAGToken struct + +TAGToken is a `record` type of: + +* Text: `string` with token text. +* &begin: `TAGTokenizerPos` with start position of token. +* &end: `TAGTokenizerPos` with end position of token. +* ended: `boolean` flag, true if it was the last token. + +Has Create constructor with all values. + +### TAGTokenizerPos + +`record`, saves position of token if format: + +* x: `integer` platform-depend-base (`NEXTGEN`) character index in line. +* y: `integer` 0-based line index. + +### `Class` TAGPasTokenizer + +Main class of tokenizer. + +* get_next(): `TAGToken` function, returns. Get next token and change pos +* pos: `TAGTokenizerPos` property, containing position whear tokenizer was stoped after last get_next(), writeble. +* is_ended: `boolean` property, check if text ended. + +Has constructor Create with `input`: `TStrings` param. + +### `Class` TAGPasTokenizerStack + +Stack of tokens automaticaly filled from input. + +* Push(t: `TAGToken`) push token to stack. +* Pop(): `TAGToken` pop token from stack. +* Last: `TAGToken` property, seek stask head. +* ended: `boolean` property, check if text ended. + +Has constructor Create with input: `TStrings` and GetComments: `boolean=True` param. + +### `Class` TAGPasTokenizerParallelStack + +Stack of tokens automaticaly filled from input(in second tread). + +* Push(t: `TAGToken`) push token to stack. +* Pop(): `TAGToken` pop token from stack. +* Last: `TAGToken` property, seek stask head. +* ended: `boolean` property, check if text ended. + +Has constructor Create with input: `TStrings`, GetComments: `boolean=True` and stackMax:`integer=1000` param. + +## Utils + +Helper functions to analyze token text. + +* IsName(s: `string`): `boolean` Check for valid identifier (can be reserved word too). +* IsComment(s: `string`): `boolean` Check for valid comment. +* IsString(s: `string`): `boolean` Check for string constant. + +## Author + +[Artem Gavrilov](https://github.com/Artem3213212) and [Vladislav Utkin](https://github.com/viad00). diff --git a/DOC_RU.MD b/DOC_RU.MD index 2a2af0f..16b711b 100644 --- a/DOC_RU.MD +++ b/DOC_RU.MD @@ -1,54 +1,67 @@ -# AGPascalTokenizer - -Библиотека токенайзер для Delphi/FPC. Основана на работе [PyPascalTokenizer](https://github.com/Artem3213212/PyPascalTokenizer) [Артёма Гаврилова](https://github.com/Artem3213212). - -## API - -### TToken - -TToken это тип `record` из: - -* Text: `string` с текстом токена. -* &begin: `TTokenizerPos` со стартовой позицией токена. -* &end: `TTokenizerPos` с конечной позицией токена. -* ended: `boolean` Истина, если это был постедний токен. - -Имеет конструктор Create с перечисленными переменными. - -### TTokenizerPos - -тип `record`, сохраняет позицию в формате: - -* x: `integer` начало зависит от платформы (`NEXTGEN`), индекс символа в строке. -* y: `integer` начало с 0, номер строки. - -### `Class` PasTokenizer - -Главный класс токенайзера. - -* s: `TStrings` приватная, используйте конструктор. Список строк исходного кода. -* get_next(): функция, возвращает `TToken`. Возвращает следующий токен и изменяет свою позицию. -* read_next(): функция, не имплементированно, Возвращает следующий токен, не изменяя свою позицию. -* is_ended(): функция, не имплементированно, проверка что текст закончился. - -Имеет конструктор Create с параметром `s`. - -### `Class` PasTokenizerStack - -Не имплементированно. - -### `Class` PasTokenizerParallelStack - -Не имплементированно. - -## Утилиты - -Вспомогательные функции для анализа текста. - -* is_name(s: `string`): `boolean` Проверка что идентификатор правильный. -* is_comment(s: `string`): `boolean` Проверка что комментарий правильный. -* is_string(s: `string`): `boolean` Проверка что это строковое значение. - -## Авторы - -[Артём Гаврилов](https://github.com/Artem3213212) и [Владислав Уткин](https://github.com/viad00). +# AGPascalTokenizer + +Библиотека токенайзер для Delphi/FPC. Основана на работе [PyPascalTokenizer](https://github.com/Artem3213212/PyPascalTokenizer) [Артёма Гаврилова](https://github.com/Artem3213212). + +## API + +### TAGToken + +TAGToken это тип `record` из: + +* Text: `string` с текстом токена. +* &begin: `TAGTokenizerPos` со стартовой позицией токена. +* &end: `TAGTokenizerPos` с конечной позицией токена. +* ended: `boolean` Истина, если это был постедний токен. + +Имеет конструктор Create с перечисленными переменными. + +### TAGTokenizerPos + +тип `record`, сохраняет позицию в формате: + +* x: `integer` начало зависит от платформы (`NEXTGEN`), индекс символа в строке. +* y: `integer` начало с 0, номер строки. + +### `Class` TAGPasTokenizer + +Главный класс токенайзера. + +* get_next(): функция, возвращает `TAGToken`. Возвращает следующий токен и изменяет позицию. +* pos: `TAGTokenizerPos` свойство, содержит позицию где токенайзер . +* is_ended: `boolean` свойство, проверка что текст закончился. + +Имеет конструктор Create с параметром `input`: `TStrings`. + +### `Class` TAGPasTokenizerStack + +Стек из токенов автоматически заполняется из input'а. + +* Push(t: `TAGToken`) кладёт токен в стек. +* Pop(): `TAGToken` достать токен из стека. +* Last: `TAGToken` свойство, выдающееся вершина стека. +* is_ended: `boolean` свойство, проверка конца текста. + +Имеет конструктор Create с параметрами `input`: `TStrings` и GetComments: `boolean=True`. + +### `Class` TAGPasTokenizerParallelStack + +Стек из токенов автоматически заполняется из input'а(во втором потоке). + +* Push(t: `TAGToken`) кладёт токен в стек. +* Pop(): `TAGToken` достать токен из стека. +* Last: `TAGToken` свойство, выдающееся вершина стека. +* ended: `boolean` свойство, проверка конца текста. + +Имеет конструктор Create с параметрами input: `TStrings`, GetComments: `boolean=True` и stackMax:`integer=1000`. + +## Утилиты + +Вспомогательные функции для анализа текста. + +* IsName(s: `string`): `boolean` Проверка что идентификатор правильный. +* IsComment(s: `string`): `boolean` Проверка что комментарий правильный. +* IsString(s: `string`): `boolean` Проверка что это строковое значение. + +## Авторы + +[Артём Гаврилов](https://github.com/Artem3213212) и [Владислав Уткин](https://github.com/viad00). diff --git a/Demo/Demo.dpr b/Demo/Demo.dpr index 191fb10..7a9f7fc 100644 --- a/Demo/Demo.dpr +++ b/Demo/Demo.dpr @@ -11,7 +11,7 @@ program Demo; uses {$IF not defined(FPC)} - FastMM4, + //FastMM4, {$endif} SysUtils, Classes, diff --git a/Demo/Demo.dproj b/Demo/Demo.dproj index 42b58fb..2329286 100644 --- a/Demo/Demo.dproj +++ b/Demo/Demo.dproj @@ -13,11 +13,6 @@ true - - true - Base - true - true Base @@ -55,19 +50,6 @@ System;Xml;Data;Datasnap;Web;Soap;$(DCC_Namespace) Demo - - DBXSqliteDriver;RESTComponents;DBXInterBaseDriver;emsclientfiredac;DataSnapFireDAC;tethering;bindcompfmx;FmxTeeUI;FireDACIBDriver;fmx;FireDACDBXDriver;dbexpress;IndyCore;dsnap;DataSnapCommon;emsclient;FireDACCommon;RESTBackendComponents;soapserver;bindengine;CloudService;FireDACCommonDriver;DataSnapClient;inet;bindcompdbx;IndyIPCommon;IndyIPServer;IndySystem;fmxFireDAC;FireDAC;FireDACSqliteDriver;FMXTee;soaprtl;DbxCommonDriver;xmlrtl;soapmidas;DataSnapNativeClient;FireDACDSDriver;rtl;DbxClientDriver;CustomIPTransport;bindcomp;IndyIPClient;dbxcds;dsnapxml;DataSnapProviderClient;dbrtl;IndyProtocols;$(DCC_UsePackage) - $(BDS)\bin\Artwork\Android\FM_LauncherIcon_36x36.png - $(BDS)\bin\Artwork\Android\FM_LauncherIcon_48x48.png - $(BDS)\bin\Artwork\Android\FM_LauncherIcon_72x72.png - $(BDS)\bin\Artwork\Android\FM_LauncherIcon_96x96.png - $(BDS)\bin\Artwork\Android\FM_LauncherIcon_144x144.png - $(BDS)\bin\Artwork\Android\FM_SplashImage_426x320.png - $(BDS)\bin\Artwork\Android\FM_SplashImage_470x320.png - $(BDS)\bin\Artwork\Android\FM_SplashImage_640x480.png - $(BDS)\bin\Artwork\Android\FM_SplashImage_960x720.png - android-support-v4.dex.jar;cloud-messaging.dex.jar;fmx.dex.jar;google-analytics-v2.dex.jar;google-play-billing.dex.jar;google-play-licensing.dex.jar;google-play-services-ads-7.0.0.dex.jar;google-play-services-analytics-7.0.0.dex.jar;google-play-services-base-7.0.0.dex.jar;google-play-services-gcm-7.0.0.dex.jar;google-play-services-identity-7.0.0.dex.jar;google-play-services-maps-7.0.0.dex.jar;google-play-services-panorama-7.0.0.dex.jar;google-play-services-plus-7.0.0.dex.jar;google-play-services-wallet-7.0.0.dex.jar - DBXSqliteDriver;RESTComponents;DataSnapServerMidas;DBXDb2Driver;DBXInterBaseDriver;vclactnband;vclFireDAC;emsclientfiredac;DataSnapFireDAC;svnui;tethering;JvGlobus;FireDACADSDriver;JvPluginSystem;DBXMSSQLDriver;JvMM;DatasnapConnectorsFreePascal;FireDACMSSQLDriver;vcltouch;JvBands;vcldb;bindcompfmx;svn;JvJans;DBXOracleDriver;JvNet;inetdb;JvAppFrm;VirtualTreesDR;FmxTeeUI;emsedge;JvDotNetCtrls;FireDACIBDriver;fmx;fmxdae;JvWizards;FireDACDBXDriver;dbexpress;IndyCore;vclx;JvPageComps;dsnap;DataSnapCommon;emsclient;FireDACCommon;JvDB;RESTBackendComponents;DataSnapConnectors;VCLRESTComponents;soapserver;JclDeveloperTools;vclie;bindengine;DBXMySQLDriver;FireDACOracleDriver;CloudService;FireDACMySQLDriver;DBXFirebirdDriver;JvCmp;JvHMI;FireDACCommonODBC;FireDACCommonDriver;DataSnapClient;inet;OverbyteIcsD103Run;bindcompdbx;IndyIPCommon;JvCustom;vcl;DBXSybaseASEDriver;IndyIPServer;JvXPCtrls;IndySystem;FireDACDb2Driver;dsnapcon;FireDACMSAccDriver;fmxFireDAC;FireDACInfxDriver;vclimg;TeeDB;FireDAC;GrtPanelPackage;Jcl;JvCore;emshosting;JvCrypt;FireDACSqliteDriver;FireDACPgDriver;FireDACASADriver;DBXOdbcDriver;FireDACTDataDriver;FMXTee;soaprtl;DbxCommonDriver;JvDlgs;JvRuntimeDesign;JvManagedThreads;Tee;DataSnapServer;xmlrtl;soapmidas;DataSnapNativeClient;fmxobj;vclwinx;FireDACDSDriver;rtl;emsserverresource;DbxClientDriver;JvTimeFramework;DBXSybaseASADriver;CustomIPTransport;vcldsnap;JvSystem;JvStdCtrls;FrameViewerXE9;bindcomp;appanalytics;DBXInformixDriver;IndyIPClient;bindcompvcl;SynEdit_R;TeeUI;JvDocking;dbxcds;VclSmp;JvPascalInterpreter;adortl;FireDACODBCDriver;JclVcl;DataSnapIndy10ServerTransport;dsnapxml;DataSnapProviderClient;dbrtl;inetdbxpress;FireDACMongoDBDriver;IndyProtocols;JvControls;JvPrintPreview;JclContainers;fmxase;$(DCC_UsePackage) Winapi;System.Win;Data.Win;Datasnap.Win;Web.Win;Soap.Win;Xml.Win;Bde;$(DCC_Namespace) @@ -139,13 +121,24 @@ + + + true + + + + + Demo.exe + true + + true - - + + true @@ -164,12 +157,6 @@ true - - - Demo.exe - true - - 1 @@ -179,6 +166,7 @@ 1 + Contents\MacOS 0 @@ -314,11 +302,6 @@ 1 .framework - - Contents\MacOS - 1 - .framework - 0 @@ -341,11 +324,6 @@ 1 .dylib - - Contents\MacOS - 1 - .dylib - 0 .dll;.bpl @@ -369,11 +347,6 @@ 1 .dylib - - Contents\MacOS - 1 - .dylib - 0 .bpl @@ -396,10 +369,6 @@ Contents\Resources\StartUp\ 0 - - Contents\Resources\StartUp\ - 0 - 0 @@ -547,30 +516,18 @@ ..\ 1 - - ..\ - 1 - Contents 1 - - Contents - 1 - Contents\Resources 1 - - Contents\Resources - 1 - @@ -593,10 +550,6 @@ Contents\MacOS 1 - - Contents\MacOS - 1 - 0 @@ -640,7 +593,6 @@ - False True False