Add Multi threading
This commit is contained in:
@@ -8,7 +8,6 @@ interface
|
||||
uses
|
||||
SysUtils,Classes,IniFiles,SyncObjs,Generics.Collections;
|
||||
|
||||
|
||||
type
|
||||
TAGTokenizerPos=record
|
||||
x,y:integer;
|
||||
@@ -18,33 +17,31 @@ interface
|
||||
Text:string;
|
||||
&begin,&end:TAGTokenizerPos;
|
||||
ended:boolean;
|
||||
procedure Make(const Text:string;&begin, &end: TAGTokenizerPos;ended: boolean);
|
||||
constructor Create(const Text:string;&begin,&end:TAGTokenizerPos;ended:boolean);
|
||||
end;
|
||||
|
||||
TAGPasTokenizer=class
|
||||
strict protected
|
||||
FStrings:TStrings;
|
||||
FLineIx: integer;
|
||||
x: integer;
|
||||
function DoReadable(): boolean;
|
||||
FLineIx,x:integer;
|
||||
ended:boolean;
|
||||
procedure DoReadable();
|
||||
function IsReadable():boolean;
|
||||
function NextReadable(): boolean;
|
||||
procedure NextReadable();
|
||||
procedure SkipSpaces();
|
||||
function GetPos():TAGTokenizerPos;
|
||||
procedure SetPos(pos:TAGTokenizerPos);
|
||||
public
|
||||
ended: boolean;
|
||||
function GetNext(): TAGToken;
|
||||
// procedure read_next();
|
||||
constructor Create(input:TStrings);
|
||||
function GetNext():TAGToken;
|
||||
property pos:TAGTokenizerPos read GetPos write SetPos;
|
||||
property is_ended:boolean read ended;
|
||||
end;
|
||||
|
||||
TAGPasTokenizerStack = class
|
||||
strict protected
|
||||
type
|
||||
GetCall=function(Tokenizer:TAGPasTokenizer):TAGToken of object;
|
||||
|
||||
var
|
||||
Stack:TQueue<TAGToken>;
|
||||
Tokenizer:TAGPasTokenizer;
|
||||
@@ -53,13 +50,13 @@ interface
|
||||
function GetLast():TAGToken;virtual;
|
||||
function GetWithComments(Tokenizer:TAGPasTokenizer):TAGToken;
|
||||
function GetWithoutComments(Tokenizer:TAGPasTokenizer):TAGToken;
|
||||
destructor Destroy;override;
|
||||
protected
|
||||
function GetCachedCount:integer;inline;
|
||||
public
|
||||
constructor Create(input:TStrings;GetComments:boolean=True);
|
||||
procedure Push(const t:TAGToken);virtual;
|
||||
function Pop():TAGToken;virtual;
|
||||
destructor Destroy;override;
|
||||
property Last:TAGToken read GetLast write Push;
|
||||
property ended:boolean read IsEnd;
|
||||
end;
|
||||
@@ -75,22 +72,20 @@ interface
|
||||
Idling:boolean;
|
||||
constructor Create(const Stack:TAGPasTokenizerParallelStack);
|
||||
end;
|
||||
|
||||
var
|
||||
FWorker:TWorkerThread;
|
||||
FStackLock:TCriticalSection;
|
||||
function AddTokenToStack:boolean;
|
||||
function GetLast():TAGToken;override;
|
||||
procedure EnsureThreadDone();
|
||||
destructor Destroy;override;
|
||||
protected
|
||||
FStackHalfMax:integer;
|
||||
FSignal:TEvent;
|
||||
procedure Push(const t: TAGToken);override;
|
||||
public
|
||||
constructor Create(const input: TStrings;GetComments: boolean = True;
|
||||
stackMax: integer = 1000);
|
||||
constructor Create(const input:TStrings;GetComments:boolean=True;stackMax:integer=1000);
|
||||
procedure Push(const t:TAGToken);override;
|
||||
function Pop():TAGToken;override;
|
||||
destructor Destroy;override;
|
||||
end;
|
||||
|
||||
function IsComment(s:string):boolean;
|
||||
@@ -99,15 +94,6 @@ interface
|
||||
|
||||
implementation
|
||||
|
||||
procedure TAGToken.Make(const Text:string;&begin, &end: TAGTokenizerPos;ended:
|
||||
boolean);
|
||||
begin
|
||||
Self.Text := Text;
|
||||
Self.&begin := &begin;
|
||||
Self.&end := &end;
|
||||
Self.ended := ended;
|
||||
end;
|
||||
|
||||
const
|
||||
SYMS1='()[]/|\@#=><:;,.$+-*^';
|
||||
SPACES=#12#10#13#9#11' ';
|
||||
@@ -121,24 +107,18 @@ implementation
|
||||
|
||||
function IsComment(s:string):boolean;
|
||||
begin
|
||||
Result :=(s.startswith('{')or s.startswith('(*')or s.startswith('//'));
|
||||
Result:=s.startswith('{')or s.startswith('(*')or s.startswith('//');
|
||||
end;
|
||||
|
||||
function IsName(s:string):boolean;
|
||||
var
|
||||
i:integer;
|
||||
begin
|
||||
if length(s)<= 0 then
|
||||
if(length(s)=0)or(s='&')or(not CHARS_ID0.Contains(s[1+fix]))then
|
||||
Exit(False);
|
||||
if s = '&' then
|
||||
Exit(False);
|
||||
if not CHARS_ID0.Contains(s[1 + fix])then
|
||||
Exit(False);
|
||||
for i := 1 to length(s)do
|
||||
begin
|
||||
for i:=2+fix to length(s)+fix do
|
||||
if not CHARS_ID.Contains(s[i])then
|
||||
Exit(False);
|
||||
end;
|
||||
Result:=True;
|
||||
end;
|
||||
|
||||
@@ -147,11 +127,22 @@ implementation
|
||||
Result:=s.startswith(#39);
|
||||
end;
|
||||
|
||||
function TAGPasTokenizer.DoReadable(): boolean;
|
||||
{TAGToken}
|
||||
|
||||
constructor TAGToken.Create(const Text:string;&begin,&end:TAGTokenizerPos;ended:boolean);
|
||||
begin
|
||||
Self.Text:=Text;
|
||||
Self.&begin:=&begin;
|
||||
Self.&end:=&end;
|
||||
Self.ended:=ended;
|
||||
end;
|
||||
|
||||
{TAGPasTokenizer}
|
||||
|
||||
procedure TAGPasTokenizer.DoReadable();
|
||||
begin
|
||||
if not IsReadable()then
|
||||
begin
|
||||
if(FLineIx + 1 = FStrings.Count)then
|
||||
if FLineIx+1=FStrings.Count then
|
||||
ended:=True
|
||||
else
|
||||
begin
|
||||
@@ -167,10 +158,6 @@ implementation
|
||||
inc(FLineIx);
|
||||
end;
|
||||
end;
|
||||
Exit(True);
|
||||
end
|
||||
else
|
||||
Exit(False);
|
||||
end;
|
||||
|
||||
function TAGPasTokenizer.IsReadable():boolean;
|
||||
@@ -178,21 +165,19 @@ implementation
|
||||
Result:=x<=length(FStrings[FLineIx])+fix;
|
||||
end;
|
||||
|
||||
function TAGPasTokenizer.NextReadable(): boolean;
|
||||
procedure TAGPasTokenizer.NextReadable();
|
||||
begin
|
||||
inc(x);
|
||||
Result := DoReadable();
|
||||
DoReadable();
|
||||
end;
|
||||
|
||||
procedure TAGPasTokenizer.SkipSpaces();
|
||||
begin
|
||||
DoReadable();
|
||||
if not ended then
|
||||
begin
|
||||
while SPACES.Contains(FStrings[FLineIx][x])do
|
||||
NextReadable();
|
||||
end;
|
||||
end;
|
||||
|
||||
function TAGPasTokenizer.GetPos():TAGTokenizerPos;
|
||||
begin
|
||||
@@ -208,6 +193,16 @@ implementation
|
||||
DoReadable();
|
||||
end;
|
||||
|
||||
constructor TAGPasTokenizer.Create(input: TStrings);
|
||||
begin
|
||||
FStrings:=input;
|
||||
FLineIx:=0;
|
||||
x:=1+fix;
|
||||
ended:=False;
|
||||
SkipSpaces;
|
||||
end;
|
||||
|
||||
{$WARN USE_BEFORE_DEF OFF}
|
||||
function TAGPasTokenizer.GetNext():TAGToken;
|
||||
var
|
||||
l,last_i0:integer;
|
||||
@@ -233,13 +228,11 @@ implementation
|
||||
if ml='' then
|
||||
begin
|
||||
if now_sym='/' then
|
||||
begin
|
||||
if next_sym='/' then
|
||||
begin
|
||||
ss:=Copy(line,x,l);
|
||||
x:=l+1+fix;
|
||||
break;
|
||||
end;
|
||||
end
|
||||
else if now_sym='{' then
|
||||
begin
|
||||
@@ -331,33 +324,13 @@ implementation
|
||||
end;
|
||||
NextReadable();
|
||||
end;
|
||||
Result.Make(ss, begin_pos, GetPos, ended);
|
||||
SkipSpaces;
|
||||
end;
|
||||
|
||||
constructor TAGPasTokenizer.Create(input: TStrings);
|
||||
begin
|
||||
FStrings := input;
|
||||
FLineIx := 0;
|
||||
x := 1 + fix;
|
||||
ended := False;
|
||||
SkipSpaces;
|
||||
Result:=TAGToken.Create(ss,begin_pos,GetPos,ended);
|
||||
SkipSpaces();
|
||||
end;
|
||||
{$WARN USE_BEFORE_DEF ON}
|
||||
|
||||
{TAGPasTokenizerStack}
|
||||
|
||||
destructor TAGPasTokenizerStack.Destroy;
|
||||
begin
|
||||
FreeAndNil(Stack);
|
||||
FreeAndNil(Tokenizer);
|
||||
inherited;
|
||||
end;
|
||||
|
||||
function TAGPasTokenizerStack.GetCachedCount: integer;
|
||||
begin
|
||||
Result := Stack.Count
|
||||
end;
|
||||
|
||||
function TAGPasTokenizerStack.GetLast(): TAGToken;
|
||||
begin
|
||||
if Stack.Count<>0 then
|
||||
@@ -369,15 +342,12 @@ implementation
|
||||
end;
|
||||
end;
|
||||
|
||||
function TAGPasTokenizerStack.GetWithComments(
|
||||
Tokenizer: TAGPasTokenizer): TAGToken;
|
||||
function TAGPasTokenizerStack.GetWithComments(Tokenizer:TAGPasTokenizer):TAGToken;
|
||||
begin
|
||||
Result:=Tokenizer.GetNext;
|
||||
end;
|
||||
|
||||
function TAGPasTokenizerStack.GetWithoutComments(
|
||||
Tokenizer: TAGPasTokenizer): TAGToken;
|
||||
var done: boolean;
|
||||
function TAGPasTokenizerStack.GetWithoutComments(Tokenizer:TAGPasTokenizer):TAGToken;
|
||||
begin
|
||||
repeat
|
||||
Result:=Tokenizer.GetNext;
|
||||
@@ -385,12 +355,15 @@ implementation
|
||||
IsEnd:=Result.ended;
|
||||
end;
|
||||
|
||||
constructor TAGPasTokenizerStack.Create(input: TStrings;
|
||||
GetComments: boolean = True);
|
||||
function TAGPasTokenizerStack.GetCachedCount:integer;
|
||||
begin
|
||||
Result:=Stack.Count
|
||||
end;
|
||||
|
||||
constructor TAGPasTokenizerStack.Create(input:TStrings;GetComments:boolean=True);
|
||||
begin
|
||||
Stack:=TQueue<TAGToken>.Create();
|
||||
Tokenizer:=TAGPasTokenizer.Create(input);
|
||||
|
||||
if GetComments then
|
||||
Get:=GetWithComments
|
||||
else
|
||||
@@ -411,10 +384,49 @@ implementation
|
||||
IsEnd:=Result.ended;
|
||||
end;
|
||||
|
||||
destructor TAGPasTokenizerStack.Destroy;
|
||||
begin
|
||||
FreeAndNil(Stack);
|
||||
FreeAndNil(Tokenizer);
|
||||
inherited;
|
||||
end;
|
||||
|
||||
{TAGPasTokenizerParallelStack}
|
||||
|
||||
{TAGPasTokenizerParallelStack.TWorkerThread}
|
||||
|
||||
procedure TAGPasTokenizerParallelStack.TWorkerThread.Execute;
|
||||
var
|
||||
Count,max:integer;
|
||||
isDone:boolean;
|
||||
begin
|
||||
Count:=FStack.GetCachedCount;
|
||||
max:=FStack.FStackHalfMax*2;
|
||||
repeat
|
||||
isDone:=FStack.AddTokenToStack();
|
||||
inc(Count);
|
||||
while not Terminated and(Count>=max)do
|
||||
begin
|
||||
Count:=FStack.GetCachedCount();
|
||||
if Count<max then
|
||||
break;
|
||||
Idling:=True;
|
||||
FStack.FSignal.WaitFor(1000);
|
||||
Count:=FStack.GetCachedCount();
|
||||
end;
|
||||
Idling:=False;
|
||||
until Terminated or isDone or FStack.ended;
|
||||
end;
|
||||
|
||||
constructor TAGPasTokenizerParallelStack.TWorkerThread.Create(const Stack:TAGPasTokenizerParallelStack);
|
||||
begin
|
||||
FStack:=Stack;
|
||||
inherited Create(False{,4*4096});
|
||||
end;
|
||||
|
||||
function TAGPasTokenizerParallelStack.AddTokenToStack():boolean;
|
||||
var tkn: TAGToken;
|
||||
var
|
||||
tkn:TAGToken;
|
||||
begin
|
||||
FStackLock.Enter;
|
||||
tkn:=Get(Tokenizer);
|
||||
@@ -423,35 +435,6 @@ implementation
|
||||
FStackLock.Leave;
|
||||
end;
|
||||
|
||||
constructor TAGPasTokenizerParallelStack.Create(const input: TStrings;
|
||||
GetComments:
|
||||
boolean = True;stackMax: integer = 1000);
|
||||
begin
|
||||
inherited Create(input, GetComments);
|
||||
FStackHalfMax := stackMax shr 1;
|
||||
FStackLock := TCriticalSection.Create();
|
||||
FSignal := TEvent.Create(nil, False, True, 'ag_tkn_work_signal');
|
||||
FWorker := TWorkerThread.Create(Self);
|
||||
end;
|
||||
|
||||
destructor TAGPasTokenizerParallelStack.Destroy;
|
||||
begin
|
||||
EnsureThreadDone();
|
||||
FreeAndNil(FStackLock);
|
||||
inherited;
|
||||
end;
|
||||
|
||||
procedure TAGPasTokenizerParallelStack.EnsureThreadDone;
|
||||
begin
|
||||
if not FWorker.Terminated then begin
|
||||
FWorker.Terminate;
|
||||
FSignal.SetEvent();// stop idling
|
||||
FWorker.WaitFor();
|
||||
end;
|
||||
FreeAndNil(FWorker);
|
||||
FreeAndNil(FSignal);
|
||||
end;
|
||||
|
||||
function TAGPasTokenizerParallelStack.GetLast:TAGToken;
|
||||
begin
|
||||
FStackLock.Enter;
|
||||
@@ -462,26 +445,25 @@ implementation
|
||||
end;
|
||||
end;
|
||||
|
||||
function TAGPasTokenizerParallelStack.Pop: TAGToken;
|
||||
var
|
||||
doReplentishStack, wasRead: boolean;
|
||||
procedure TAGPasTokenizerParallelStack.EnsureThreadDone;
|
||||
begin
|
||||
if not FWorker.Terminated then
|
||||
begin
|
||||
FWorker.Terminate;
|
||||
FSignal.SetEvent();
|
||||
FWorker.WaitFor();
|
||||
end;
|
||||
FreeAndNil(FWorker);
|
||||
FreeAndNil(FSignal);
|
||||
end;
|
||||
|
||||
FStackLock.Enter;
|
||||
|
||||
wasRead := Stack.Count > 0;
|
||||
if wasRead then
|
||||
Result := Stack.Dequeue
|
||||
else
|
||||
Result := Get(Tokenizer);
|
||||
|
||||
doReplentishStack := FWorker.Idling and(Stack.Count < FStackHalfMax);
|
||||
FStackLock.Leave;
|
||||
|
||||
IsEnd := Result.ended;
|
||||
if doReplentishStack then
|
||||
FSignal.SetEvent;
|
||||
|
||||
constructor TAGPasTokenizerParallelStack.Create(const input:TStrings;GetComments:boolean=True;stackMax:integer=1000);
|
||||
begin
|
||||
inherited Create(input,GetComments);
|
||||
FStackHalfMax:=stackMax shr 1;
|
||||
FStackLock:=TCriticalSection.Create();
|
||||
FSignal:=TEvent.Create(nil,False,True,'ag_tkn_work_signal');
|
||||
FWorker:=TWorkerThread.Create(Self);
|
||||
end;
|
||||
|
||||
procedure TAGPasTokenizerParallelStack.Push(const t:TAGToken);
|
||||
@@ -494,39 +476,33 @@ implementation
|
||||
end;
|
||||
end;
|
||||
|
||||
{ TAGPasTokenizerParallelStack.TWorkerThread }
|
||||
|
||||
constructor TAGPasTokenizerParallelStack.TWorkerThread.Create(const Stack:
|
||||
TAGPasTokenizerParallelStack);
|
||||
function TAGPasTokenizerParallelStack.Pop:TAGToken;
|
||||
var
|
||||
doReplentishStack:boolean;
|
||||
begin
|
||||
FStack := Stack;
|
||||
inherited Create(False, 4 * 4096);
|
||||
FStackLock.Enter;
|
||||
|
||||
if Stack.Count > 0 then
|
||||
Result:=Stack.Dequeue
|
||||
else
|
||||
Result:=Get(Tokenizer);
|
||||
|
||||
doReplentishStack:=FWorker.Idling and(Stack.Count<FStackHalfMax);
|
||||
FStackLock.Leave;
|
||||
|
||||
IsEnd:=Result.ended;
|
||||
if doReplentishStack then
|
||||
FSignal.SetEvent;
|
||||
end;
|
||||
|
||||
procedure TAGPasTokenizerParallelStack.TWorkerThread.Execute;
|
||||
var Count: integer;
|
||||
isDone: boolean;
|
||||
max: integer;
|
||||
destructor TAGPasTokenizerParallelStack.Destroy;
|
||||
begin
|
||||
Count := FStack.GetCachedCount;
|
||||
max := FStack.FStackHalfMax * 2;
|
||||
repeat
|
||||
isDone := FStack.AddTokenToStack();
|
||||
inc(Count);
|
||||
while not Terminated and(Count >= max)do begin
|
||||
Count := FStack.GetCachedCount();// fetch real count
|
||||
if Count < max then
|
||||
break;
|
||||
Idling := True;
|
||||
FStack.FSignal.WaitFor(1000);
|
||||
Count := FStack.GetCachedCount();
|
||||
end;
|
||||
Idling := False;
|
||||
until Terminated or isDone or FStack.ended;
|
||||
EnsureThreadDone();
|
||||
FreeAndNil(FStackLock);
|
||||
inherited;
|
||||
end;
|
||||
|
||||
initialization
|
||||
|
||||
SYMS2:=THashedStringList.Create();
|
||||
SYMS2.Add('>=');
|
||||
SYMS2.Add('<=');
|
||||
@@ -545,5 +521,4 @@ initialization
|
||||
SYMS2.Add('>>');
|
||||
finalization
|
||||
FreeAndNil(SYMS2);
|
||||
|
||||
end.
|
||||
|
||||
49
DOC_EN.MD
49
DOC_EN.MD
@@ -4,50 +4,63 @@ Tokenizer Library for Delphi/FPC. Based on [PyPascalTokenizer](https://github.co
|
||||
|
||||
## API
|
||||
|
||||
### TToken struct
|
||||
### TAGToken struct
|
||||
|
||||
TToken is a `record` type of:
|
||||
TAGToken is a `record` type of:
|
||||
|
||||
* Text: `string` with token text.
|
||||
* &begin: `TTokenizerPos` with start position of token.
|
||||
* &end: `TTokenizerPos` with end position of token.
|
||||
* &begin: `TAGTokenizerPos` with start position of token.
|
||||
* &end: `TAGTokenizerPos` with end position of token.
|
||||
* ended: `boolean` flag, true if it was the last token.
|
||||
|
||||
Has Create constructor with all values.
|
||||
|
||||
### TTokenizerPos
|
||||
### TAGTokenizerPos
|
||||
|
||||
`record`, saves position of token if format:
|
||||
|
||||
* x: `integer` platform-depend-base (`NEXTGEN`) character index in line.
|
||||
* y: `integer` 0-based line index.
|
||||
|
||||
### `Class` PasTokenizer
|
||||
### `Class` TAGPasTokenizer
|
||||
|
||||
Main class of tokenizer.
|
||||
|
||||
* s: `TStrings` private, use constructor. List of source code strings.
|
||||
* get_next(): function, returns `TToken`. Get next token and change end pos
|
||||
* read_next(): function, not implemented, get next token, but don't change end pos.
|
||||
* is_ended(): function, not implemented, check if text ended.
|
||||
* get_next(): `TAGToken` function, returns. Get next token and change pos
|
||||
* pos: `TAGTokenizerPos` property, containing position whear tokenizer was stoped after last get_next(), writeble.
|
||||
* is_ended: `boolean` property, check if text ended.
|
||||
|
||||
Has constructor Create with `s` param.
|
||||
Has constructor Create with `input`: `TStrings` param.
|
||||
|
||||
### `Class` PasTokenizerStack
|
||||
### `Class` TAGPasTokenizerStack
|
||||
|
||||
Not implemented.
|
||||
Stack of tokens automaticaly filled from input.
|
||||
|
||||
### `Class` PasTokenizerParallelStack
|
||||
* Push(t: `TAGToken`) push token to stack.
|
||||
* Pop(): `TAGToken` pop token from stack.
|
||||
* Last: `TAGToken` property, seek stask head.
|
||||
* ended: `boolean` property, check if text ended.
|
||||
|
||||
Not implemented.
|
||||
Has constructor Create with input: `TStrings` and GetComments: `boolean=True` param.
|
||||
|
||||
### `Class` TAGPasTokenizerParallelStack
|
||||
|
||||
Stack of tokens automaticaly filled from input(in second tread).
|
||||
|
||||
* Push(t: `TAGToken`) push token to stack.
|
||||
* Pop(): `TAGToken` pop token from stack.
|
||||
* Last: `TAGToken` property, seek stask head.
|
||||
* ended: `boolean` property, check if text ended.
|
||||
|
||||
Has constructor Create with input: `TStrings`, GetComments: `boolean=True` and stackMax:`integer=1000` param.
|
||||
|
||||
## Utils
|
||||
|
||||
Helper functions to analyze token text.
|
||||
|
||||
* is_name(s: `string`): `boolean` Check for valid identifier (can be reserved word too).
|
||||
* is_comment(s: `string`): `boolean` Check for valid comment.
|
||||
* is_string(s: `string`): `boolean` Check for string constant.
|
||||
* IsName(s: `string`): `boolean` Check for valid identifier (can be reserved word too).
|
||||
* IsComment(s: `string`): `boolean` Check for valid comment.
|
||||
* IsString(s: `string`): `boolean` Check for string constant.
|
||||
|
||||
## Author
|
||||
|
||||
|
||||
51
DOC_RU.MD
51
DOC_RU.MD
@@ -1,53 +1,66 @@
|
||||
# AGPascalTokenizer
|
||||
# AGPascalTokenizer
|
||||
|
||||
Библиотека токенайзер для Delphi/FPC. Основана на работе [PyPascalTokenizer](https://github.com/Artem3213212/PyPascalTokenizer) [Артёма Гаврилова](https://github.com/Artem3213212).
|
||||
|
||||
## API
|
||||
|
||||
### TToken
|
||||
### TAGToken
|
||||
|
||||
TToken это тип `record` из:
|
||||
TAGToken это тип `record` из:
|
||||
|
||||
* Text: `string` с текстом токена.
|
||||
* &begin: `TTokenizerPos` со стартовой позицией токена.
|
||||
* &end: `TTokenizerPos` с конечной позицией токена.
|
||||
* &begin: `TAGTokenizerPos` со стартовой позицией токена.
|
||||
* &end: `TAGTokenizerPos` с конечной позицией токена.
|
||||
* ended: `boolean` Истина, если это был постедний токен.
|
||||
|
||||
Имеет конструктор Create с перечисленными переменными.
|
||||
|
||||
### TTokenizerPos
|
||||
### TAGTokenizerPos
|
||||
|
||||
тип `record`, сохраняет позицию в формате:
|
||||
|
||||
* x: `integer` начало зависит от платформы (`NEXTGEN`), индекс символа в строке.
|
||||
* y: `integer` начало с 0, номер строки.
|
||||
|
||||
### `Class` PasTokenizer
|
||||
### `Class` TAGPasTokenizer
|
||||
|
||||
Главный класс токенайзера.
|
||||
|
||||
* s: `TStrings` приватная, используйте конструктор. Список строк исходного кода.
|
||||
* get_next(): функция, возвращает `TToken`. Возвращает следующий токен и изменяет свою позицию.
|
||||
* read_next(): функция, не имплементированно, Возвращает следующий токен, не изменяя свою позицию.
|
||||
* is_ended(): функция, не имплементированно, проверка что текст закончился.
|
||||
* get_next(): функция, возвращает `TAGToken`. Возвращает следующий токен и изменяет позицию.
|
||||
* pos: `TAGTokenizerPos` свойство, содержит позицию где токенайзер .
|
||||
* is_ended: `boolean` свойство, проверка что текст закончился.
|
||||
|
||||
Имеет конструктор Create с параметром `s`.
|
||||
Имеет конструктор Create с параметром `input`: `TStrings`.
|
||||
|
||||
### `Class` PasTokenizerStack
|
||||
### `Class` TAGPasTokenizerStack
|
||||
|
||||
Не имплементированно.
|
||||
Стек из токенов автоматически заполняется из input'а.
|
||||
|
||||
### `Class` PasTokenizerParallelStack
|
||||
* Push(t: `TAGToken`) кладёт токен в стек.
|
||||
* Pop(): `TAGToken` достать токен из стека.
|
||||
* Last: `TAGToken` свойство, выдающееся вершина стека.
|
||||
* is_ended: `boolean` свойство, проверка конца текста.
|
||||
|
||||
Не имплементированно.
|
||||
Имеет конструктор Create с параметрами `input`: `TStrings` и GetComments: `boolean=True`.
|
||||
|
||||
### `Class` TAGPasTokenizerParallelStack
|
||||
|
||||
Стек из токенов автоматически заполняется из input'а(во втором потоке).
|
||||
|
||||
* Push(t: `TAGToken`) кладёт токен в стек.
|
||||
* Pop(): `TAGToken` достать токен из стека.
|
||||
* Last: `TAGToken` свойство, выдающееся вершина стека.
|
||||
* ended: `boolean` свойство, проверка конца текста.
|
||||
|
||||
Имеет конструктор Create с параметрами input: `TStrings`, GetComments: `boolean=True` и stackMax:`integer=1000`.
|
||||
|
||||
## Утилиты
|
||||
|
||||
Вспомогательные функции для анализа текста.
|
||||
|
||||
* is_name(s: `string`): `boolean` Проверка что идентификатор правильный.
|
||||
* is_comment(s: `string`): `boolean` Проверка что комментарий правильный.
|
||||
* is_string(s: `string`): `boolean` Проверка что это строковое значение.
|
||||
* IsName(s: `string`): `boolean` Проверка что идентификатор правильный.
|
||||
* IsComment(s: `string`): `boolean` Проверка что комментарий правильный.
|
||||
* IsString(s: `string`): `boolean` Проверка что это строковое значение.
|
||||
|
||||
## Авторы
|
||||
|
||||
|
||||
@@ -11,7 +11,7 @@ program Demo;
|
||||
|
||||
uses
|
||||
{$IF not defined(FPC)}
|
||||
FastMM4,
|
||||
//FastMM4,
|
||||
{$endif}
|
||||
SysUtils,
|
||||
Classes,
|
||||
|
||||
@@ -13,11 +13,6 @@
|
||||
<PropertyGroup Condition="'$(Config)'=='Base' or '$(Base)'!=''">
|
||||
<Base>true</Base>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="('$(Platform)'=='Android' and '$(Base)'=='true') or '$(Base_Android)'!=''">
|
||||
<Base_Android>true</Base_Android>
|
||||
<CfgParent>Base</CfgParent>
|
||||
<Base>true</Base>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="('$(Platform)'=='Win32' and '$(Base)'=='true') or '$(Base_Win32)'!=''">
|
||||
<Base_Win32>true</Base_Win32>
|
||||
<CfgParent>Base</CfgParent>
|
||||
@@ -55,19 +50,6 @@
|
||||
<DCC_Namespace>System;Xml;Data;Datasnap;Web;Soap;$(DCC_Namespace)</DCC_Namespace>
|
||||
<SanitizedProjectName>Demo</SanitizedProjectName>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Base_Android)'!=''">
|
||||
<DCC_UsePackage>DBXSqliteDriver;RESTComponents;DBXInterBaseDriver;emsclientfiredac;DataSnapFireDAC;tethering;bindcompfmx;FmxTeeUI;FireDACIBDriver;fmx;FireDACDBXDriver;dbexpress;IndyCore;dsnap;DataSnapCommon;emsclient;FireDACCommon;RESTBackendComponents;soapserver;bindengine;CloudService;FireDACCommonDriver;DataSnapClient;inet;bindcompdbx;IndyIPCommon;IndyIPServer;IndySystem;fmxFireDAC;FireDAC;FireDACSqliteDriver;FMXTee;soaprtl;DbxCommonDriver;xmlrtl;soapmidas;DataSnapNativeClient;FireDACDSDriver;rtl;DbxClientDriver;CustomIPTransport;bindcomp;IndyIPClient;dbxcds;dsnapxml;DataSnapProviderClient;dbrtl;IndyProtocols;$(DCC_UsePackage)</DCC_UsePackage>
|
||||
<Android_LauncherIcon36>$(BDS)\bin\Artwork\Android\FM_LauncherIcon_36x36.png</Android_LauncherIcon36>
|
||||
<Android_LauncherIcon48>$(BDS)\bin\Artwork\Android\FM_LauncherIcon_48x48.png</Android_LauncherIcon48>
|
||||
<Android_LauncherIcon72>$(BDS)\bin\Artwork\Android\FM_LauncherIcon_72x72.png</Android_LauncherIcon72>
|
||||
<Android_LauncherIcon96>$(BDS)\bin\Artwork\Android\FM_LauncherIcon_96x96.png</Android_LauncherIcon96>
|
||||
<Android_LauncherIcon144>$(BDS)\bin\Artwork\Android\FM_LauncherIcon_144x144.png</Android_LauncherIcon144>
|
||||
<Android_SplashImage426>$(BDS)\bin\Artwork\Android\FM_SplashImage_426x320.png</Android_SplashImage426>
|
||||
<Android_SplashImage470>$(BDS)\bin\Artwork\Android\FM_SplashImage_470x320.png</Android_SplashImage470>
|
||||
<Android_SplashImage640>$(BDS)\bin\Artwork\Android\FM_SplashImage_640x480.png</Android_SplashImage640>
|
||||
<Android_SplashImage960>$(BDS)\bin\Artwork\Android\FM_SplashImage_960x720.png</Android_SplashImage960>
|
||||
<EnabledSysJars>android-support-v4.dex.jar;cloud-messaging.dex.jar;fmx.dex.jar;google-analytics-v2.dex.jar;google-play-billing.dex.jar;google-play-licensing.dex.jar;google-play-services-ads-7.0.0.dex.jar;google-play-services-analytics-7.0.0.dex.jar;google-play-services-base-7.0.0.dex.jar;google-play-services-gcm-7.0.0.dex.jar;google-play-services-identity-7.0.0.dex.jar;google-play-services-maps-7.0.0.dex.jar;google-play-services-panorama-7.0.0.dex.jar;google-play-services-plus-7.0.0.dex.jar;google-play-services-wallet-7.0.0.dex.jar</EnabledSysJars>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Base_Win32)'!=''">
|
||||
<DCC_UsePackage>DBXSqliteDriver;RESTComponents;DataSnapServerMidas;DBXDb2Driver;DBXInterBaseDriver;vclactnband;vclFireDAC;emsclientfiredac;DataSnapFireDAC;svnui;tethering;JvGlobus;FireDACADSDriver;JvPluginSystem;DBXMSSQLDriver;JvMM;DatasnapConnectorsFreePascal;FireDACMSSQLDriver;vcltouch;JvBands;vcldb;bindcompfmx;svn;JvJans;DBXOracleDriver;JvNet;inetdb;JvAppFrm;VirtualTreesDR;FmxTeeUI;emsedge;JvDotNetCtrls;FireDACIBDriver;fmx;fmxdae;JvWizards;FireDACDBXDriver;dbexpress;IndyCore;vclx;JvPageComps;dsnap;DataSnapCommon;emsclient;FireDACCommon;JvDB;RESTBackendComponents;DataSnapConnectors;VCLRESTComponents;soapserver;JclDeveloperTools;vclie;bindengine;DBXMySQLDriver;FireDACOracleDriver;CloudService;FireDACMySQLDriver;DBXFirebirdDriver;JvCmp;JvHMI;FireDACCommonODBC;FireDACCommonDriver;DataSnapClient;inet;OverbyteIcsD103Run;bindcompdbx;IndyIPCommon;JvCustom;vcl;DBXSybaseASEDriver;IndyIPServer;JvXPCtrls;IndySystem;FireDACDb2Driver;dsnapcon;FireDACMSAccDriver;fmxFireDAC;FireDACInfxDriver;vclimg;TeeDB;FireDAC;GrtPanelPackage;Jcl;JvCore;emshosting;JvCrypt;FireDACSqliteDriver;FireDACPgDriver;FireDACASADriver;DBXOdbcDriver;FireDACTDataDriver;FMXTee;soaprtl;DbxCommonDriver;JvDlgs;JvRuntimeDesign;JvManagedThreads;Tee;DataSnapServer;xmlrtl;soapmidas;DataSnapNativeClient;fmxobj;vclwinx;FireDACDSDriver;rtl;emsserverresource;DbxClientDriver;JvTimeFramework;DBXSybaseASADriver;CustomIPTransport;vcldsnap;JvSystem;JvStdCtrls;FrameViewerXE9;bindcomp;appanalytics;DBXInformixDriver;IndyIPClient;bindcompvcl;SynEdit_R;TeeUI;JvDocking;dbxcds;VclSmp;JvPascalInterpreter;adortl;FireDACODBCDriver;JclVcl;DataSnapIndy10ServerTransport;dsnapxml;DataSnapProviderClient;dbrtl;inetdbxpress;FireDACMongoDBDriver;IndyProtocols;JvControls;JvPrintPreview;JclContainers;fmxase;$(DCC_UsePackage)</DCC_UsePackage>
|
||||
<DCC_Namespace>Winapi;System.Win;Data.Win;Datasnap.Win;Web.Win;Soap.Win;Xml.Win;Bde;$(DCC_Namespace)</DCC_Namespace>
|
||||
@@ -139,13 +121,24 @@
|
||||
</Excluded_Packages>
|
||||
</Delphi.Personality>
|
||||
<Deployment Version="3">
|
||||
<DeployFile LocalName="$(BDS)\Redist\osx32\libcgunwind.1.0.dylib" Class="DependencyModule">
|
||||
<Platform Name="OSX32">
|
||||
<Overwrite>true</Overwrite>
|
||||
</Platform>
|
||||
</DeployFile>
|
||||
<DeployFile LocalName="Win32\Debug\Demo.exe" Configuration="Debug" Class="ProjectOutput">
|
||||
<Platform Name="Win32">
|
||||
<RemoteName>Demo.exe</RemoteName>
|
||||
<Overwrite>true</Overwrite>
|
||||
</Platform>
|
||||
</DeployFile>
|
||||
<DeployFile LocalName="$(BDS)\Redist\osx64\libcgsqlite3.dylib" Class="DependencyModule">
|
||||
<Platform Name="OSX64">
|
||||
<Overwrite>true</Overwrite>
|
||||
</Platform>
|
||||
</DeployFile>
|
||||
<DeployFile LocalName="$(BDS)\Redist\osx32\libcgunwind.1.0.dylib" Class="DependencyModule">
|
||||
<Platform Name="OSX32">
|
||||
<DeployFile LocalName="$(BDS)\Redist\iossimulator\libPCRE.dylib" Class="DependencyModule">
|
||||
<Platform Name="iOSSimulator">
|
||||
<Overwrite>true</Overwrite>
|
||||
</Platform>
|
||||
</DeployFile>
|
||||
@@ -164,12 +157,6 @@
|
||||
<Overwrite>true</Overwrite>
|
||||
</Platform>
|
||||
</DeployFile>
|
||||
<DeployFile LocalName="Win32\Debug\Demo.exe" Configuration="Debug" Class="ProjectOutput">
|
||||
<Platform Name="Win32">
|
||||
<RemoteName>Demo.exe</RemoteName>
|
||||
<Overwrite>true</Overwrite>
|
||||
</Platform>
|
||||
</DeployFile>
|
||||
<DeployClass Name="AdditionalDebugSymbols">
|
||||
<Platform Name="iOSSimulator">
|
||||
<Operation>1</Operation>
|
||||
@@ -179,6 +166,7 @@
|
||||
<Operation>1</Operation>
|
||||
</Platform>
|
||||
<Platform Name="Win32">
|
||||
<RemoteDir>Contents\MacOS</RemoteDir>
|
||||
<Operation>0</Operation>
|
||||
</Platform>
|
||||
</DeployClass>
|
||||
@@ -314,11 +302,6 @@
|
||||
<Operation>1</Operation>
|
||||
<Extensions>.framework</Extensions>
|
||||
</Platform>
|
||||
<Platform Name="OSX64">
|
||||
<RemoteDir>Contents\MacOS</RemoteDir>
|
||||
<Operation>1</Operation>
|
||||
<Extensions>.framework</Extensions>
|
||||
</Platform>
|
||||
<Platform Name="Win32">
|
||||
<Operation>0</Operation>
|
||||
</Platform>
|
||||
@@ -341,11 +324,6 @@
|
||||
<Operation>1</Operation>
|
||||
<Extensions>.dylib</Extensions>
|
||||
</Platform>
|
||||
<Platform Name="OSX64">
|
||||
<RemoteDir>Contents\MacOS</RemoteDir>
|
||||
<Operation>1</Operation>
|
||||
<Extensions>.dylib</Extensions>
|
||||
</Platform>
|
||||
<Platform Name="Win32">
|
||||
<Operation>0</Operation>
|
||||
<Extensions>.dll;.bpl</Extensions>
|
||||
@@ -369,11 +347,6 @@
|
||||
<Operation>1</Operation>
|
||||
<Extensions>.dylib</Extensions>
|
||||
</Platform>
|
||||
<Platform Name="OSX64">
|
||||
<RemoteDir>Contents\MacOS</RemoteDir>
|
||||
<Operation>1</Operation>
|
||||
<Extensions>.dylib</Extensions>
|
||||
</Platform>
|
||||
<Platform Name="Win32">
|
||||
<Operation>0</Operation>
|
||||
<Extensions>.bpl</Extensions>
|
||||
@@ -396,10 +369,6 @@
|
||||
<RemoteDir>Contents\Resources\StartUp\</RemoteDir>
|
||||
<Operation>0</Operation>
|
||||
</Platform>
|
||||
<Platform Name="OSX64">
|
||||
<RemoteDir>Contents\Resources\StartUp\</RemoteDir>
|
||||
<Operation>0</Operation>
|
||||
</Platform>
|
||||
<Platform Name="Win32">
|
||||
<Operation>0</Operation>
|
||||
</Platform>
|
||||
@@ -547,30 +516,18 @@
|
||||
<RemoteDir>..\</RemoteDir>
|
||||
<Operation>1</Operation>
|
||||
</Platform>
|
||||
<Platform Name="OSX64">
|
||||
<RemoteDir>..\</RemoteDir>
|
||||
<Operation>1</Operation>
|
||||
</Platform>
|
||||
</DeployClass>
|
||||
<DeployClass Name="ProjectOSXInfoPList">
|
||||
<Platform Name="OSX32">
|
||||
<RemoteDir>Contents</RemoteDir>
|
||||
<Operation>1</Operation>
|
||||
</Platform>
|
||||
<Platform Name="OSX64">
|
||||
<RemoteDir>Contents</RemoteDir>
|
||||
<Operation>1</Operation>
|
||||
</Platform>
|
||||
</DeployClass>
|
||||
<DeployClass Name="ProjectOSXResource">
|
||||
<Platform Name="OSX32">
|
||||
<RemoteDir>Contents\Resources</RemoteDir>
|
||||
<Operation>1</Operation>
|
||||
</Platform>
|
||||
<Platform Name="OSX64">
|
||||
<RemoteDir>Contents\Resources</RemoteDir>
|
||||
<Operation>1</Operation>
|
||||
</Platform>
|
||||
</DeployClass>
|
||||
<DeployClass Required="true" Name="ProjectOutput">
|
||||
<Platform Name="Android">
|
||||
@@ -593,10 +550,6 @@
|
||||
<RemoteDir>Contents\MacOS</RemoteDir>
|
||||
<Operation>1</Operation>
|
||||
</Platform>
|
||||
<Platform Name="OSX64">
|
||||
<RemoteDir>Contents\MacOS</RemoteDir>
|
||||
<Operation>1</Operation>
|
||||
</Platform>
|
||||
<Platform Name="Win32">
|
||||
<Operation>0</Operation>
|
||||
</Platform>
|
||||
@@ -640,7 +593,6 @@
|
||||
<ProjectRoot Platform="iOSSimulator" Name="$(PROJECTNAME).app"/>
|
||||
</Deployment>
|
||||
<Platforms>
|
||||
<Platform value="Android">False</Platform>
|
||||
<Platform value="Win32">True</Platform>
|
||||
<Platform value="Win64">False</Platform>
|
||||
</Platforms>
|
||||
|
||||
Reference in New Issue
Block a user