Add Multi threading

AG.PascalTokenizer.pas

@@ -5,260 +5,253 @@ unit AG.PascalTokenizer;
 
 interface
 
-  uses
+uses
-    SysUtils, Classes, IniFiles, SyncObjs, Generics.Collections;
+  SysUtils,Classes,IniFiles,SyncObjs,Generics.Collections;
 
-
+type
-  type
+  TAGTokenizerPos=record
-    TAGTokenizerPos = record
+    x,y:integer;
-      x, y: integer;
   end;
 
   TAGToken = record
     Text:string;
-      &begin, &end: TAGTokenizerPos;
+    &begin,&end:TAGTokenizerPos;
-      ended: boolean;
+    ended:boolean;
-      procedure Make(const Text:string;&begin, &end: TAGTokenizerPos;ended: boolean);
+    constructor Create(const Text:string;&begin,&end:TAGTokenizerPos;ended:boolean);
   end;
 
-    TAGPasTokenizer = class
+  TAGPasTokenizer=class
     strict protected
-        FStrings: TStrings;
+      FStrings:TStrings;
-        FLineIx: integer;
+      FLineIx,x:integer;
-        x: integer;
+      ended:boolean;
-        function DoReadable(): boolean;
+      procedure DoReadable();
-        function IsReadable(): boolean;
+      function IsReadable():boolean;
-        function NextReadable(): boolean;
+      procedure NextReadable();
       procedure SkipSpaces();
-        function GetPos(): TAGTokenizerPos;
+      function GetPos():TAGTokenizerPos;
-        procedure SetPos(pos: TAGTokenizerPos);
+      procedure SetPos(pos:TAGTokenizerPos);
     public
-        ended: boolean;
+      constructor Create(input:TStrings);
-        function GetNext(): TAGToken;
+      function GetNext():TAGToken;
-        // procedure read_next();
+      property pos:TAGTokenizerPos read GetPos write SetPos;
-        constructor Create(input: TStrings);
+      property is_ended:boolean read ended;
-        property pos: TAGTokenizerPos read GetPos write SetPos;
   end;
 
   TAGPasTokenizerStack = class
     strict protected
       type
-        GetCall = function(Tokenizer: TAGPasTokenizer): TAGToken of object;
+        GetCall=function(Tokenizer:TAGPasTokenizer):TAGToken of object;
-
       var
-        Stack: TQueue<TAGToken>;
+        Stack:TQueue<TAGToken>;
-        Tokenizer: TAGPasTokenizer;
+        Tokenizer:TAGPasTokenizer;
-        Get: GetCall;
+        Get:GetCall;
-        IsEnd: boolean;
+        IsEnd:boolean;
-        function GetLast(): TAGToken;virtual;
+      function GetLast():TAGToken;virtual;
-        function GetWithComments(Tokenizer: TAGPasTokenizer): TAGToken;
+      function GetWithComments(Tokenizer:TAGPasTokenizer):TAGToken;
-        function GetWithoutComments(Tokenizer: TAGPasTokenizer): TAGToken;
+      function GetWithoutComments(Tokenizer:TAGPasTokenizer):TAGToken;
-        destructor Destroy;override;
     protected
-        function GetCachedCount: integer;inline;
+      function GetCachedCount:integer;inline;
     public
-        constructor Create(input: TStrings;GetComments: boolean = True);
+      constructor Create(input:TStrings;GetComments:boolean=True);
-        procedure Push(const t: TAGToken);virtual;
+      procedure Push(const t:TAGToken);virtual;
-        function Pop(): TAGToken;virtual;
+      function Pop():TAGToken;virtual;
-        property Last: TAGToken read GetLast write Push;
+      destructor Destroy;override;
-        property ended: boolean read IsEnd;
+      property Last:TAGToken read GetLast write Push;
+      property ended:boolean read IsEnd;
   end;
 
-    TAGPasTokenizerParallelStack = class(TAGPasTokenizerStack)
+  TAGPasTokenizerParallelStack=class(TAGPasTokenizerStack)
     strict protected
       type
-        TWorkerThread = class(TThread)
+        TWorkerThread=class(TThread)
           strict protected
-            FStack: TAGPasTokenizerParallelStack;
+            FStack:TAGPasTokenizerParallelStack;
             procedure Execute;override;
           public
-            Idling: boolean;
+            Idling:boolean;
-            constructor Create(const Stack: TAGPasTokenizerParallelStack);
+            constructor Create(const Stack:TAGPasTokenizerParallelStack);
         end;
-
     var
-        FWorker: TWorkerThread;
+      FWorker:TWorkerThread;
-        FStackLock: TCriticalSection;
+      FStackLock:TCriticalSection;
-        function AddTokenToStack: boolean;
+      function AddTokenToStack:boolean;
-        function GetLast(): TAGToken;override;
+      function GetLast():TAGToken;override;
       procedure EnsureThreadDone();
-        destructor Destroy;override;
     protected
-        FStackHalfMax: integer;
+      FStackHalfMax:integer;
-        FSignal: TEvent;
+      FSignal:TEvent;
-        procedure Push(const t: TAGToken);override;
     public
-        constructor Create(const input: TStrings;GetComments: boolean = True;
+      constructor Create(const input:TStrings;GetComments:boolean=True;stackMax:integer=1000);
-        stackMax: integer = 1000);
+      procedure Push(const t:TAGToken);override;
-        function Pop(): TAGToken;override;
+      function Pop():TAGToken;override;
+      destructor Destroy;override;
   end;
 
-  function IsComment(s:string): boolean;
+function IsComment(s:string):boolean;
-  function IsName(s:string): boolean;
+function IsName(s:string): boolean;
-  function IsString(const s:string): boolean;
+function IsString(const s:string): boolean;
 
 implementation
 
-  procedure TAGToken.Make(const Text:string;&begin, &end: TAGTokenizerPos;ended:
+const
-  boolean);
+  SYMS1='()[]/|\@#=><:;,.$+-*^';
-  begin
+  SPACES=#12#10#13#9#11' ';
-    Self.Text := Text;
+  NO_NAME_SYMS=SYMS1+SPACES+'{}';
-    Self.&begin := &begin;
+  CHARS_ID0='&abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_';
-    Self.&end := &end;
+  CHARS_ID='abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_';
-    Self.ended := ended;
+  fix={$IFDEF NEXTGEN}-1{$ELSE}0{$ENDIF};
-  end;
 
-  const
+var
-    SYMS1 = '()[]/|\@#=><:;,.$+-*^';
+  SYMS2:THashedStringList;
-    SPACES = #12#10#13#9#11' ';
-    NO_NAME_SYMS = SYMS1 + SPACES + '{}';
-    CHARS_ID0 = '&abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_';
-    CHARS_ID = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_';
-    fix = {$IFDEF NEXTGEN}-1{$ELSE}0{$ENDIF};
 
-  var
+function IsComment(s:string):boolean;
-    SYMS2: THashedStringList;
+begin
+Result:=s.startswith('{')or s.startswith('(*')or s.startswith('//');
+end;
 
-  function IsComment(s:string): boolean;
+function IsName(s:string):boolean;
-  begin
+var
-    Result :=(s.startswith('{')or s.startswith('(*')or s.startswith('//'));
+  i:integer;
-  end;
+begin
-
+if(length(s)=0)or(s='&')or(not CHARS_ID0.Contains(s[1+fix]))then
-  function IsName(s:string): boolean;
-  var
-    i: integer;
-  begin
-    if length(s)<= 0 then
   Exit(False);
-    if s = '&' then
+for i:=2+fix to length(s)+fix do
-      Exit(False);
-    if not CHARS_ID0.Contains(s[1 + fix])then
-      Exit(False);
-    for i := 1 to length(s)do
-    begin
   if not CHARS_ID.Contains(s[i])then
     Exit(False);
-    end;
+Result:=True;
-    Result := True;
+end;
-  end;
 
-  function IsString(const s:string): boolean;
+function IsString(const s:string):boolean;
-  begin
+begin
-    Result := s.startswith(#39);
+Result:=s.startswith(#39);
-  end;
+end;
 
-  function TAGPasTokenizer.DoReadable(): boolean;
+{TAGToken}
-  begin
+
-    if not IsReadable()then
+constructor TAGToken.Create(const Text:string;&begin,&end:TAGTokenizerPos;ended:boolean);
-    begin
+begin
-      if(FLineIx + 1 = FStrings.Count)then
+Self.Text:=Text;
-        ended := True
+Self.&begin:=&begin;
+Self.&end:=&end;
+Self.ended:=ended;
+end;
+
+{TAGPasTokenizer}
+
+procedure TAGPasTokenizer.DoReadable();
+begin
+if not IsReadable()then
+  if FLineIx+1=FStrings.Count then
+    ended:=True
   else
   begin
     inc(FLineIx);
-        x := 1 + fix;
+    x:=1+fix;
-        while FStrings[FLineIx]<= '' do
+    while FStrings[FLineIx]<='' do
     begin
-          if FLineIx + 1 = FStrings.Count then
+      if FLineIx+1=FStrings.Count then
       begin
-            ended := True;
+        ended:=True;
         break;
       end;
       inc(FLineIx);
     end;
   end;
-      Exit(True);
+end;
-    end
-    else
-      Exit(False);
-  end;
 
-  function TAGPasTokenizer.IsReadable(): boolean;
+function TAGPasTokenizer.IsReadable():boolean;
-  begin
+begin
-    Result := x <= length(FStrings[FLineIx])+ fix;
+Result:=x<=length(FStrings[FLineIx])+fix;
-  end;
+end;
 
-  function TAGPasTokenizer.NextReadable(): boolean;
+procedure TAGPasTokenizer.NextReadable();
-  begin
+begin
-    inc(x);
+inc(x);
-    Result := DoReadable();
+DoReadable();
-  end;
+end;
 
-  procedure TAGPasTokenizer.SkipSpaces();
+procedure TAGPasTokenizer.SkipSpaces();
-  begin
+begin
-    DoReadable();
+DoReadable();
-    if not ended then
+if not ended then
-    begin
   while SPACES.Contains(FStrings[FLineIx][x])do
     NextReadable();
-    end;
+end;
-  end;
 
-  function TAGPasTokenizer.GetPos(): TAGTokenizerPos;
+function TAGPasTokenizer.GetPos():TAGTokenizerPos;
-  begin
+begin
-    Result.x := x;
+Result.x:=x;
-    Result.y := FLineIx;
+Result.y:=FLineIx;
-  end;
+end;
 
-  procedure TAGPasTokenizer.SetPos(pos: TAGTokenizerPos);
+procedure TAGPasTokenizer.SetPos(pos:TAGTokenizerPos);
-  begin
+begin
-    FLineIx := pos.x;
+FLineIx:=pos.x;
-    x := pos.y;
+x:=pos.y;
-    ended := False;
+ended:=False;
-    DoReadable();
+DoReadable();
-  end;
+end;
 
-  function TAGPasTokenizer.GetNext(): TAGToken;
+constructor TAGPasTokenizer.Create(input: TStrings);
-  var
+begin
-    l,  last_i0: integer;
+FStrings:=input;
-    ml, ss, line:string;
+FLineIx:=0;
-    now_sym, next_sym: char;
+x:=1+fix;
-    f: boolean;
+ended:=False;
-    begin_pos: TAGTokenizerPos;
+SkipSpaces;
-  begin
+end;
-    ml := '';
+
-    ss := '';
+{$WARN USE_BEFORE_DEF OFF}
-    f := True;
+function TAGPasTokenizer.GetNext():TAGToken;
-    begin_pos := GetPos();
+var
+  l,last_i0:integer;
+  ml,ss,line:string;
+  now_sym,next_sym:char;
+  f:boolean;
+  begin_pos:TAGTokenizerPos;
+begin
+  ml:='';
+  ss:='';
+  f:=True;
+  begin_pos:=GetPos();
   while f and not ended do
   begin
-      line := FStrings[FLineIx];
+    line:=FStrings[FLineIx];
-      now_sym := line[x];
+    now_sym:=line[x];
-      l := length(line);
+    l:=length(line);
-      if x < l + fix then
+    if x<l+fix then
-        next_sym := line[x + 1]
+      next_sym:=line[x+1]
     else
-        next_sym := #0;
+      next_sym:=#0;
 
-      if ml = '' then
+    if ml='' then
     begin
-        if now_sym = '/' then
+      if now_sym='/' then
+        if next_sym='/' then
         begin
-          if next_sym = '/' then
+          ss:=Copy(line,x,l);
-          begin
+          x:=l+1+fix;
-            ss := Copy(line, x, l);
-            x := l + 1 + fix;
           break;
-          end;
         end
-        else if now_sym = '{' then
+      else if now_sym='{' then
       begin
-          ml := '}';
+        ml:='}';
-          ss := now_sym;
+        ss:=now_sym;
-          last_i0 := FLineIx;
+        last_i0:=FLineIx;
       end
       else if now_sym = '(' then
       begin
         if next_sym = '*' then
         begin
-            ml := ')';
+          ml:=')';
           inc(x);
-            last_i0 := FLineIx;
+          last_i0:=FLineIx;
-            ss := now_sym + next_sym;
+          ss:=now_sym+next_sym;
         end
         else
         begin
-            ss := '(';
+          ss:='(';
           inc(x);
           break;
         end;
@@ -300,7 +293,7 @@ implementation
         begin
           while not NO_NAME_SYMS.Contains(line[x])do
           begin
-              ss := ss + line[x];
+            ss:=ss+line[x];
             inc(x);
             if not IsReadable()then
               break;
@@ -311,19 +304,19 @@ implementation
     end
     else
     begin
-        while last_i0 <> FLineIx do
+      while last_i0<>FLineIx do
       begin
-          ss := ss + #10;
+        ss:=ss+#10;
         inc(last_i0);
       end;
-        ss := ss + now_sym;
+      ss:=ss+now_sym;
-        if now_sym = ml then
+      if now_sym=ml then
-          if ml = '}' then
+        if ml='}' then
         begin
           inc(x);
           break;
         end
-          else if(x <> 0)and(line[x - 1]= '*')then
+        else if(x<>0)and(line[x - 1]='*')then
         begin
           inc(x);
           break;
@@ -331,219 +324,201 @@ implementation
     end;
     NextReadable();
   end;
-    Result.Make(ss, begin_pos, GetPos, ended);
+  Result:=TAGToken.Create(ss,begin_pos,GetPos,ended);
-    SkipSpaces;
+  SkipSpaces();
-  end;
+end;
+{$WARN USE_BEFORE_DEF ON}
 
-  constructor TAGPasTokenizer.Create(input: TStrings);
+{TAGPasTokenizerStack}
-  begin
-    FStrings := input;
-    FLineIx := 0;
-    x := 1 + fix;
-    ended := False;
-    SkipSpaces;
-  end;
 
-  { TAGPasTokenizerStack }
+function TAGPasTokenizerStack.GetLast(): TAGToken;
-
+begin
-  destructor TAGPasTokenizerStack.Destroy;
+if Stack.Count<>0 then
-  begin
+  Result:=Stack.Peek
-    FreeAndNil(Stack);
+else
-    FreeAndNil(Tokenizer);
+begin
-    inherited;
+  Result:=Get(Tokenizer);
-  end;
-
-  function TAGPasTokenizerStack.GetCachedCount: integer;
-  begin
-    Result := Stack.Count
-  end;
-
-  function TAGPasTokenizerStack.GetLast(): TAGToken;
-  begin
-    if Stack.Count <> 0 then
-      Result := Stack.Peek
-    else
-    begin
-      Result := Get(Tokenizer);
   Stack.Enqueue(Result);
-    end;
+end;
-  end;
+end;
 
-  function TAGPasTokenizerStack.GetWithComments(
+function TAGPasTokenizerStack.GetWithComments(Tokenizer:TAGPasTokenizer):TAGToken;
-  Tokenizer: TAGPasTokenizer): TAGToken;
+begin
-  begin
+Result:=Tokenizer.GetNext;
-    Result := Tokenizer.GetNext;
+end;
-  end;
 
-  function TAGPasTokenizerStack.GetWithoutComments(
+function TAGPasTokenizerStack.GetWithoutComments(Tokenizer:TAGPasTokenizer):TAGToken;
-  Tokenizer: TAGPasTokenizer): TAGToken;
+begin
-  var done: boolean;
+repeat
-  begin
+  Result:=Tokenizer.GetNext;
-    repeat
+until Result.ended or not IsComment(Result.Text);
-      Result := Tokenizer.GetNext;
+IsEnd:=Result.ended;
-    until Result.ended or not IsComment(Result.Text);
+end;
-    IsEnd := Result.ended;
-  end;
 
-  constructor TAGPasTokenizerStack.Create(input: TStrings;
+function TAGPasTokenizerStack.GetCachedCount:integer;
-  GetComments: boolean = True);
+begin
-  begin
+Result:=Stack.Count
-    Stack := TQueue<TAGToken>.Create();
+end;
-    Tokenizer := TAGPasTokenizer.Create(input);
 
-    if GetComments then
+constructor TAGPasTokenizerStack.Create(input:TStrings;GetComments:boolean=True);
-      Get := GetWithComments
+begin
-    else
+Stack:=TQueue<TAGToken>.Create();
-      Get := GetWithoutComments;
+Tokenizer:=TAGPasTokenizer.Create(input);
-  end;
+if GetComments then
+  Get:=GetWithComments
+else
+  Get:=GetWithoutComments;
+end;
 
-  procedure TAGPasTokenizerStack.Push(const t: TAGToken);
+procedure TAGPasTokenizerStack.Push(const t: TAGToken);
-  begin
+begin
-    Stack.Enqueue(t);
+Stack.Enqueue(t);
-  end;
+end;
 
-  function TAGPasTokenizerStack.Pop(): TAGToken;
+function TAGPasTokenizerStack.Pop():TAGToken;
-  begin
+begin
-    if Stack.Count > 0 then
+if Stack.Count>0 then
-      Result := Stack.Dequeue
+  Result:=Stack.Dequeue
-    else
+else
-      Result := Get(Tokenizer);
+  Result:=Get(Tokenizer);
-    IsEnd := Result.ended;
+IsEnd:=Result.ended;
-  end;
+end;
 
-  { TAGPasTokenizerParallelStack }
+destructor TAGPasTokenizerStack.Destroy;
+begin
+FreeAndNil(Stack);
+FreeAndNil(Tokenizer);
+inherited;
+end;
 
-  function TAGPasTokenizerParallelStack.AddTokenToStack(): boolean;
+{TAGPasTokenizerParallelStack}
-  var tkn: TAGToken;
-  begin
-    FStackLock.Enter;
-    tkn := Get(Tokenizer);
-    Result := tkn.ended;
-    Stack.Enqueue(tkn);
-    FStackLock.Leave;
-  end;
 
-  constructor TAGPasTokenizerParallelStack.Create(const input: TStrings;
+{TAGPasTokenizerParallelStack.TWorkerThread}
-  GetComments:
-  boolean = True;stackMax: integer = 1000);
-  begin
-    inherited Create(input, GetComments);
-    FStackHalfMax := stackMax shr 1;
-    FStackLock := TCriticalSection.Create();
-    FSignal := TEvent.Create(nil, False, True, 'ag_tkn_work_signal');
-    FWorker := TWorkerThread.Create(Self);
-  end;
 
-  destructor TAGPasTokenizerParallelStack.Destroy;
+procedure TAGPasTokenizerParallelStack.TWorkerThread.Execute;
-  begin
+var
-    EnsureThreadDone();
+  Count,max:integer;
-    FreeAndNil(FStackLock);
+  isDone:boolean;
-    inherited;
+begin
-  end;
+Count:=FStack.GetCachedCount;
-
+max:=FStack.FStackHalfMax*2;
-  procedure TAGPasTokenizerParallelStack.EnsureThreadDone;
+repeat
-  begin
+  isDone:=FStack.AddTokenToStack();
-    if not FWorker.Terminated then begin
-      FWorker.Terminate;
-      FSignal.SetEvent();// stop idling
-      FWorker.WaitFor();
-    end;
-    FreeAndNil(FWorker);
-    FreeAndNil(FSignal);
-  end;
-
-  function TAGPasTokenizerParallelStack.GetLast: TAGToken;
-  begin
-    FStackLock.Enter;
-    try
-      Result := inherited GetLast();
-    finally
-      FStackLock.Leave;
-    end;
-  end;
-
-  function TAGPasTokenizerParallelStack.Pop: TAGToken;
-  var
-    doReplentishStack, wasRead: boolean;
-  begin
-
-    FStackLock.Enter;
-
-    wasRead := Stack.Count > 0;
-    if wasRead then
-      Result := Stack.Dequeue
-    else
-      Result := Get(Tokenizer);
-
-    doReplentishStack := FWorker.Idling and(Stack.Count < FStackHalfMax);
-    FStackLock.Leave;
-
-    IsEnd := Result.ended;
-    if doReplentishStack then
-      FSignal.SetEvent;
-
-  end;
-
-  procedure TAGPasTokenizerParallelStack.Push(const t: TAGToken);
-  begin
-    FStackLock.Enter;
-    try
-      inherited Push(t);
-    finally
-      FStackLock.Leave;
-    end;
-  end;
-
-  { TAGPasTokenizerParallelStack.TWorkerThread }
-
-  constructor TAGPasTokenizerParallelStack.TWorkerThread.Create(const Stack:
-  TAGPasTokenizerParallelStack);
-  begin
-    FStack := Stack;
-    inherited Create(False, 4 * 4096);
-  end;
-
-  procedure TAGPasTokenizerParallelStack.TWorkerThread.Execute;
-  var Count: integer;
-    isDone: boolean;
-    max: integer;
-  begin
-    Count := FStack.GetCachedCount;
-    max := FStack.FStackHalfMax * 2;
-    repeat
-      isDone := FStack.AddTokenToStack();
   inc(Count);
-      while not Terminated and(Count >= max)do begin
+  while not Terminated and(Count>=max)do
-        Count := FStack.GetCachedCount();// fetch real count
+  begin
-        if Count < max then
+    Count:=FStack.GetCachedCount();
+    if Count<max then
       break;
-        Idling := True;
+    Idling:=True;
     FStack.FSignal.WaitFor(1000);
-        Count := FStack.GetCachedCount();
+    Count:=FStack.GetCachedCount();
-      end;
-      Idling := False;
-    until Terminated or isDone or FStack.ended;
   end;
+  Idling:=False;
+until Terminated or isDone or FStack.ended;
+end;
+
+constructor TAGPasTokenizerParallelStack.TWorkerThread.Create(const Stack:TAGPasTokenizerParallelStack);
+begin
+FStack:=Stack;
+inherited Create(False{,4*4096});
+end;
+
+function TAGPasTokenizerParallelStack.AddTokenToStack():boolean;
+var
+  tkn:TAGToken;
+begin
+FStackLock.Enter;
+tkn:=Get(Tokenizer);
+Result:=tkn.ended;
+Stack.Enqueue(tkn);
+FStackLock.Leave;
+end;
+
+function TAGPasTokenizerParallelStack.GetLast:TAGToken;
+begin
+FStackLock.Enter;
+try
+  Result:=inherited GetLast();
+finally
+  FStackLock.Leave;
+end;
+end;
+
+procedure TAGPasTokenizerParallelStack.EnsureThreadDone;
+begin
+if not FWorker.Terminated then
+begin
+  FWorker.Terminate;
+  FSignal.SetEvent();
+  FWorker.WaitFor();
+end;
+FreeAndNil(FWorker);
+FreeAndNil(FSignal);
+end;
+
+constructor TAGPasTokenizerParallelStack.Create(const input:TStrings;GetComments:boolean=True;stackMax:integer=1000);
+begin
+inherited Create(input,GetComments);
+FStackHalfMax:=stackMax shr 1;
+FStackLock:=TCriticalSection.Create();
+FSignal:=TEvent.Create(nil,False,True,'ag_tkn_work_signal');
+FWorker:=TWorkerThread.Create(Self);
+end;
+
+procedure TAGPasTokenizerParallelStack.Push(const t:TAGToken);
+begin
+FStackLock.Enter;
+try
+  inherited Push(t);
+finally
+  FStackLock.Leave;
+end;
+end;
+
+function TAGPasTokenizerParallelStack.Pop:TAGToken;
+var
+  doReplentishStack:boolean;
+begin
+FStackLock.Enter;
+
+if Stack.Count > 0 then
+  Result:=Stack.Dequeue
+else
+  Result:=Get(Tokenizer);
+
+doReplentishStack:=FWorker.Idling and(Stack.Count<FStackHalfMax);
+FStackLock.Leave;
+
+IsEnd:=Result.ended;
+if doReplentishStack then
+  FSignal.SetEvent;
+end;
+
+destructor TAGPasTokenizerParallelStack.Destroy;
+begin
+EnsureThreadDone();
+FreeAndNil(FStackLock);
+inherited;
+end;
 
 initialization
-
+SYMS2:=THashedStringList.Create();
-  SYMS2 := THashedStringList.Create();
+SYMS2.Add('>=');
-  SYMS2.Add('>=');
+SYMS2.Add('<=');
-  SYMS2.Add('<=');
+SYMS2.Add('<>');
-  SYMS2.Add('<>');
+SYMS2.Add(':=');
-  SYMS2.Add(':=');
+SYMS2.Add('..');
-  SYMS2.Add('..');
+SYMS2.Add('-=');
-  SYMS2.Add('-=');
+SYMS2.Add('+=');
-  SYMS2.Add('+=');
+SYMS2.Add('/=');
-  SYMS2.Add('/=');
+SYMS2.Add('*=');
-  SYMS2.Add('*=');
+SYMS2.Add('**');
-  SYMS2.Add('**');
+SYMS2.Add('><');
-  SYMS2.Add('><');
+SYMS2.Add('(.');
-  SYMS2.Add('(.');
+SYMS2.Add('.)');
-  SYMS2.Add('.)');
+SYMS2.Add('<<');
-  SYMS2.Add('<<');
+SYMS2.Add('>>');
-  SYMS2.Add('>>');
+finalization
-  finalization
+FreeAndNil(SYMS2);
-  FreeAndNil(SYMS2);
-
 end.

DOC_EN.MD

@@ -4,50 +4,63 @@ Tokenizer Library for Delphi/FPC. Based on [PyPascalTokenizer](https://github.co
 
 ## API
 
-### TToken struct
+### TAGToken struct
 
-TToken is a `record` type of:
+TAGToken is a `record` type of:
 
 * Text: `string` with token text.
-* &begin: `TTokenizerPos` with start position of token.
+* &begin: `TAGTokenizerPos` with start position of token.
-* &end: `TTokenizerPos` with end position of token.
+* &end: `TAGTokenizerPos` with end position of token.
 * ended: `boolean` flag, true if it was the last token.
 
 Has Create constructor with all values.
 
-### TTokenizerPos
+### TAGTokenizerPos
 
 `record`, saves position of token if format:
 
 * x: `integer` platform-depend-base (`NEXTGEN`) character index in line.
 * y: `integer` 0-based line index.
 
-### `Class` PasTokenizer
+### `Class` TAGPasTokenizer
 
 Main class of tokenizer.
 
-* s: `TStrings` private, use constructor. List of source code strings.
+* get_next(): `TAGToken` function, returns. Get next token and change pos
-* get_next(): function, returns `TToken`. Get next token and change end pos
+* pos: `TAGTokenizerPos` property, containing position whear tokenizer was stoped after last get_next(), writeble.
-* read_next(): function, not implemented, get next token, but don't change end pos.
+* is_ended: `boolean` property, check if text ended.
-* is_ended(): function, not implemented, check if text ended.
 
-Has constructor Create with `s` param.
+Has constructor Create with `input`: `TStrings` param.
 
-### `Class` PasTokenizerStack
+### `Class` TAGPasTokenizerStack
 
-Not implemented.
+Stack of tokens automaticaly filled from input.
 
-### `Class` PasTokenizerParallelStack
+* Push(t: `TAGToken`) push token to stack.
+* Pop(): `TAGToken` pop token from stack.
+* Last: `TAGToken` property, seek stask head.
+* ended: `boolean` property, check if text ended.
 
-Not implemented.
+Has constructor Create with input: `TStrings` and GetComments: `boolean=True` param.
+
+### `Class` TAGPasTokenizerParallelStack
+
+Stack of tokens automaticaly filled from input(in second tread).
+
+* Push(t: `TAGToken`) push token to stack.
+* Pop(): `TAGToken` pop token from stack.
+* Last: `TAGToken` property, seek stask head.
+* ended: `boolean` property, check if text ended.
+
+Has constructor Create with input: `TStrings`, GetComments: `boolean=True` and stackMax:`integer=1000` param.
 
 ## Utils
 
 Helper functions to analyze token text.
 
-* is_name(s: `string`): `boolean` Check for valid identifier (can be reserved word too).
+* IsName(s: `string`): `boolean` Check for valid identifier (can be reserved word too).
-* is_comment(s: `string`): `boolean` Check for valid comment.
+* IsComment(s: `string`): `boolean` Check for valid comment.
-* is_string(s: `string`): `boolean` Check for string constant.
+* IsString(s: `string`): `boolean` Check for string constant.
 
 ## Author
 

DOC_RU.MD

@@ -1,53 +1,66 @@
-# AGPascalTokenizer
+# AGPascalTokenizer
 
 Библиотека токенайзер для Delphi/FPC. Основана на работе [PyPascalTokenizer](https://github.com/Artem3213212/PyPascalTokenizer) [Артёма Гаврилова](https://github.com/Artem3213212).
 
 ## API
 
-### TToken
+### TAGToken
 
-TToken это тип `record` из:
+TAGToken это тип `record` из:
 
 * Text: `string` с текстом токена.
-* &begin: `TTokenizerPos` со стартовой позицией токена.
+* &begin: `TAGTokenizerPos` со стартовой позицией токена.
-* &end: `TTokenizerPos` с конечной позицией токена.
+* &end: `TAGTokenizerPos` с конечной позицией токена.
 * ended: `boolean` Истина, если это был постедний токен.
 
 Имеет конструктор Create с перечисленными переменными.
 
-### TTokenizerPos
+### TAGTokenizerPos
 
 тип `record`, сохраняет позицию в формате:
 
 * x: `integer` начало зависит от платформы (`NEXTGEN`), индекс символа в строке.
 * y: `integer` начало с 0, номер строки.
 
-### `Class` PasTokenizer
+### `Class` TAGPasTokenizer
 
 Главный класс токенайзера.
 
-* s: `TStrings` приватная, используйте конструктор. Список строк исходного кода.
+* get_next(): функция, возвращает `TAGToken`. Возвращает следующий токен и изменяет позицию.
-* get_next(): функция, возвращает `TToken`. Возвращает следующий токен и изменяет свою позицию.
+* pos: `TAGTokenizerPos` свойство, содержит позицию где токенайзер .
-* read_next(): функция, не имплементированно, Возвращает следующий токен, не изменяя свою позицию.
+* is_ended: `boolean` свойство, проверка что текст закончился.
-* is_ended(): функция, не имплементированно, проверка что текст закончился.
 
-Имеет конструктор Create с параметром `s`.
+Имеет конструктор Create с параметром `input`: `TStrings`.
 
-### `Class` PasTokenizerStack
+### `Class` TAGPasTokenizerStack
 
-Не имплементированно.
+Стек из токенов автоматически заполняется из input'а.
 
-### `Class` PasTokenizerParallelStack
+* Push(t: `TAGToken`) кладёт токен в стек.
+* Pop(): `TAGToken` достать токен из стека.
+* Last: `TAGToken` свойство, выдающееся вершина стека.
+* is_ended: `boolean` свойство, проверка конца текста.
 
-Не имплементированно.
+Имеет конструктор Create с параметрами `input`: `TStrings` и GetComments: `boolean=True`.
+
+### `Class` TAGPasTokenizerParallelStack
+
+Стек из токенов автоматически заполняется из input'а(во втором потоке).
+
+* Push(t: `TAGToken`) кладёт токен в стек.
+* Pop(): `TAGToken` достать токен из стека.
+* Last: `TAGToken` свойство, выдающееся вершина стека.
+* ended: `boolean` свойство, проверка конца текста.
+
+Имеет конструктор Create с параметрами input: `TStrings`, GetComments: `boolean=True` и stackMax:`integer=1000`.
 
 ## Утилиты
 
 Вспомогательные функции для анализа текста.
 
-* is_name(s: `string`): `boolean` Проверка что идентификатор правильный.
+* IsName(s: `string`): `boolean` Проверка что идентификатор правильный.
-* is_comment(s: `string`): `boolean` Проверка что комментарий правильный.
+* IsComment(s: `string`): `boolean` Проверка что комментарий правильный.
-* is_string(s: `string`): `boolean` Проверка что это строковое значение.
+* IsString(s: `string`): `boolean` Проверка что это строковое значение.
 
 ## Авторы
 

Demo/Demo.dpr

@@ -11,7 +11,7 @@ program Demo;
 
 uses
   {$IF not defined(FPC)}
-  FastMM4,
+  //FastMM4,
   {$endif}
   SysUtils,
   Classes,

Demo/Demo.dproj

@@ -13,11 +13,6 @@
     <PropertyGroup Condition="'$(Config)'=='Base' or '$(Base)'!=''">
         <Base>true</Base>
     </PropertyGroup>
-    <PropertyGroup Condition="('$(Platform)'=='Android' and '$(Base)'=='true') or '$(Base_Android)'!=''">
-        <Base_Android>true</Base_Android>
-        <CfgParent>Base</CfgParent>
-        <Base>true</Base>
-    </PropertyGroup>
     <PropertyGroup Condition="('$(Platform)'=='Win32' and '$(Base)'=='true') or '$(Base_Win32)'!=''">
         <Base_Win32>true</Base_Win32>
         <CfgParent>Base</CfgParent>
@@ -55,19 +50,6 @@
         <DCC_Namespace>System;Xml;Data;Datasnap;Web;Soap;$(DCC_Namespace)</DCC_Namespace>
         <SanitizedProjectName>Demo</SanitizedProjectName>
     </PropertyGroup>
-    <PropertyGroup Condition="'$(Base_Android)'!=''">
-        <DCC_UsePackage>DBXSqliteDriver;RESTComponents;DBXInterBaseDriver;emsclientfiredac;DataSnapFireDAC;tethering;bindcompfmx;FmxTeeUI;FireDACIBDriver;fmx;FireDACDBXDriver;dbexpress;IndyCore;dsnap;DataSnapCommon;emsclient;FireDACCommon;RESTBackendComponents;soapserver;bindengine;CloudService;FireDACCommonDriver;DataSnapClient;inet;bindcompdbx;IndyIPCommon;IndyIPServer;IndySystem;fmxFireDAC;FireDAC;FireDACSqliteDriver;FMXTee;soaprtl;DbxCommonDriver;xmlrtl;soapmidas;DataSnapNativeClient;FireDACDSDriver;rtl;DbxClientDriver;CustomIPTransport;bindcomp;IndyIPClient;dbxcds;dsnapxml;DataSnapProviderClient;dbrtl;IndyProtocols;$(DCC_UsePackage)</DCC_UsePackage>
-        <Android_LauncherIcon36>$(BDS)\bin\Artwork\Android\FM_LauncherIcon_36x36.png</Android_LauncherIcon36>
-        <Android_LauncherIcon48>$(BDS)\bin\Artwork\Android\FM_LauncherIcon_48x48.png</Android_LauncherIcon48>
-        <Android_LauncherIcon72>$(BDS)\bin\Artwork\Android\FM_LauncherIcon_72x72.png</Android_LauncherIcon72>
-        <Android_LauncherIcon96>$(BDS)\bin\Artwork\Android\FM_LauncherIcon_96x96.png</Android_LauncherIcon96>
-        <Android_LauncherIcon144>$(BDS)\bin\Artwork\Android\FM_LauncherIcon_144x144.png</Android_LauncherIcon144>
-        <Android_SplashImage426>$(BDS)\bin\Artwork\Android\FM_SplashImage_426x320.png</Android_SplashImage426>
-        <Android_SplashImage470>$(BDS)\bin\Artwork\Android\FM_SplashImage_470x320.png</Android_SplashImage470>
-        <Android_SplashImage640>$(BDS)\bin\Artwork\Android\FM_SplashImage_640x480.png</Android_SplashImage640>
-        <Android_SplashImage960>$(BDS)\bin\Artwork\Android\FM_SplashImage_960x720.png</Android_SplashImage960>
-        <EnabledSysJars>android-support-v4.dex.jar;cloud-messaging.dex.jar;fmx.dex.jar;google-analytics-v2.dex.jar;google-play-billing.dex.jar;google-play-licensing.dex.jar;google-play-services-ads-7.0.0.dex.jar;google-play-services-analytics-7.0.0.dex.jar;google-play-services-base-7.0.0.dex.jar;google-play-services-gcm-7.0.0.dex.jar;google-play-services-identity-7.0.0.dex.jar;google-play-services-maps-7.0.0.dex.jar;google-play-services-panorama-7.0.0.dex.jar;google-play-services-plus-7.0.0.dex.jar;google-play-services-wallet-7.0.0.dex.jar</EnabledSysJars>
-    </PropertyGroup>
     <PropertyGroup Condition="'$(Base_Win32)'!=''">
         <DCC_UsePackage>DBXSqliteDriver;RESTComponents;DataSnapServerMidas;DBXDb2Driver;DBXInterBaseDriver;vclactnband;vclFireDAC;emsclientfiredac;DataSnapFireDAC;svnui;tethering;JvGlobus;FireDACADSDriver;JvPluginSystem;DBXMSSQLDriver;JvMM;DatasnapConnectorsFreePascal;FireDACMSSQLDriver;vcltouch;JvBands;vcldb;bindcompfmx;svn;JvJans;DBXOracleDriver;JvNet;inetdb;JvAppFrm;VirtualTreesDR;FmxTeeUI;emsedge;JvDotNetCtrls;FireDACIBDriver;fmx;fmxdae;JvWizards;FireDACDBXDriver;dbexpress;IndyCore;vclx;JvPageComps;dsnap;DataSnapCommon;emsclient;FireDACCommon;JvDB;RESTBackendComponents;DataSnapConnectors;VCLRESTComponents;soapserver;JclDeveloperTools;vclie;bindengine;DBXMySQLDriver;FireDACOracleDriver;CloudService;FireDACMySQLDriver;DBXFirebirdDriver;JvCmp;JvHMI;FireDACCommonODBC;FireDACCommonDriver;DataSnapClient;inet;OverbyteIcsD103Run;bindcompdbx;IndyIPCommon;JvCustom;vcl;DBXSybaseASEDriver;IndyIPServer;JvXPCtrls;IndySystem;FireDACDb2Driver;dsnapcon;FireDACMSAccDriver;fmxFireDAC;FireDACInfxDriver;vclimg;TeeDB;FireDAC;GrtPanelPackage;Jcl;JvCore;emshosting;JvCrypt;FireDACSqliteDriver;FireDACPgDriver;FireDACASADriver;DBXOdbcDriver;FireDACTDataDriver;FMXTee;soaprtl;DbxCommonDriver;JvDlgs;JvRuntimeDesign;JvManagedThreads;Tee;DataSnapServer;xmlrtl;soapmidas;DataSnapNativeClient;fmxobj;vclwinx;FireDACDSDriver;rtl;emsserverresource;DbxClientDriver;JvTimeFramework;DBXSybaseASADriver;CustomIPTransport;vcldsnap;JvSystem;JvStdCtrls;FrameViewerXE9;bindcomp;appanalytics;DBXInformixDriver;IndyIPClient;bindcompvcl;SynEdit_R;TeeUI;JvDocking;dbxcds;VclSmp;JvPascalInterpreter;adortl;FireDACODBCDriver;JclVcl;DataSnapIndy10ServerTransport;dsnapxml;DataSnapProviderClient;dbrtl;inetdbxpress;FireDACMongoDBDriver;IndyProtocols;JvControls;JvPrintPreview;JclContainers;fmxase;$(DCC_UsePackage)</DCC_UsePackage>
         <DCC_Namespace>Winapi;System.Win;Data.Win;Datasnap.Win;Web.Win;Soap.Win;Xml.Win;Bde;$(DCC_Namespace)</DCC_Namespace>
@@ -139,13 +121,24 @@
                 </Excluded_Packages>
             </Delphi.Personality>
             <Deployment Version="3">
+                <DeployFile LocalName="$(BDS)\Redist\osx32\libcgunwind.1.0.dylib" Class="DependencyModule">
+                    <Platform Name="OSX32">
+                        <Overwrite>true</Overwrite>
+                    </Platform>
+                </DeployFile>
+                <DeployFile LocalName="Win32\Debug\Demo.exe" Configuration="Debug" Class="ProjectOutput">
+                    <Platform Name="Win32">
+                        <RemoteName>Demo.exe</RemoteName>
+                        <Overwrite>true</Overwrite>
+                    </Platform>
+                </DeployFile>
                 <DeployFile LocalName="$(BDS)\Redist\osx64\libcgsqlite3.dylib" Class="DependencyModule">
                     <Platform Name="OSX64">
                         <Overwrite>true</Overwrite>
                     </Platform>
                 </DeployFile>
-                <DeployFile LocalName="$(BDS)\Redist\osx32\libcgunwind.1.0.dylib" Class="DependencyModule">
+                <DeployFile LocalName="$(BDS)\Redist\iossimulator\libPCRE.dylib" Class="DependencyModule">
-                    <Platform Name="OSX32">
+                    <Platform Name="iOSSimulator">
                         <Overwrite>true</Overwrite>
                     </Platform>
                 </DeployFile>
@@ -164,12 +157,6 @@
                         <Overwrite>true</Overwrite>
                     </Platform>
                 </DeployFile>
-                <DeployFile LocalName="Win32\Debug\Demo.exe" Configuration="Debug" Class="ProjectOutput">
-                    <Platform Name="Win32">
-                        <RemoteName>Demo.exe</RemoteName>
-                        <Overwrite>true</Overwrite>
-                    </Platform>
-                </DeployFile>
                 <DeployClass Name="AdditionalDebugSymbols">
                     <Platform Name="iOSSimulator">
                         <Operation>1</Operation>
@@ -179,6 +166,7 @@
                         <Operation>1</Operation>
                     </Platform>
                     <Platform Name="Win32">
+                        <RemoteDir>Contents\MacOS</RemoteDir>
                         <Operation>0</Operation>
                     </Platform>
                 </DeployClass>
@@ -314,11 +302,6 @@
                         <Operation>1</Operation>
                         <Extensions>.framework</Extensions>
                     </Platform>
-                    <Platform Name="OSX64">
-                        <RemoteDir>Contents\MacOS</RemoteDir>
-                        <Operation>1</Operation>
-                        <Extensions>.framework</Extensions>
-                    </Platform>
                     <Platform Name="Win32">
                         <Operation>0</Operation>
                     </Platform>
@@ -341,11 +324,6 @@
                         <Operation>1</Operation>
                         <Extensions>.dylib</Extensions>
                     </Platform>
-                    <Platform Name="OSX64">
-                        <RemoteDir>Contents\MacOS</RemoteDir>
-                        <Operation>1</Operation>
-                        <Extensions>.dylib</Extensions>
-                    </Platform>
                     <Platform Name="Win32">
                         <Operation>0</Operation>
                         <Extensions>.dll;.bpl</Extensions>
@@ -369,11 +347,6 @@
                         <Operation>1</Operation>
                         <Extensions>.dylib</Extensions>
                     </Platform>
-                    <Platform Name="OSX64">
-                        <RemoteDir>Contents\MacOS</RemoteDir>
-                        <Operation>1</Operation>
-                        <Extensions>.dylib</Extensions>
-                    </Platform>
                     <Platform Name="Win32">
                         <Operation>0</Operation>
                         <Extensions>.bpl</Extensions>
@@ -396,10 +369,6 @@
                         <RemoteDir>Contents\Resources\StartUp\</RemoteDir>
                         <Operation>0</Operation>
                     </Platform>
-                    <Platform Name="OSX64">
-                        <RemoteDir>Contents\Resources\StartUp\</RemoteDir>
-                        <Operation>0</Operation>
-                    </Platform>
                     <Platform Name="Win32">
                         <Operation>0</Operation>
                     </Platform>
@@ -547,30 +516,18 @@
                         <RemoteDir>..\</RemoteDir>
                         <Operation>1</Operation>
                     </Platform>
-                    <Platform Name="OSX64">
-                        <RemoteDir>..\</RemoteDir>
-                        <Operation>1</Operation>
-                    </Platform>
                 </DeployClass>
                 <DeployClass Name="ProjectOSXInfoPList">
                     <Platform Name="OSX32">
                         <RemoteDir>Contents</RemoteDir>
                         <Operation>1</Operation>
                     </Platform>
-                    <Platform Name="OSX64">
-                        <RemoteDir>Contents</RemoteDir>
-                        <Operation>1</Operation>
-                    </Platform>
                 </DeployClass>
                 <DeployClass Name="ProjectOSXResource">
                     <Platform Name="OSX32">
                         <RemoteDir>Contents\Resources</RemoteDir>
                         <Operation>1</Operation>
                     </Platform>
-                    <Platform Name="OSX64">
-                        <RemoteDir>Contents\Resources</RemoteDir>
-                        <Operation>1</Operation>
-                    </Platform>
                 </DeployClass>
                 <DeployClass Required="true" Name="ProjectOutput">
                     <Platform Name="Android">
@@ -593,10 +550,6 @@
                         <RemoteDir>Contents\MacOS</RemoteDir>
                         <Operation>1</Operation>
                     </Platform>
-                    <Platform Name="OSX64">
-                        <RemoteDir>Contents\MacOS</RemoteDir>
-                        <Operation>1</Operation>
-                    </Platform>
                     <Platform Name="Win32">
                         <Operation>0</Operation>
                     </Platform>
@@ -640,7 +593,6 @@
                 <ProjectRoot Platform="iOSSimulator" Name="$(PROJECTNAME).app"/>
             </Deployment>
             <Platforms>
-                <Platform value="Android">False</Platform>
                 <Platform value="Win32">True</Platform>
                 <Platform value="Win64">False</Platform>
             </Platforms>