Update

2018-10-28 18:57:21 +03:00
parent adaa0dac12
commit 540f953c21
5 changed files with 357 additions and 15 deletions
--- a/AG.PascalTokenizer.pas
+++ b/AG.PascalTokenizer.pas
@@ -3,10 +3,15 @@ unit AG.PascalTokenizer;
 interface

 uses
-  System.Generics.Collections, System.SysUtils,
-  System.Classes;
+  {$IFDEF FPC}fgl,{$ELSE}System.Generics.Collections,System.{$ENDIF}SysUtils,{$IFNDEF FPC}System.{$ENDIF}Classes;
+
+{$IFDEF FPC}
+  {$mode Delphi}
+{$ENDIF}

 type
+  {$IFDEF FPC}TFpcList=specialize TFPGList<String>;{$ENDIF}
+
  TTokenizerPos = record
    x, y: integer;
  end;
@@ -15,8 +20,7 @@ type
    Text: string;
    &begin, &end: TTokenizerPos;
    ended: boolean;
-    constructor Create(Text: string; &begin, &end: TTokenizerPos;
-      ended: boolean);
+    {$IFNDEF FPC}constructor Create(Text: string; &begin, &end: TTokenizerPos;ended:boolean);{$ENDIF}
  end;

  TPasTokenizer = class
@@ -57,6 +61,7 @@ function is_string(s: string): boolean;

 implementation

+{$IFNDEF FPC}
 constructor TToken.Create(Text: string; &begin, &end: TTokenizerPos;
  ended: boolean);
 begin
@@ -65,9 +70,10 @@ begin
  Self.&end := &end;
  Self.ended := ended;
 end;
+{$ENDIF}

 const
-  SYMS1 = '()[]/|\@#=><:;,.$+-*';
+  SYMS1 = '()[]/|\@#=><:;,.$+-*^';
  SPACES = #12#10#13#9#11' ';
  NO_NAME_SYMS = SYMS1 + SPACES + '{}';
  CHARS_ID0 = '&abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_';
@@ -75,7 +81,7 @@ const
  fix = {$IFDEF NEXTGEN}-1{$ELSE}0{$ENDIF};

 var
-  SYMS2: TList<String>; // array[0..8]of string=();
+  SYMS2:{$IFDEF FPC}TFpcList{$ELSE}TList<string>{$ENDIF}; // array[0..8]of string=();

 function is_comment(s: string): boolean;
 begin
@@ -170,7 +176,7 @@ var
  l,i,last_i0:integer;
  ml,ss,line:string;
  now_sym,next_sym:char;
-  f,str_changed:boolean;
+  f,{$IFDEF FPC}ff,{$ENDIF}str_changed:boolean;
  begin_pos:TTokenizerPos;
 begin
  ml := '';
@@ -227,7 +233,18 @@ begin
 	begin
 	  ss := now_sym;
 	  inc(x);
+          {$IFDEF FPC}
+          ff:=false;
+          for i:=0 to SYMS2.Count-1 do
+            if SYMS2[i]=now_sym+next_sym then
+            begin
+              ff:=true;
+              break;
+            end;
+          if ff then
+          {$ELSE}
 	  if SYMS2.Contains(now_sym+next_sym) then
+          {$ENDIF}
 	  begin
 	    inc(x);
 	    ss := ss + next_sym;
@@ -275,11 +292,7 @@ begin
 	ss := ss + #10;
 	inc(last_i0);
      end;
-      if ss[length(ss) + fix] = #10 then
-      begin
-	ss[length(ss) + fix] := now_sym;
-	ss := ss + #10;
-      end;
+      ss:=ss+now_sym;
      if now_sym = ml then
 	if ml = '}' then
 	begin
@@ -294,7 +307,14 @@ begin
    end;
    _next_readable();
  end;
+  {$IFDEF FPC}    
+  Result.Text:=ss;
+  Result.&begin:=begin_pos;
+  Result.&end:=_get_pos;
+  Result.ended:=ended;
+  {$ELSE}
  Result := TToken.Create(ss, begin_pos, _get_pos, ended);
+  {$ENDIF}
  _skip_spaces;
 end;

@@ -308,7 +328,7 @@ begin
 end;

 initialization
-SYMS2 := TList<string>.Create();
+SYMS2 := {$IFDEF FPC}TFpcList{$ELSE}TList<string>{$ENDIF}.Create();
 SYMS2.Add('>=');
 SYMS2.Add('<=');
 SYMS2.Add('<>');
@@ -318,5 +338,10 @@ SYMS2.Add('-=');
 SYMS2.Add('+=');
 SYMS2.Add('/=');
 SYMS2.Add('*=');
-
+SYMS2.Add('**');
+SYMS2.Add('><');
+SYMS2.Add('(.');
+SYMS2.Add('.)');
+SYMS2.Add('<<');
+SYMS2.Add('>>');
 end.
--- a/FPCTests/fpcunitproject1.lpi
+++ b/FPCTests/fpcunitproject1.lpi
@@ -0,0 +1,85 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<CONFIG>
+  <ProjectOptions>
+    <Version Value="10"/>
+    <PathDelim Value="\"/>
+    <General>
+      <SessionStorage Value="InProjectDir"/>
+      <MainUnit Value="0"/>
+      <Title Value="fpcunitproject1"/>
+      <ResourceType Value="res"/>
+      <UseXPManifest Value="True"/>
+      <Icon Value="0"/>
+    </General>
+    <VersionInfo>
+      <StringTable ProductVersion=""/>
+    </VersionInfo>
+    <BuildModes Count="1">
+      <Item1 Name="Default" Default="True"/>
+    </BuildModes>
+    <PublishOptions>
+      <Version Value="2"/>
+    </PublishOptions>
+    <RunParams>
+      <local>
+        <FormatVersion Value="1"/>
+      </local>
+    </RunParams>
+    <RequiredPackages Count="3">
+      <Item1>
+        <PackageName Value="fpcunittestrunner"/>
+      </Item1>
+      <Item2>
+        <PackageName Value="LCL"/>
+      </Item2>
+      <Item3>
+        <PackageName Value="FCL"/>
+      </Item3>
+    </RequiredPackages>
+    <Units Count="2">
+      <Unit0>
+        <Filename Value="fpcunitproject1.lpr"/>
+        <IsPartOfProject Value="True"/>
+      </Unit0>
+      <Unit1>
+        <Filename Value="testcase1.pas"/>
+        <IsPartOfProject Value="True"/>
+        <UnitName Value="TestCase1"/>
+      </Unit1>
+    </Units>
+  </ProjectOptions>
+  <CompilerOptions>
+    <Version Value="11"/>
+    <PathDelim Value="\"/>
+    <Target>
+      <Filename Value="fpcunitproject1"/>
+    </Target>
+    <SearchPaths>
+      <IncludeFiles Value="$(ProjOutDir)"/>
+      <UnitOutputDirectory Value="lib\$(TargetCPU)-$(TargetOS)"/>
+    </SearchPaths>
+    <Linking>
+      <Options>
+        <Win32>
+          <GraphicApplication Value="True"/>
+        </Win32>
+      </Options>
+    </Linking>
+  </CompilerOptions>
+  <Debugging>
+    <Exceptions Count="4">
+      <Item1>
+        <Name Value="EAbort"/>
+      </Item1>
+      <Item2>
+        <Name Value="ECodetoolError"/>
+      </Item2>
+      <Item3>
+        <Name Value="EFOpenError"/>
+      </Item3>
+      <Item4>
+        <Name Value="EListError"/>
+      </Item4>
+    </Exceptions>
+  </Debugging>
+</CONFIG>
--- a/FPCTests/fpcunitproject1.lpr
+++ b/FPCTests/fpcunitproject1.lpr
@@ -0,0 +1,16 @@
+program fpcunitproject1;
+
+{$mode objfpc}{$H+}
+
+uses
+  Interfaces, Forms, GuiTestRunner, TestCase1,
+  AG.PascalTokenizer in '..\AG.PascalTokenizer.pas';
+
+{$R *.res}
+
+begin
+  Application.Initialize;
+  Application.CreateForm(TGuiTestRunner, TestRunner);
+  Application.Run;
+end.
+
--- a/FPCTests/fpcunitproject1.lps
+++ b/FPCTests/fpcunitproject1.lps
@@ -0,0 +1,174 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<CONFIG>
+  <ProjectSession>
+    <PathDelim Value="\"/>
+    <Version Value="10"/>
+    <BuildModes Active="Default"/>
+    <Units Count="5">
+      <Unit0>
+        <Filename Value="fpcunitproject1.lpr"/>
+        <IsPartOfProject Value="True"/>
+        <CursorPos X="60" Y="13"/>
+        <UsageCount Value="20"/>
+        <Loaded Value="True"/>
+      </Unit0>
+      <Unit1>
+        <Filename Value="testcase1.pas"/>
+        <IsPartOfProject Value="True"/>
+        <UnitName Value="TestCase1"/>
+        <EditorIndex Value="1"/>
+        <TopLine Value="7"/>
+        <CursorPos X="68" Y="18"/>
+        <UsageCount Value="20"/>
+        <Loaded Value="True"/>
+      </Unit1>
+      <Unit2>
+        <Filename Value="C:\lazarus\components\fpcunit\guitestrunner.pas"/>
+        <UnitName Value="GuiTestRunner"/>
+        <EditorIndex Value="-1"/>
+        <TopLine Value="44"/>
+        <CursorPos X="12" Y="54"/>
+        <UsageCount Value="10"/>
+      </Unit2>
+      <Unit3>
+        <Filename Value="..\AG.PascalTokenizer.pas"/>
+        <IsVisibleTab Value="True"/>
+        <EditorIndex Value="2"/>
+        <TopLine Value="278"/>
+        <CursorPos X="9" Y="295"/>
+        <UsageCount Value="10"/>
+        <Loaded Value="True"/>
+      </Unit3>
+      <Unit4>
+        <Filename Value="C:\lazarus\fpc\3.0.4\source\rtl\objpas\classes\classesh.inc"/>
+        <EditorIndex Value="3"/>
+        <TopLine Value="715"/>
+        <CursorPos X="47" Y="732"/>
+        <UsageCount Value="10"/>
+        <Loaded Value="True"/>
+      </Unit4>
+    </Units>
+    <JumpHistory Count="30" HistoryIndex="29">
+      <Position1>
+        <Filename Value="..\AG.PascalTokenizer.pas"/>
+        <Caret Line="211" TopLine="191"/>
+      </Position1>
+      <Position2>
+        <Filename Value="..\AG.PascalTokenizer.pas"/>
+        <Caret Line="212" TopLine="191"/>
+      </Position2>
+      <Position3>
+        <Filename Value="..\AG.PascalTokenizer.pas"/>
+        <Caret Line="312" TopLine="295"/>
+      </Position3>
+      <Position4>
+        <Filename Value="..\AG.PascalTokenizer.pas"/>
+        <Caret Line="189" TopLine="171"/>
+      </Position4>
+      <Position5>
+        <Filename Value="..\AG.PascalTokenizer.pas"/>
+        <Caret Line="190" TopLine="171"/>
+      </Position5>
+      <Position6>
+        <Filename Value="..\AG.PascalTokenizer.pas"/>
+        <Caret Line="191" TopLine="171"/>
+      </Position6>
+      <Position7>
+        <Filename Value="..\AG.PascalTokenizer.pas"/>
+        <Caret Line="192" TopLine="171"/>
+      </Position7>
+      <Position8>
+        <Filename Value="..\AG.PascalTokenizer.pas"/>
+        <Caret Line="193" TopLine="171"/>
+      </Position8>
+      <Position9>
+        <Filename Value="..\AG.PascalTokenizer.pas"/>
+        <Caret Line="196" TopLine="171"/>
+      </Position9>
+      <Position10>
+        <Filename Value="..\AG.PascalTokenizer.pas"/>
+        <Caret Line="295" TopLine="278"/>
+      </Position10>
+      <Position11>
+        <Filename Value="..\AG.PascalTokenizer.pas"/>
+        <Caret Line="300" TopLine="278"/>
+      </Position11>
+      <Position12>
+        <Filename Value="..\AG.PascalTokenizer.pas"/>
+        <Caret Line="312" TopLine="284"/>
+      </Position12>
+      <Position13>
+        <Filename Value="..\AG.PascalTokenizer.pas"/>
+        <Caret Line="189" TopLine="171"/>
+      </Position13>
+      <Position14>
+        <Filename Value="..\AG.PascalTokenizer.pas"/>
+        <Caret Line="190" TopLine="171"/>
+      </Position14>
+      <Position15>
+        <Filename Value="..\AG.PascalTokenizer.pas"/>
+        <Caret Line="191" TopLine="171"/>
+      </Position15>
+      <Position16>
+        <Filename Value="..\AG.PascalTokenizer.pas"/>
+        <Caret Line="192" TopLine="171"/>
+      </Position16>
+      <Position17>
+        <Filename Value="..\AG.PascalTokenizer.pas"/>
+        <Caret Line="193" TopLine="171"/>
+      </Position17>
+      <Position18>
+        <Filename Value="..\AG.PascalTokenizer.pas"/>
+        <Caret Line="196" TopLine="171"/>
+      </Position18>
+      <Position19>
+        <Filename Value="..\AG.PascalTokenizer.pas"/>
+        <Caret Line="295" TopLine="278"/>
+      </Position19>
+      <Position20>
+        <Filename Value="..\AG.PascalTokenizer.pas"/>
+        <Caret Line="300" TopLine="278"/>
+      </Position20>
+      <Position21>
+        <Filename Value="..\AG.PascalTokenizer.pas"/>
+        <Caret Line="312" TopLine="284"/>
+      </Position21>
+      <Position22>
+        <Filename Value="..\AG.PascalTokenizer.pas"/>
+        <Caret Line="189" TopLine="171"/>
+      </Position22>
+      <Position23>
+        <Filename Value="..\AG.PascalTokenizer.pas"/>
+        <Caret Line="190" TopLine="171"/>
+      </Position23>
+      <Position24>
+        <Filename Value="..\AG.PascalTokenizer.pas"/>
+        <Caret Line="191" TopLine="171"/>
+      </Position24>
+      <Position25>
+        <Filename Value="..\AG.PascalTokenizer.pas"/>
+        <Caret Line="192" TopLine="171"/>
+      </Position25>
+      <Position26>
+        <Filename Value="..\AG.PascalTokenizer.pas"/>
+        <Caret Line="193" TopLine="171"/>
+      </Position26>
+      <Position27>
+        <Filename Value="..\AG.PascalTokenizer.pas"/>
+        <Caret Line="196" TopLine="171"/>
+      </Position27>
+      <Position28>
+        <Filename Value="..\AG.PascalTokenizer.pas"/>
+        <Caret Line="295" Column="7" TopLine="278"/>
+      </Position28>
+      <Position29>
+        <Filename Value="testcase1.pas"/>
+        <Caret Line="28" Column="11" TopLine="4"/>
+      </Position29>
+      <Position30>
+        <Filename Value="testcase1.pas"/>
+        <Caret Line="29" Column="15" TopLine="4"/>
+      </Position30>
+    </JumpHistory>
+  </ProjectSession>
+</CONFIG>
--- a/FPCTests/testcase1.pas
+++ b/FPCTests/testcase1.pas
@@ -0,0 +1,42 @@
+unit TestCase1;
+
+{$mode objfpc}{$H+}
+
+interface
+
+uses
+  Classes, SysUtils, fpcunit, testutils,GuiTestRunner, testregistry,AG.PascalTokenizer;
+
+type
+
+  TTestCase1= class(TTestCase)
+  published
+    procedure TestHookUp;
+  end;
+
+implementation
+
+procedure TTestCase1.TestHookUp;
+var
+  input:TStrings;
+  tokenizer:TPasTokenizer;
+  token:TToken;
+begin
+  input:= TStringList.Create();
+  input.LoadFromFile('testcase1.pas');
+  tokenizer:=TPasTokenizer.Create(input);
+  token.ended:=False;
+  while not token.ended do
+  begin
+    token:=tokenizer.get_next;
+    TestRunner.MemoLog.Append(token.Text);
+  end;
+end;
+
+
+
+initialization
+
+  RegisterTest(TTestCase1);
+end.
+