* Codepage-aware, allow unicode, refactoring of SQLDB code, add PG connector

git-svn-id: https://svn.freepascal.org/svn/fpc/trunk@39381 3ad0048d-3df7-0310-abae-a5850022a9f2

12 files changed, 606 insertions, 376 deletions

diff --git a/packages/fpindexer/fpmake.pp b/packages/fpindexer/fpmake.pp
index ffcf9e8ab1..8ae28c28bb 100644
--- a/packages/fpindexer/fpmake.pp
+++ b/packages/fpindexer/fpmake.pp
@@ -61,6 +61,9 @@ begin
       T:=P.Targets.AddUnit('fbindexdb.pp',SqliteOSes);
       T.Dependencies.AddUnit('fpindexer');
       
+      T:=P.Targets.AddUnit('pgindexdb.pp',SqldbConnectionOSes);
+      T.Dependencies.AddUnit('fpindexer');
+      
       T:=P.Targets.AddUnit('dbindexer.pp',SqldbConnectionOSes);
       T.Dependencies.AddUnit('fpindexer');
       T.Dependencies.AddUnit('ireadertxt');
diff --git a/packages/fpindexer/src/dbindexer.pp b/packages/fpindexer/src/dbindexer.pp
index 715b716cd7..3e3ee7b227 100644
--- a/packages/fpindexer/src/dbindexer.pp
+++ b/packages/fpindexer/src/dbindexer.pp
@@ -395,9 +395,9 @@ Var
 begin
   Result:=0;
   T:='';
-  R:=TIReaderTXT.Create;
+  URL:=TableName+'/'+KeyField.AsString;
+  R:=TIReaderTXT.Create(URL,CP_UTF8);
   try
-    URL:=TableName+'/'+KeyField.AsString;
     For I:=0 to List.Count-1 do
       begin
       F:=TField(List.Objects[i]);
diff --git a/packages/fpindexer/src/fbindexdb.pp b/packages/fpindexer/src/fbindexdb.pp
index 977837c13d..d561f546d5 100644
--- a/packages/fpindexer/src/fbindexdb.pp
+++ b/packages/fpindexer/src/fbindexdb.pp
@@ -26,26 +26,17 @@ type
 
   TFBIndexDB = class(TSQLDBIndexDB)
   private
-    FIB: TIBConnection;
     FGenQuery: Array[TIndexTable] of TSQLQuery;
     Function GetGenQuery(ATable : TIndexTable) : TSQLQuery;
-    function GetS(AIndex: integer): string;
-    procedure SetS(AIndex: integer; const AValue: string);
   protected
     procedure InsertMatch(AWordID, aFileID, aLanguageID: int64; const ASearchData: TSearchWordData); override;
-    function InsertWord(const AWord: string): int64; override;
-    function InsertURL(const URL: string; ATimeStamp: TDateTime; ALanguageID: int64): int64; override;
-    function InsertLanguage(const ALanguage: string): int64; override;
-    function GetConnection: TSQLConnection; override;
+    function InsertWord(const AWord: UTF8string): int64; override;
+    function InsertURL(const URL: UTF8string; ATimeStamp: TDateTime; ALanguageID: int64): int64; override;
+    function InsertLanguage(const ALanguage: UTF8string): int64; override;
+    function CreateConnection: TSQLConnection; override;
     function GetID(TableType: TIndexTable): int64;
     procedure FinishCreateTable(const TableType: TIndexTable); override;
-  public
-    constructor Create(AOwner: TComponent); override;
-    destructor Destroy; override;
-  published
-    property DatabasePath: string Index 0 read GetS write SetS;
-    property UserName: string Index 1 read GetS write SetS;
-    property Password: string Index 2 read GetS write SetS;
+    procedure FinishDropTable(const TableType: TIndexTable);override;
   end;
 
 implementation
@@ -56,7 +47,6 @@ function TFBIndexDB.GetID(TableType: TIndexTable): int64;
 
 var
   Q: TSQLQuery;
-  S: string;
 
 begin
   Q := GetGenQuery(TableType);
@@ -70,7 +60,7 @@ begin
   end;
 end;
 
-function TFBIndexDB.InsertLanguage(const ALanguage: string): int64;
+function TFBIndexDB.InsertLanguage(const ALanguage: UTF8string): int64;
 var
   Q: TSQLQuery;
 begin
@@ -81,7 +71,7 @@ begin
   Q.ExecSQL;
 end;
 
-function TFBIndexDB.InsertWord(const AWord: string): int64;
+function TFBIndexDB.InsertWord(const AWord: UTF8string): int64;
 var
   Q: TSQLQuery;
 begin
@@ -92,7 +82,7 @@ begin
   Q.ExecSQL;
 end;
 
-function TFBIndexDB.InsertURL(const URL: string; ATimeStamp: TDateTime; ALanguageID: int64): int64;
+function TFBIndexDB.InsertURL(const URL: UTF8string; ATimeStamp: TDateTime; ALanguageID: int64): int64;
 var
   Q: TSQLQuery;
 begin
@@ -117,13 +107,19 @@ begin
   Q.ParamByName(GetFieldName(ifMatchesWordID)).AsLargeInt := aWordID;
   Q.ParamByName(GetFieldName(ifMatchesFileID)).AsLargeInt := aFileID;
   Q.ParamByName(GetFieldName(ifMatchesPosition)).AsLargeInt := ASearchData.Position;
-  Q.ParamByName(GetFieldName(ifMatchesContext)).AsString := ASearchData.Context;
+  Q.ParamByName(GetFieldName(ifMatchesContext)).AsString := Copy(ASearchData.Context,1,MaxContextLen);
   Q.ExecSQL;
 end;
 
-function TFBIndexDB.GetConnection: TSQLConnection;
+function TFBIndexDB.CreateConnection: TSQLConnection;
+Var
+  T : TIndexTable;
+
 begin
-  Result := FiB;
+  // So they are recreated
+  for T in TIndexTable do
+    FreeAndNil(FGenQuery[T]);
+  Result := TIBConnection.Create(Self);
 end;
 
 procedure TFBIndexDB.FinishCreateTable(const TableType: TIndexTable);
@@ -131,16 +127,9 @@ begin
   Execute('CREATE GENERATOR ' + DefaultGeneratorNames[TableType], True);
 end;
 
-constructor TFBIndexDB.Create(AOwner: TComponent);
-begin
-  inherited Create(AOwner);
-  FIB := TIBConnection.Create(nil);
-end;
-
-destructor TFBIndexDB.Destroy;
+procedure TFBIndexDB.FinishDropTable(const TableType: TIndexTable);
 begin
-  // Parent destroys FIB.
-  inherited Destroy;
+  Execute('DROP GENERATOR ' + DefaultGeneratorNames[TableType], True);
 end;
 
 function TFBIndexDB.GetGenQuery(ATable: TIndexTable): TSQLQuery;
@@ -153,25 +142,5 @@ begin
   Result:=FGenQuery[ATable];
 end;
 
-function TFBIndexDB.GetS(AIndex: integer): string;
-begin
-  case Aindex of
-    0: Result := FIB.DatabaseName;
-    1: Result := FIB.UserName;
-    2: Result := FIB.Password;
-    3: Result := FIB.HostName;
-  end;
-end;
-
-procedure TFBIndexDB.SetS(AIndex: integer; const AValue: string);
-begin
-  case Aindex of
-    0: FIB.DatabaseName := AValue;
-    1: FIB.UserName := AValue;
-    2: FIB.Password := AValue;
-    3: FIB.HostName := AValue;
-  end;
-end;
-
 end.
 
diff --git a/packages/fpindexer/src/fpindexer.pp b/packages/fpindexer/src/fpindexer.pp
index 217a5088ae..3a201aaa6d 100644
--- a/packages/fpindexer/src/fpindexer.pp
+++ b/packages/fpindexer/src/fpindexer.pp
@@ -28,7 +28,7 @@ type
   TWordTokenType = (wtOr, wtAnd, wtWord);
 
   TWordToken = record
-    Value: string;
+    Value: UTF8String;
     TokenType: TWordTokenType;
   end;
 
@@ -41,17 +41,15 @@ type
     FCount: integer;
     FWildCardChar: char;
     WordList: array of TWordToken;
-
-    procedure AddToken(AValue: string; ATokenType: TWordTokenType);
-    function GetSearchWordQuery: string;
+    procedure AddToken(AValue: UTF8String; ATokenType: TWordTokenType);
+    function GetSearchWordQuery: UTF8String;
     function GetToken(index: integer): TWordToken;
     procedure SetCount(AValue: integer);
   public
-    constructor Create(ASearchWords: string);
-
+    constructor Create(ASearchWords: UTF8String);
     property Count: integer read FCount write SetCount;
     property WildCardChar: char read FWildCardChar write FWildCardChar;
-    property SearchWordQuery: string read GetSearchWordQuery;
+    property SearchWordQuery: UTF8String read GetSearchWordQuery;
     property Token[index: integer]: TWordToken read GetToken;
   end;
 
@@ -59,13 +57,13 @@ type
   TSearchOptions = set of TSearchOption;
 
   TSearchWordData = record
-    Context: string;
+    Context: UTF8String;
     FileDate: TDateTime;
-    Language: string;
+    Language: UTF8String;
     Position: int64;
     Rank: integer;
-    SearchWord: string;
-    URL: string;
+    SearchWord: UTF8String;
+    URL: UTF8String;
   end;
 
   TFPSearch = class;
@@ -80,14 +78,14 @@ type
     procedure CompactDB; virtual; abstract;
     procedure BeginTrans; virtual; abstract;
     procedure CommitTrans; virtual; abstract;
-    procedure DeleteWordsFromFile(URL: string); virtual; abstract;
+    procedure DeleteWordsFromFile(URL: UTF8String); virtual; abstract;
     procedure AddSearchData(ASearchData: TSearchWordData); virtual; abstract;
     procedure FindSearchData(SearchWord: TWordParser; FPSearch: TFPSearch; SearchOptions: TSearchOptions); virtual; abstract;
     procedure CreateIndexerTables; virtual; abstract;
   end;
 
   TDatabaseID = record
-    Name: string;
+    Name: UTF8String;
     ID: integer;
   end;
 
@@ -115,9 +113,9 @@ const
     itLanguages, itLanguages,
     itFiles, itFiles, itFiles, itFiles, itFiles, itFiles);
 
-  DefaultTableNames: array[TIndexTable] of string = ('WORDS', 'FILELANGUAGES', 'FILENAMES', 'WORDMATCHES');
-  DefaultIndexNames: array[TIndexIndex] of string = ('I_WORDS', 'I_WORDMATCHES', 'I_FILELANGUAGES', 'I_FILENAMES');
-  DefaultFieldNames: array[TIndexField] of string = (
+  DefaultTableNames: array[TIndexTable] of UTF8String = ('WORDS', 'FILELANGUAGES', 'FILENAMES', 'WORDMATCHES');
+  DefaultIndexNames: array[TIndexIndex] of UTF8String = ('I_WORDS', 'I_WORDMATCHES', 'I_FILELANGUAGES', 'I_FILENAMES');
+  DefaultFieldNames: array[TIndexField] of UTF8String = (
     'W_ID', 'W_WORD',
     'WM_ID', 'WM_WORD_FK', 'WM_FILE_FK', 'WM_LANGUAGE_FK', 'WM_POSITION', 'WM_CONTEXT',
     'FL_ID', 'FL_NAME',
@@ -126,7 +124,7 @@ const
   ForeignKeyTargets: array[TIndexForeignKey] of TIndexTable = (itLanguages, itWords, itFiles, itLanguages);
   ForeignKeyFields: array[TIndexForeignKey] of TIndexField = (ifFilesLanguageID, ifMatchesWordID, ifMatchesFileID, ifMatchesLanguageID);
   ForeignKeyTargetFields: array[TIndexForeignKey] of TIndexField = (ifLanguagesID, ifWordsID, ifFilesID, ifLanguagesID);
-  DefaultForeignKeyNames: array[TIndexForeignKey] of string = ('R_FILES_LANGUAGE', 'R_MATCHES_WORD', 'R_MATCHES_FILE', 'R_MATCHES_LANGUAGE');
+  DefaultForeignKeyNames: array[TIndexForeignKey] of UTF8String = ('R_FILES_LANGUAGE', 'R_MATCHES_WORD', 'R_MATCHES_FILE', 'R_MATCHES_LANGUAGE');
   IdFieldType = 'BIGINT NOT NULL';
   PrimaryFieldType = IdFieldType + ' PRIMARY KEY';
   PosFieldType = 'BIGINT';
@@ -134,7 +132,7 @@ const
   TextFieldType = 'VARCHAR(100) NOT NULL';
   LargeTextFieldType = 'VARCHAR(255) NOT NULL';
   TimeStampFieldType = 'TIMESTAMP';
-  DefaultFieldTypes: array[TIndexField] of string = (
+  DefaultFieldTypes: array[TIndexField] of UTF8String = (
     PrimaryFieldType, TextFieldType, PrimaryFieldType, IdFieldType, IdFieldType,
     IdFieldType, PosFieldType, LargeTextFieldType, PrimaryFieldType, TextFieldType,
     PrimaryFieldType, LargeTextFieldType, FlagFieldType, FlagFieldType, TimeStampFieldType,
@@ -146,32 +144,33 @@ type
 
   TSQLIndexDB = class(TCustomIndexDB)
   protected
-    function CreateForeignKey(const ForeignKey: TIndexForeignKey; ForCreate: boolean = False): string;
-    function CreateIndexSQL(const AIndexName, ATableName: string; const AFieldList: array of string): string; virtual;
-    function CreateTableIndex(IndexType: TIndexIndex): string; virtual;
-    function CreateTableSQL(const TableType: TIndexTable): string; virtual;
-    function DeleteWordsSQL(UseParams: boolean = True): string; virtual;
-    function DropTableSQl(TableType: TIndexTable): string; virtual;
-    function GetFieldName(FieldType: TIndexField): string; virtual;
-    function GetFieldType(FieldType: TIndexField): string; virtual;
-    function GetForeignKeyName(ForeignKey: TIndexForeignKey): string; virtual;
-    function GetIndexName(IndexType: TIndexIndex): string; virtual;
-    function GetLanguageSQL(UseParams: boolean = True): string; virtual;
-    function GetMatchSQL(SearchOptions: TSearchOptions; SearchWord: TWordParser; UseParams: boolean = True): string; virtual;
-    function GetSearchFileSQL(UseParams: boolean = True): string; virtual;
-    function GetSearchSQL(ATable: TIndexTable; IDField, SearchField: TINdexField; UseParams: boolean = True): string; virtual;
-    function GetTableName(TableType: TIndexTable): string; virtual;
-    function GetUrlSQL(UseParams: boolean = True): string; virtual;
-    function GetWordSQL(UseParams: boolean = True): string; virtual;
-    function InsertSQL(const TableType: TIndexTable; UseParams: boolean = True): string; virtual;
+    function CreateForeignKey(const ForeignKey: TIndexForeignKey; ForCreate: boolean = False): UTF8String;
+    function CreateIndexSQL(const AIndexName, ATableName: UTF8String; const AFieldList: array of UTF8String): UTF8String; virtual;
+    function CreateTableIndex(IndexType: TIndexIndex): UTF8String; virtual;
+    function CreateTableSQL(const TableType: TIndexTable): UTF8String; virtual;
+    function DeleteWordsSQL(UseParams: boolean = True): UTF8String; virtual;
+    function DropTableSQl(TableType: TIndexTable): UTF8String; virtual;
+    function GetFieldName(FieldType: TIndexField): UTF8String; virtual;
+    function GetFieldType(FieldType: TIndexField): UTF8String; virtual;
+    function GetForeignKeyName(ForeignKey: TIndexForeignKey): UTF8String; virtual;
+    function GetIndexName(IndexType: TIndexIndex): UTF8String; virtual;
+    function GetLanguageSQL(UseParams: boolean = True): UTF8String; virtual;
+    function GetMatchSQL(SearchOptions: TSearchOptions; SearchWord: TWordParser; UseParams: boolean = True): UTF8String; virtual;
+    function GetSearchFileSQL(UseParams: boolean = True): UTF8String; virtual;
+    function GetSearchSQL(ATable: TIndexTable; IDField, SearchField: TINdexField; UseParams: boolean = True): UTF8String; virtual;
+    function GetTableName(TableType: TIndexTable): UTF8String; virtual;
+    function GetUrlSQL(UseParams: boolean = True): UTF8String; virtual;
+    function GetWordSQL(UseParams: boolean = True): UTF8String; virtual;
+    function InsertSQL(const TableType: TIndexTable; UseParams: boolean = True): UTF8String; virtual;
     procedure FinishCreateTable(const TableType: TIndexTable); virtual;
+    procedure FinishDropTable(const TableType: TIndexTable); virtual;
   protected
     class function AllowForeignKeyInTable: boolean; virtual;
-    procedure Execute(const sql: string; ignoreErrors: boolean = True); virtual; abstract;
-    function GetURLID(const URL: string; ATimeStamp: TDateTime; ALanguageID: int64; DoCreate: boolean = True): int64; virtual; abstract;
+    procedure Execute(const sql: UTF8String; ignoreErrors: boolean = True); virtual; abstract;
+    function GetURLID(const URL: UTF8String; ATimeStamp: TDateTime; ALanguageID: int64; DoCreate: boolean = True): int64; virtual; abstract;
   public
     procedure CreateIndexerTables; override;
-    procedure DeleteWordsFromFile(URL: string); override;
+    procedure DeleteWordsFromFile(URL: UTF8String); override;
   end;
 
   TCustomFileReader = class;
@@ -181,28 +180,30 @@ type
 
   TCustomFileReader = class
   private
+    FCodePage: TSystemCodePage;
     FCount: integer;
     FDetectLanguage: boolean;
     FIgnoreNumeric: boolean;
-    FLanguage: string;
+    FLanguage: UTF8String;
     FOnAdd: TOnSearchWordEvent;
     FSearchWord: array of TSearchWordData;
     FStream: TStream;
     FStreamPos: integer;
+    FURL: UTF8String;
     FUseIgnoreList: boolean;
     FIgnoreListDef: TIgnoreListDef;
     FNoListFound: boolean;
     FTokenStartPos: integer;
-    FContext: string;
+    FContext: UTF8String;
     function GetCapacity: integer;
     function GetSearchWord(index: integer): TSearchWordData;
     procedure SetCapacity(AValue: integer);
     procedure SetStream(AValue: TStream);
     procedure SetStreamPos(AValue: integer);
   protected
-    function AllowedToken(token: string): boolean; virtual;
-    function GetToken: string; virtual;
-    function GetContext: string;
+    function AllowedToken(token: UTF8String): boolean; virtual;
+    function GetToken: UTF8String; virtual;
+    function GetContext: UTF8String;
     function AllowWord(var ASearchWord: TSearchWordData): boolean;
     procedure Add(var ASearchWord: TSearchWordData);
     procedure DoDetectLanguage;
@@ -210,10 +211,10 @@ type
     property StreamPos: integer read FStreamPos write SetStreamPos;
     property TokenStartPos: integer read FTokenStartPos;
   public
-    constructor Create;
-    destructor Destroy; override;
+    Constructor Create(Const aURL : UTF8String; aCodePage : TSystemCodePage); virtual;
+    Destructor Destroy; override;
     property DetectLanguage: boolean read FDetectLanguage write FDetectLanguage;
-    property Language: string read FLanguage write FLanguage;
+    property Language: UTF8String read FLanguage write FLanguage;
     procedure LoadFromStream(FileStream: TStream); virtual;
     property SearchWord[index: integer]: TSearchWordData read GetSearchWord;
     property Count: integer read FCount;
@@ -221,6 +222,8 @@ type
     property OnAddSearchWord: TOnSearchWordEvent read FOnAdd write FOnAdd;
     property UseIgnoreList: boolean read FUseIgnoreList write FUseIgnoreList;
     property IgnoreNumeric: boolean read FIgnoreNumeric write FIgnoreNumeric;
+    Property CodePage : TSystemCodePage Read FCodePage Write FCodePage;
+    Property URL : UTF8String Read FURL Write FURL;
   end;
 
   TCustomFileReaderClass = class of TCustomFileReader;
@@ -229,78 +232,81 @@ type
   TAddWordStub = class(TObject)
   private
     FCount: int64;
-    FURL: string;
+    FURL: UTF8String;
     FDateTime: TDateTime;
     FDatabase: TCustomIndexDB;
   public
-    constructor Create(const AURL: string; const ADateTime: TDateTime; ADatabase: TCustomIndexDB);
+    constructor Create(const AURL: UTF8String; const ADateTime: TDateTime; ADatabase: TCustomIndexDB);
     procedure DoAddWord(AReader: TCustomFileReader; var AWord: TSearchWordData); virtual;
     property Count: int64 read FCount;
   end;
 
   { TFPIndexer }
 
-  TIndexProgressEvent = procedure(Sender: TObject; const ACurrent, ACount: integer; const AURL: string) of object;
+  TIndexProgressEvent = procedure(Sender: TObject; const ACurrent, ACount: integer; const AURL: UTF8String) of object;
 
   TFPIndexer = class(TComponent)
   private
+    FCodePage: TSystemCodePage;
     FCommitFiles: boolean;
     FDatabase: TCustomIndexDB;
     FDetectLanguage: boolean;
     FErrorCount: int64;
-    FExcludeFileMask: string;
-    FFileMask: string;
+    FExcludeFileMask: UTF8String;
+    FFileMask: UTF8String;
     FIgnoreNumeric: boolean;
-    FLanguage: string;
+    FLanguage: UTF8String;
     FOnProgress: TIndexProgressEvent;
-    FSearchPath: string;
+    FSearchPath: UTF8String;
     FSearchRecursive: boolean;
     FUseIgnoreList: boolean;
     ExcludeMaskPatternList: TStrings;
     MaskPatternList: TStrings;
     procedure SetDatabase(AValue: TCustomIndexDB);
-    procedure SetExcludeFileMask(AValue: string);
-    procedure SetFileMask(AValue: string);
-    procedure SetSearchPath(AValue: string);
+    procedure SetExcludeFileMask(AValue: UTF8String);
+    procedure SetFileMask(AValue: UTF8String);
+    procedure SetSearchPath(AValue: UTF8String);
   protected
-    procedure DoProgress(const ACurrent, ACount: integer; const URL: string); virtual;
-    procedure SearchFiles(const PathName, FileName: string; const Recursive: boolean; AList: TStrings); virtual;
-    procedure ExcludeFiles(const ExcludeMask: string; AList: TStrings); virtual;
+    function DoCodePageConversion(aCodePage: TSystemCodePage; S: TStream): TStream; virtual;
+    function DoIndexStream(const AURL: UTF8String; ADateTime: TDateTime; S: TStream; Reader: TCustomFileReader): int64; virtual;
+    procedure DoProgress(const ACurrent, ACount: integer; const URL: UTF8String); virtual;
+    procedure SearchFiles(const PathName, FileName: UTF8String; const Recursive: boolean; AList: TStrings); virtual;
+    procedure ExcludeFiles(const ExcludeMask: UTF8String; AList: TStrings); virtual;
   public
-    constructor Create(AOwner: TComponent); overload;
+    constructor Create(AOwner: TComponent); override;
     destructor Destroy; override;
-
-    function IndexStream(const AURL: string; ADateTime: TDateTime; S: TStream; Reader: TCustomFileReader): int64; virtual;
-    function IndexFile(AURL: string; AllowErrors: boolean; const ALanguage: string = ''): int64;
+    function IndexStream(const AURL: UTF8String; ADateTime: TDateTime; S: TStream; Reader: TCustomFileReader): int64;
+    function IndexFile(AURL: UTF8String; AllowErrors: boolean; const ALanguage: UTF8String = ''): int64;
     function Execute(AllowErrors: boolean): int64;
     property ErrorCount: int64 read FErrorCount;
-    property Language: string read FLanguage write FLanguage;
+  published
+    property Language: UTF8String read FLanguage write FLanguage;
     property OnProgress: TIndexProgressEvent read FOnProgress write FOnProgress;
     property UseIgnoreList: boolean read FUseIgnoreList write FUseIgnoreList;
     property IgnoreNumeric: boolean read FIgnoreNumeric write FIgnoreNumeric;
     property CommitFiles: boolean read FCommitFiles write FCommitFiles;
-  published
     property Database: TCustomIndexDB read FDatabase write SetDatabase;
-    property ExcludeFileMask: string read FExcludeFileMask write SetExcludeFileMask;
-    property FileMask: string read FFileMask write SetFileMask;
-    property SearchPath: string read FSearchPath write SetSearchPath;
+    property ExcludeFileMask: UTF8String read FExcludeFileMask write SetExcludeFileMask;
+    property FileMask: UTF8String read FFileMask write SetFileMask;
+    property SearchPath: UTF8String read FSearchPath write SetSearchPath;
     property SearchRecursive: boolean read FSearchRecursive write FSearchRecursive;
     property DetectLanguage: boolean read FDetectLanguage write FDetectLanguage;
+    Property CodePage : TSystemCodePage Read FCodePage Write FCodePage;
   end;
 
   { TFileReaderDef }
 
   TFileReaderDef = class(TCollectionItem)
   private
-    FExtensions, FTypeName, FDefaultExt: string;
+    FExtensions, FTypeName, FDefaultExt: UTF8String;
     FReader: TCustomFileReaderClass;
   public
-    function HandlesExtension(const Ext: string): boolean; virtual;
-    function CreateReader(const AURL: string): TCustomFileReader; virtual;
+    function HandlesExtension(const Ext: UTF8String): boolean; virtual;
+    function CreateReader(const AURL: UTF8String; aCodePage: TSystemCodePage): TCustomFileReader; virtual;
     procedure DisposeReader(AReader: TCustomFileReader); virtual;
-    property Extensions: string read FExtensions write FExtensions;
-    property TypeName: string read FTypeName write FTypeName;
-    property DefaultExt: string read FDefaultExt write FDefaultExt;
+    property Extensions: UTF8String read FExtensions write FExtensions;
+    property TypeName: UTF8String read FTypeName write FTypeName;
+    property DefaultExt: UTF8String read FDefaultExt write FDefaultExt;
     property Reader: TCustomFileReaderClass read FReader write FReader;
   end;
 
@@ -311,8 +317,8 @@ type
     function GetD(AIndex: integer): TFileReaderDef;
     procedure SetD(AIndex: integer; AValue: TFileReaderDef);
   public
-    function AddFileReader(const ATypeName: string): TFileReaderDef;
-    function IndexOfTypeName(const ATypeName: string): integer;
+    function AddFileReader(const ATypeName: UTF8String): TFileReaderDef;
+    function IndexOfTypeName(const ATypeName: UTF8String): integer;
     property Defs[AIndex: integer]: TFileReaderDef read GetD write SetD; default;
   end;
 
@@ -322,24 +328,24 @@ type
   private
     FData: TFileReaderDefs;
     function GetCount: integer;
-    function GetData(const ATypeName: string): TFileReaderDef;
+    function GetData(const ATypeName: UTF8String): TFileReaderDef;
     function GetData(index: integer): TFileReaderDef;
-    function GetDefExt(const TypeName: string): string;
-    function GetExt(const TypeName: string): string;
-    function GetReader(const TypeName: string): TCustomFileReaderClass;
-    function GetTypeName(index: integer): string;
+    function GetDefExt(const TypeName: UTF8String): UTF8String;
+    function GetExt(const TypeName: UTF8String): UTF8String;
+    function GetReader(const TypeName: UTF8String): TCustomFileReaderClass;
+    function GetTypeName(index: integer): UTF8String;
   protected
     function CreateFileReaderDefs: TFileReaderDefs; virtual;
   public
     constructor Create;
     destructor Destroy; override;
-    function GetDefsForExtension(const Extension: string; List: TStrings): integer;
-    procedure RegisterFileReader(const ATypeName, TheExtensions: string; AReader: TCustomFileReaderClass);
+    function GetDefsForExtension(const Extension: UTF8String; List: TStrings): integer;
+    procedure RegisterFileReader(const ATypeName, TheExtensions: UTF8String; AReader: TCustomFileReaderClass);
     property Count: integer read GetCount;
-    property DefaultExtension[const TypeName: string]: string read GetDefExt;
-    property Extensions[const TypeName: string]: string read GetExt;
-    property FileReader[const TypeName: string]: TCustomFileReaderClass read GetReader;
-    property TypeNames[index: integer]: string read GetTypeName;
+    property DefaultExtension[const TypeName: UTF8String]: UTF8String read GetDefExt;
+    property Extensions[const TypeName: UTF8String]: UTF8String read GetExt;
+    property FileReader[const TypeName: UTF8String]: TCustomFileReaderClass read GetReader;
+    property TypeNames[index: integer]: UTF8String read GetTypeName;
   end;
 
   { TFPSearch }
@@ -368,7 +374,7 @@ type
     property RankedCount: integer read FRankedCount write SetRankedCount;
     property Results[index: integer]: TSearchWordData read GetResults;
     property RankedResults[index: integer]: TSearchWordData read GetRankedResults;
-    procedure SetSearchWord(AValue: string);
+    procedure SetSearchWord(AValue: UTF8String);
   published
     property Database: TCustomIndexDB read FDatabase write SetDatabase;
     property Options: TSearchOptions read FOptions write FOptions;
@@ -379,7 +385,7 @@ type
 
   TIgnoreListDef = class(TCollectionItem)
   private
-    FLanguage: string;
+    FLanguage: UTF8String;
     FList: TStrings;
     procedure SetStrings(AValue: TStrings);
   public
@@ -387,9 +393,9 @@ type
     destructor Destroy; override;
     procedure BeginLoading;
     procedure EndLoading;
-    function IgnoreWord(const AWord: string): boolean;
+    function IgnoreWord(const AWord: UTF8String): boolean;
     property List: TStrings read FList write SetStrings;
-    property Language: string read FLanguage write FLanguage;
+    property Language: UTF8String read FLanguage write FLanguage;
   end;
 
   { TIgnoreLists }
@@ -399,10 +405,10 @@ type
     function getL(AIndex: integer): TIgnoreListDef;
     procedure SetL(AIndex: integer; AValue: TIgnoreListDef);
   public
-    function IndexOfLanguage(const ALanguage: string): integer;
-    function FindLanguage(const ALanguage: string): TIgnoreListDef;
-    function LanguageByName(const ALanguage: string): TIgnoreListDef;
-    function AddLanguage(const ALanguage: string): TIgnoreListDef;
+    function IndexOfLanguage(const ALanguage: UTF8String): integer;
+    function FindLanguage(const ALanguage: UTF8String): TIgnoreListDef;
+    function LanguageByName(const ALanguage: UTF8String): TIgnoreListDef;
+    function AddLanguage(const ALanguage: UTF8String): TIgnoreListDef;
     property Lists[AIndex: integer]: TIgnoreListDef read getL write SetL; default;
   end;
 
@@ -414,8 +420,8 @@ type
   public
     constructor Create(AOwner: TComponent); override;
     destructor Destroy; override;
-    procedure RegisterIgnoreWords(const ALanguage: string; AList: TStrings);
-    procedure LoadIgnoreWordsFromFile(const ALanguage, AFileName: string);
+    procedure RegisterIgnoreWords(const ALanguage: UTF8String; AList: TStrings);
+    procedure LoadIgnoreWordsFromFile(const ALanguage, AFileName: UTF8String);
     property Lists: TIgnoreLists read FLists;
   end;
 
@@ -425,10 +431,10 @@ var
   FileHandlers: TFileHandlersManager;
   IgnoreListManager: TIgnoreListManager;
 
-function DateToISO8601(DateTime: TDateTime): string;
-function ISO8601ToDate(DateTime: string): TDateTime;
+function DateToISO8601(DateTime: TDateTime): UTF8String;
+function ISO8601ToDate(DateTime: UTF8String): TDateTime;
 
-function QuoteString(S: string): string;
+function QuoteString(S: UTF8String): UTF8String;
 
 implementation
 
@@ -441,13 +447,13 @@ uses
 resourcestring
   SErrNoSuchLanguage = 'Unknown language : "%s".';
 
-function DateToISO8601(DateTime: TDateTime): string;
+function DateToISO8601(DateTime: TDateTime): UTF8String;
 begin
   Result := FormatDateTime('yyyy-mm-dd', DateTime) + 'T' +
             FormatDateTime('hh:mm:ss', DateTime)
 end;
 
-function ISO8601ToDate(DateTime: string): TDateTime;
+function ISO8601ToDate(DateTime: UTF8String): TDateTime;
 begin
   Result := EncodeDate(StrToInt(copy(DateTime, 1, 4)),
                        StrToInt(copy(DateTime, 6, 2)),
@@ -458,12 +464,12 @@ begin
                        0);
 end;
 
-function QuoteString(S: string): string;
+function QuoteString(S: UTF8String): UTF8String;
 begin
   Result := '''' + S + '''';
 end;
 
-function CalcDefExt(TheExtensions: string): string;
+function CalcDefExt(TheExtensions: UTF8String): UTF8String;
 var
   p: integer;
 begin
@@ -488,7 +494,7 @@ begin
   inherited Destroy;
 end;
 
-procedure TIgnoreListManager.RegisterIgnoreWords(const ALanguage: string; AList: TStrings);
+procedure TIgnoreListManager.RegisterIgnoreWords(const ALanguage: UTF8String; AList: TStrings);
 
 var
   L: TIgnoreListDef;
@@ -507,7 +513,7 @@ begin
   end;
 end;
 
-procedure TIgnoreListManager.LoadIgnoreWordsFromFile(const ALanguage, AFileName: string);
+procedure TIgnoreListManager.LoadIgnoreWordsFromFile(const ALanguage, AFileName: UTF8String);
 
 var
   L: TStringList;
@@ -534,14 +540,14 @@ begin
   Items[AIndex] := AValue;
 end;
 
-function TIgnoreLists.IndexOfLanguage(const ALanguage: string): integer;
+function TIgnoreLists.IndexOfLanguage(const ALanguage: UTF8String): integer;
 begin
   Result := Count - 1;
   while (Result >= 0) and (CompareText(ALanguage, GetL(Result).Language) <> 0) do
     Dec(Result);
 end;
 
-function TIgnoreLists.FindLanguage(const ALanguage: string): TIgnoreListDef;
+function TIgnoreLists.FindLanguage(const ALanguage: UTF8String): TIgnoreListDef;
 
 var
   i: integer;
@@ -554,14 +560,14 @@ begin
     Result := GetL(I);
 end;
 
-function TIgnoreLists.LanguageByName(const ALanguage: string): TIgnoreListDef;
+function TIgnoreLists.LanguageByName(const ALanguage: UTF8String): TIgnoreListDef;
 begin
   Result := FindLanguage(Alanguage);
   if (Result = nil) then
     raise EFPIndexer.CreateFmt(SErrNoSuchLanguage, [ALanguage]);
 end;
 
-function TIgnoreLists.AddLanguage(const ALanguage: string): TIgnoreListDef;
+function TIgnoreLists.AddLanguage(const ALanguage: UTF8String): TIgnoreListDef;
 begin
   Result := Add as TIgnoreListDef;
   Result.Language := ALanguage;
@@ -598,7 +604,7 @@ begin
   TStringList(FList).Sorted := True;
 end;
 
-function TIgnoreListDef.IgnoreWord(const AWord: string): boolean;
+function TIgnoreListDef.IgnoreWord(const AWord: UTF8String): boolean;
 begin
   Result := FList.IndexOf(AWord) <> -1;
 end;
@@ -610,14 +616,14 @@ begin
   // Do nothing
 end;
 
-function TFileReaderDef.HandlesExtension(const Ext: string): boolean;
+function TFileReaderDef.HandlesExtension(const Ext: UTF8String): boolean;
 begin
   Result := Pos(lowercase(ext) + ';', FExtensions + ';') <> 0;
 end;
 
-function TFileReaderDef.CreateReader(const AURL: string): TCustomFileReader;
+function TFileReaderDef.CreateReader(const AURL: UTF8String; aCodePage : TSystemCodePage): TCustomFileReader;
 begin
-  Result := FReader.Create;
+  Result := FReader.Create(aURL,aCodePage);
 end;
 
 procedure TFileReaderDef.DisposeReader(AReader: TCustomFileReader);
@@ -637,13 +643,13 @@ begin
   Items[AIndex] := AValue;
 end;
 
-function TFileReaderDefs.AddFileReader(const ATypeName: string): TFileReaderDef;
+function TFileReaderDefs.AddFileReader(const ATypeName: UTF8String): TFileReaderDef;
 begin
   Result := Add as TFileReaderDef;
   Result.FTypeName := ATypeName;
 end;
 
-function TFileReaderDefs.IndexOfTypeName(const ATypeName: string): integer;
+function TFileReaderDefs.IndexOfTypeName(const ATypeName: UTF8String): integer;
 begin
   Result := Count - 1;
   while (Result >= 0) and (CompareText(AtypeName, GetD(Result).TypeName) <> 0) do
@@ -653,7 +659,7 @@ end;
 
 { TWordParser }
 
-procedure TWordParser.AddToken(AValue: string; ATokenType: TWordTokenType);
+procedure TWordParser.AddToken(AValue: UTF8String; ATokenType: TWordTokenType);
 begin
   Count := Count + 1;
 
@@ -669,10 +675,12 @@ begin
   WordList[FCount - 1].TokenType := ATokenType;
 end;
 
-function TWordParser.GetSearchWordQuery: string;
+function TWordParser.GetSearchWordQuery: UTF8String;
+
 var
-  s: string;
+  s: UTF8String;
   i: integer;
+
 begin
   s := '';
   for i := 0 to FCount - 1 do
@@ -680,16 +688,15 @@ begin
       s := S+WordList[i].Value
     else
       s := S+WordList[i].Value + ' ';
-
   //replace wildcard '*' with the SQL variant '%'
   Result := StringReplace(s, '*', WildCardChar, [rfReplaceAll, rfIgnoreCase]);
 end;
 
 function TWordParser.GetToken(index: integer): TWordToken;
 begin
+  Result:=Default(TWordToken);
   if (index >= 0) and (index < FCount) then
     Result := WordList[index];
-
   Result.Value := StringReplace(Result.Value, '*', WildCardChar, [rfReplaceAll, rfIgnoreCase]);
 end;
 
@@ -698,28 +705,22 @@ begin
   if FCount = AValue then
     Exit;
   FCount := AValue;
-
   SetLength(WordList, FCount);
 end;
 
-constructor TWordParser.Create(ASearchWords: string);
+constructor TWordParser.Create(ASearchWords: UTF8String);
 var
   list: TStringList;
   i: integer;
 begin
   //erase list
   FCount := 0;
-
   FWildCardChar := '%';
-
   list := TStringList.Create;
-
   try
     list.Delimiter := ' ';
     list.StrictDelimiter := True;
-
     list.DelimitedText := LowerCase(ASearchWords);
-
     //create the search clause
     for i := 0 to list.Count - 1 do
     begin
@@ -799,7 +800,6 @@ procedure TFPSearch.SetRankedCount(AValue: integer);
 begin
   if FRankedCount = AValue then
     Exit;
-
   FRankedCount := AValue;
   SetLength(RankedList, AValue);
 end;
@@ -812,15 +812,13 @@ begin
     FCount := index;
     SetLength(ResultList, FCount + 1);
   end;
-
   ResultList[index] := AValue;
 end;
 
-procedure TFPSearch.SetSearchWord(AValue: string);
+procedure TFPSearch.SetSearchWord(AValue: UTF8String);
 begin
   if Assigned(FSearchWord) then
     FreeAndNil(FSearchWord);
-
   FSearchWord := TWordParser.Create(AValue);
   FSearchWord.WildCardChar := '%';   //should come from DataBase
 end;
@@ -838,7 +836,6 @@ end;
 constructor TFPSearch.Create(AOwner: TComponent);
 begin
   inherited Create(AOwner);
-
   FCount := 0;
   FRankedCount := 0;
 end;
@@ -851,14 +848,12 @@ end;
 function TFPSearch.Execute: int64;
 begin
   Result := 0;
-
   //reset previous searches
   FCount := 0;
   SetLength(ResultList, FCount);
   Database.Connect;
   Database.FindSearchData(SearchWord, Self, Options);
   Result := Count;
-
   //rank the results
   RankResults;
 end;
@@ -881,17 +876,23 @@ begin
 end;
 
 //a very basic tokenizer that only returns numeric and alphanumeric characters
-function TCustomFileReader.GetToken: string;
+function TCustomFileReader.GetToken: UTF8String;
+
+Const
+  NonChars = #9#32#10#13#12'|&@#"''(§^!{})-[]*%`=+/.;:,?';
+  NonCharset = [#9,#32,#10,#13,#12,'|','&','@','#',
+               '"','''','(','^','!','{','}',')',
+               '-','[',']','*','%','`','=','+','/',
+               '.',';',':',',','?',
+               '0'..'9'];
 
 var
-  s: string;
+  s: UTF8String;
   c: char;
-begin
-  Result := '';
 
+begin
   if not Assigned(Stream) then
     exit;
-
   try
     //writeln('pos:', Stream.Position, ' size:', Stream.Size);
     if Stream.Position >= Stream.Size - 1 then
@@ -902,13 +903,13 @@ begin
       exit;
 
     //eat all invalid characters
-    while not (C in ['a'..'z', 'A'..'Z', '0'..'9']) and (Stream.Position < Stream.Size) do
+    while (C in NonCharSet) and (Stream.Position < Stream.Size) do
       c := Chr(Stream.ReadByte);
     S := c;
     //now read all valid characters from stream and append
     FTokenStartPos := Stream.Position;
     c := Chr(Stream.ReadByte);
-    while (c in ['a'..'z', 'A'..'Z', '0'..'9']) and (Stream.Position < Stream.Size) do
+    while (Not (c in NonCharSet)) and (Stream.Position < Stream.Size) do
     begin
       s :=S+ c;
       c := chr(Stream.ReadByte);
@@ -921,7 +922,7 @@ begin
   end;
 end;
 
-function TCustomFileReader.GetContext: string;
+function TCustomFileReader.GetContext: UTF8String;
 begin
   Result := FContext;
 end;
@@ -952,7 +953,7 @@ begin
   end;
 end;
 
-function TCustomFileReader.AllowedToken(token: string): boolean;
+function TCustomFileReader.AllowedToken(token: UTF8String): boolean;
 begin
   Result := True;
 end;
@@ -1002,7 +1003,7 @@ procedure TCustomFileReader.DoDetectLanguage;
 var
   tc: TFPTextCat;
   i: integer;
-  s: string = '';
+  s: UTF8String = '';
 {$endif}
 begin
 {$ifdef LangDetect}
@@ -1020,11 +1021,14 @@ begin
 {$endif}
 end;
 
-constructor TCustomFileReader.Create;
+
+constructor TCustomFileReader.Create(const aURL: UTF8String; aCodePage: TSystemCodePage);
 begin
   FCount := 0;
   FLanguage := 'unknown';
   FignoreNumeric := True;
+  FCodePage:=aCodePage;
+  FURL:=aURL;
 end;
 
 destructor TCustomFileReader.Destroy;
@@ -1043,7 +1047,7 @@ end;
 
 { TFileHandlersManager }
 
-function TFileHandlersManager.GetReader(const TypeName: string): TCustomFileReaderClass;
+function TFileHandlersManager.GetReader(const TypeName: UTF8String): TCustomFileReaderClass;
 var
   ih: TFileReaderDef;
 begin
@@ -1054,7 +1058,7 @@ begin
     Result := nil;
 end;
 
-function TFileHandlersManager.GetExt(const TypeName: string): string;
+function TFileHandlersManager.GetExt(const TypeName: UTF8String): UTF8String;
 
 var
   ih: TFileReaderDef;
@@ -1066,7 +1070,7 @@ begin
     Result := '';
 end;
 
-function TFileHandlersManager.GetDefExt(const TypeName: string): string;
+function TFileHandlersManager.GetDefExt(const TypeName: UTF8String): UTF8String;
 var
   ih: TFileReaderDef;
 begin
@@ -1077,7 +1081,7 @@ begin
     Result := '';
 end;
 
-function TFileHandlersManager.GetTypeName(index: integer): string;
+function TFileHandlersManager.GetTypeName(index: integer): UTF8String;
 var
   ih: TFileReaderDef;
 begin
@@ -1085,7 +1089,7 @@ begin
   Result := ih.FTypeName;
 end;
 
-function TFileHandlersManager.GetData(const ATypeName: string): TFileReaderDef;
+function TFileHandlersManager.GetData(const ATypeName: UTF8String): TFileReaderDef;
 var
   r: integer;
 begin
@@ -1126,21 +1130,27 @@ begin
   inherited Destroy;
 end;
 
-{$note function result is not set, convert to procedure?}
-function TFileHandlersManager.GetDefsForExtension(const Extension: string; List: TStrings): integer;
+
+function TFileHandlersManager.GetDefsForExtension(const Extension: UTF8String; List: TStrings): integer;
+
 var
   I: integer;
   D: TFileReaderDef;
+
 begin
+  Result:=0;
   for I := 0 to FData.Count - 1 do
   begin
     D := FData[i];
     if D.HandlesExtension(Extension) and (D.Reader <> nil) then
+      begin
       List.AddObject(D.TypeName, D);
+      Inc(Result);
+      end;
   end;
 end;
 
-procedure TFileHandlersManager.RegisterFileReader(const ATypeName, TheExtensions: string; AReader: TCustomFileReaderClass);
+procedure TFileHandlersManager.RegisterFileReader(const ATypeName, TheExtensions: UTF8String; AReader: TCustomFileReaderClass);
 var
   ih: TFileReaderDef;
 begin
@@ -1164,7 +1174,7 @@ end;
 
 { TAddWordStub }
 
-constructor TAddWordStub.Create(const AURL: string; const ADateTime: TDateTime; ADatabase: TCustomIndexDB);
+constructor TAddWordStub.Create(const AURL: UTF8String; const ADateTime: TDateTime; ADatabase: TCustomIndexDB);
 begin
   FURL := AURl;
   FDateTime := ADateTime;
@@ -1192,7 +1202,7 @@ begin
   FDatabase := AValue;
 end;
 
-procedure TFPIndexer.SetExcludeFileMask(AValue: string);
+procedure TFPIndexer.SetExcludeFileMask(AValue: UTF8String);
 begin
   if FExcludeFileMask = AValue then
     exit;
@@ -1201,10 +1211,10 @@ begin
   ExcludeMaskPatternList.DelimitedText := FExcludeFileMask;
 end;
 
-procedure TFPIndexer.SearchFiles(const PathName, FileName: string; const Recursive: boolean; AList: TStrings);
+procedure TFPIndexer.SearchFiles(const PathName, FileName: UTF8String; const Recursive: boolean; AList: TStrings);
 var
   Rec: TSearchRec;
-  Path: string;
+  Path: UTF8String;
 begin
   Path := IncludeTrailingBackslash(PathName);
   try
@@ -1228,7 +1238,7 @@ begin
     end;
 end;
 
-procedure TFPIndexer.ExcludeFiles(const ExcludeMask: string; AList: TStrings);
+procedure TFPIndexer.ExcludeFiles(const ExcludeMask: UTF8String; AList: TStrings);
 var
   i: integer;
 begin
@@ -1237,7 +1247,7 @@ begin
       AList.Delete(i);
 end;
 
-procedure TFPIndexer.SetFileMask(AValue: string);
+procedure TFPIndexer.SetFileMask(AValue: UTF8String);
 begin
   if FFileMask = AValue then
     exit;
@@ -1246,14 +1256,14 @@ begin
   MaskPatternList.DelimitedText := FFileMask;
 end;
 
-procedure TFPIndexer.SetSearchPath(AValue: string);
+procedure TFPIndexer.SetSearchPath(AValue: UTF8String);
 begin
   if FSearchPath = AValue then
     exit;
   FSearchPath := ExtractFilePath(ExpandFileName(IncludeTrailingPathDelimiter(AValue)));
 end;
 
-procedure TFPIndexer.DoProgress(const ACurrent, ACount: integer; const URL: string);
+procedure TFPIndexer.DoProgress(const ACurrent, ACount: integer; const URL: UTF8String);
 begin
   if Assigned(FOnProgress) then
     FOnProgress(Self, ACurrent, ACount, URL);
@@ -1286,14 +1296,14 @@ begin
   inherited Destroy;
 end;
 
-function TFPIndexer.IndexStream(const AURL: string; ADateTime: TDateTime; S: TStream; Reader: TCustomFileReader): int64;
-var
+function TFPIndexer.DoIndexStream(const AURL: UTF8String; ADateTime: TDateTime; S: TStream; Reader: TCustomFileReader): int64;
+
+Var
   i: integer;
   Stub: TAddWordStub;
   AWord: TSearchWordData;
+
 begin
-  Result := 0;
-  DataBase.DeleteWordsFromFile(AURL);
   // If reader must detect language, the stub cannot be used.
   if not DetectLanguage then
   begin
@@ -1320,22 +1330,60 @@ begin
       Inc(Result);
     end;
   end;
-  if CommitFiles then
-  begin
-    Database.CommitTrans;
-    Database.BeginTrans;
+end;
+
+function TFPIndexer.DoCodePageConversion(aCodePage : TSystemCodePage;S: TStream) : TStream;
+
+Var
+  R : RawByteString;
+
+begin
+  R:='';
+  if (aCodePage=CP_UTF8) then
+    Result:=S
+  else
+    begin
+    SetLength(R,S.Size);
+    SetCodePage(R,aCodePage,false);
+    S.ReadBuffer(R[1],S.Size);
+    SetCodePage(R,CP_UTF8,True);
+    Result:=TStringStream.Create(R,CP_UTF8);
+    end;
+end;
+
+function TFPIndexer.IndexStream(const AURL: UTF8String; ADateTime: TDateTime; S: TStream; Reader: TCustomFileReader): int64;
+
+var
+  CS : TStream;
+
+begin
+  Result := 0;
+  CS:=DoCodePageConversion(Reader.CodePage,S);
+  try
+    DataBase.DeleteWordsFromFile(AURL);
+    Result:=DoIndexStream(aURL,aDateTime,CS,Reader);
+    if CommitFiles then
+      begin
+      Database.CommitTrans;
+      Database.BeginTrans;
+      end;
+  Finally
+    If (CS<>S) then
+      CS.Free;
   end;
 end;
 
-function TFPIndexer.IndexFile(AURL: string; AllowErrors: boolean; const ALanguage: string): int64;
+function TFPIndexer.IndexFile(AURL: UTF8String; AllowErrors: boolean; const ALanguage: UTF8String): int64;
+
 var
-  e: string;
+  e: UTF8String;
   i: integer;
   d: TFileReaderDef;
   reader: TCustomFileReader;
   fs: TFileStream;
   L: TStringList;
   DT: TDateTime;
+
 begin
   Result := 0;
   if not FileExists(AURL) then
@@ -1350,7 +1398,7 @@ begin
     for I := 0 to L.Count - 1 do
     begin
       d := L.Objects[i] as TFileReaderDef;
-      reader := D.CreateReader(AURL);
+      reader := D.CreateReader(AURL,CodePage);
       try
         try
           Reader.IgnoreNumeric := True;
@@ -1391,7 +1439,7 @@ function TFPIndexer.Execute(AllowErrors: boolean): int64;
 var
   m: integer;
   List: TStringList;
-  url: string;
+  url: UTF8String;
 begin
   Result := 0;
   FErrorCount := 0;
@@ -1440,8 +1488,11 @@ begin
   //create a new database
   BeginTrans;
 
-  for t := low(TIndexTable) to High(TindexTable) do
-    Execute(DropTableSQl(t));
+  for t := high(TIndexTable) downto low(TindexTable) do
+    begin
+    Execute(DropTableSQl(t),True);
+    FinishDropTable(T);
+    end;
   CommitTrans;
 
   BeginTrans;
@@ -1473,41 +1524,42 @@ begin
   CommitTrans;
 end;
 
-function TSQLIndexDB.GetTableName(TableType: TIndexTable): string;
+function TSQLIndexDB.GetTableName(TableType: TIndexTable): UTF8String;
 begin
   Result := DefaultTableNames[TableType];
 end;
 
-function TSQLIndexDB.GetIndexName(IndexType: TIndexIndex): string;
+function TSQLIndexDB.GetIndexName(IndexType: TIndexIndex): UTF8String;
 begin
   Result := DefaultIndexNames[IndexType];
 end;
 
-function TSQLIndexDB.GetFieldName(FieldType: TIndexField): string;
+function TSQLIndexDB.GetFieldName(FieldType: TIndexField): UTF8String;
 begin
   Result := DefaultFieldNames[FieldType];
 end;
 
-function TSQLIndexDB.GetForeignKeyName(ForeignKey: TIndexForeignKey): string;
+function TSQLIndexDB.GetForeignKeyName(ForeignKey: TIndexForeignKey): UTF8String;
 begin
   Result := DefaultForeignKeyNames[ForeignKey];
 end;
 
-function TSQLIndexDB.GetFieldType(FieldType: TIndexField): string;
+function TSQLIndexDB.GetFieldType(FieldType: TIndexField): UTF8String;
 begin
   Result := DefaultFieldTypes[FieldType];
 end;
 
-function TSQLIndexDB.DropTableSQl(TableType: TIndexTable): string;
+function TSQLIndexDB.DropTableSQl(TableType: TIndexTable): UTF8String;
 begin
   Result := 'DROP TABLE ' + GetTableName(TableType);
 end;
 
-function TSQLIndexDB.CreateTableSQL(const TableType: TIndexTable): string;
+function TSQLIndexDB.CreateTableSQL(const TableType: TIndexTable): UTF8String;
 var
   f: TIndexField;
   K: TIndexForeignKey;
 begin
+  Result:='';
   for F := Low(TIndexField) to High(TIndexField) do
     if TableFields[F] = TableType then
     begin
@@ -1526,10 +1578,11 @@ begin
   Result := 'CREATE TABLE ' + GetTableName(TableType) + ' (' + Result + ')';
 end;
 
-function TSQLIndexDB.CreateForeignKey(const ForeignKey: TIndexForeignKey; ForCreate: boolean = False): string;
+function TSQLIndexDB.CreateForeignKey(const ForeignKey: TIndexForeignKey; ForCreate: boolean = False): UTF8String;
 var
-  STN, TTN, FKN, FKF, FTK: string;
+  STN, TTN, FKN, FKF, FTK: UTF8String;
 begin
+  Result:='';
   STN := GetTableName(ForeignKeyTables[ForeignKey]);
   TTN := GetTableName(ForeignKeyTargets[ForeignKey]);
   FKN := GetForeignKeyName(Foreignkey);
@@ -1546,10 +1599,15 @@ begin
   // Do nothing
 end;
 
-function TSQLIndexDB.InsertSQL(const TableType: TIndexTable; UseParams: boolean = True): string;
+procedure TSQLIndexDB.FinishDropTable(const TableType: TIndexTable);
+begin
+  // Do nothing
+end;
+
+function TSQLIndexDB.InsertSQL(const TableType: TIndexTable; UseParams: boolean = True): UTF8String;
 var
-  FL: string = '';
-  VL: string = '';
+  FL: UTF8String = '';
+  VL: UTF8String = '';
   F: TIndexField;
 begin
   for F := Low(TIndexField) to High(TIndexField) do
@@ -1569,10 +1627,11 @@ begin
   Result := Format('INSERT INTO %s (%s) VALUES (%s)', [GetTableName(TableType), FL, VL]);
 end;
 
-function TSQLIndexDB.CreateTableIndex(IndexType: TIndexIndex): string;
+function TSQLIndexDB.CreateTableIndex(IndexType: TIndexIndex): UTF8String;
 var
-  TIN: string;
+  TIN: UTF8String;
 begin
+  Result:='';
   TIN := GetindexName(IndexType);
   case IndexType of
     iiWords: Result := CreateIndexSQL(TIN, GetTableName(itWords), [GetFieldName(ifWordsWord)]);
@@ -1581,7 +1640,7 @@ begin
   end;
 end;
 
-function TSQLIndexDB.CreateIndexSQL(const AIndexName, ATableName: string; const AFieldList: array of string): string;
+function TSQLIndexDB.CreateIndexSQL(const AIndexName, ATableName: UTF8String; const AFieldList: array of UTF8String): UTF8String;
 var
   I: integer;
 begin
@@ -1595,14 +1654,14 @@ begin
   Result := Result + ');';
 end;
 
-function TSQLIndexDB.GetUrlSQL(UseParams: boolean): string;
+function TSQLIndexDB.GetUrlSQL(UseParams: boolean): UTF8String;
 begin
   Result := GetSearchSQL(itFiles, ifFilesID, ifFilesURL, UseParams);
 end;
 
-function TSQLIndexDB.GetSearchSQL(ATable: TIndexTable; IDField, SearchField: TINdexField; UseParams: boolean = True): string;
+function TSQLIndexDB.GetSearchSQL(ATable: TIndexTable; IDField, SearchField: TINdexField; UseParams: boolean = True): UTF8String;
 var
-  IDF, TN, URLF: string;
+  IDF, TN, URLF: UTF8String;
 begin
   TN := GetTableName(ATable);
   IDF := GetFieldName(IDField);
@@ -1615,22 +1674,22 @@ begin
     Result := Result + '%s)';
 end;
 
-function TSQLIndexDB.GetLanguageSQL(UseParams: boolean = True): string;
+function TSQLIndexDB.GetLanguageSQL(UseParams: boolean = True): UTF8String;
 begin
   Result := GetSearchSQL(itLanguages, ifLanguagesID, ifLanguagesName, UseParams);
 end;
 
-function TSQLIndexDB.GetWordSQL(UseParams: boolean = True): string;
+function TSQLIndexDB.GetWordSQL(UseParams: boolean = True): UTF8String;
 begin
   Result := GetSearchSQL(itWords, ifWordsID, ifWordsWord, UseParams);
 end;
 
-function TSQLIndexDB.GetSearchFileSQL(UseParams: boolean = True): string;
+function TSQLIndexDB.GetSearchFileSQL(UseParams: boolean = True): UTF8String;
 begin
   Result := GetSearchSQL(itFiles, ifFilesID, ifFilesURL, UseParams);
 end;
 
-function TSQLIndexDB.DeleteWordsSQL(UseParams: boolean): string;
+function TSQLIndexDB.DeleteWordsSQL(UseParams: boolean): UTF8String;
 begin
   Result := Format('DELETE FROM %s WHERE (%s =', [GetTableName(itMatches), GetFieldName(ifMatchesFileID)]);
   if UseParams then
@@ -1644,9 +1703,9 @@ begin
   Result := False;
 end;
 
-function TSQLIndexDB.GetMatchSQL(SearchOptions: TSearchOptions; SearchWord: TWordParser; UseParams: boolean = True): string;
+function TSQLIndexDB.GetMatchSQL(SearchOptions: TSearchOptions; SearchWord: TWordParser; UseParams: boolean = True): UTF8String;
 var
-  WW, MN, FN, WN, LN: string;
+  WW, MN, FN, WN, LN: UTF8String;
   i: integer;
 begin
   WW := getFieldName(ifWordsWord);
@@ -1683,7 +1742,7 @@ begin
   Result := Result + Format(') ORDER BY %s', [GetFieldName(ifFilesURL)]);
 end;
 
-procedure TSQLIndexDB.DeleteWordsFromFile(URL: string);
+procedure TSQLIndexDB.DeleteWordsFromFile(URL: UTF8String);
 var
   FID: integer;
 begin
diff --git a/packages/fpindexer/src/fpmasks.pp b/packages/fpindexer/src/fpmasks.pp
index f90803a8d9..4cc777969a 100644
--- a/packages/fpindexer/src/fpmasks.pp
+++ b/packages/fpindexer/src/fpmasks.pp
@@ -54,17 +54,17 @@ type
   private
     FMask: TMaskString;
   public
-    constructor Create(const AValue: String);
+    constructor Create(const AValue: UTF8string);
     destructor Destroy; override;
     
-    function Matches(const AFileName: String): Boolean;
+    function Matches(const AFileName: UTF8string): Boolean;
   end;
   
   { TParseStringList }
 
   TParseStringList = class(TStringList)
   public
-    constructor Create(const AText, ASeparators: String);
+    constructor Create(const AText, ASeparators: UTF8string);
   end;
   
   { TMaskList }
@@ -75,21 +75,21 @@ type
     function GetCount: Integer;
     function GetItem(Index: Integer): TMask;
   public
-    constructor Create(const AValue: String; ASeparator: Char = ';');
+    constructor Create(const AValue: UTF8string; ASeparator: Char = ';');
     destructor Destroy; override;
     
-    function Matches(const AFileName: String): Boolean;
+    function Matches(const AFileName: UTF8string): Boolean;
     
     property Count: Integer read GetCount;
     property Items[Index: Integer]: TMask read GetItem;
   end;
 
-function MatchesMask(const FileName, Mask: String): Boolean;
-function MatchesMaskList(const FileName, Mask: String; Separator: Char = ';'): Boolean;
+function MatchesMask(const FileName, Mask: UTF8string): Boolean;
+function MatchesMaskList(const FileName, Mask: UTF8string; Separator: Char = ';'): Boolean;
 
 implementation
 
-function MatchesMask(const FileName, Mask: String): Boolean;
+function MatchesMask(const FileName, Mask: UTF8string): Boolean;
 var
   AMask: TMask;
 begin
@@ -101,7 +101,7 @@ begin
   end;
 end;
 
-function MatchesMaskList(const FileName, Mask: String; Separator: Char): Boolean;
+function MatchesMaskList(const FileName, Mask: UTF8string; Separator: Char): Boolean;
 var
   AMaskList: TMaskList;
 begin
@@ -115,7 +115,7 @@ end;
 
 { TMask }
 
-constructor TMask.Create(const AValue: String);
+constructor TMask.Create(const AValue: UTF8string);
 var
   I: Integer;
   SkipAnyText: Boolean;
@@ -260,10 +260,10 @@ begin
   inherited Destroy;
 end;
 
-function TMask.Matches(const AFileName: String): Boolean;
+function TMask.Matches(const AFileName: UTF8string): Boolean;
 var
   L: Integer;
-  S: String;
+  S: UTF8string;
   
   function MatchToEnd(MaskIndex, CharIndex: Integer): Boolean;
   var
@@ -331,7 +331,7 @@ end;
 
 { TParseStringList }
 
-constructor TParseStringList.Create(const AText, ASeparators: String);
+constructor TParseStringList.Create(const AText, ASeparators: UTF8string);
 var
   I, S: Integer;
 begin
@@ -362,7 +362,7 @@ begin
   Result := FMasks.Count;
 end;
 
-constructor TMaskList.Create(const AValue: String; ASeparator: Char);
+constructor TMaskList.Create(const AValue: UTF8string; ASeparator: Char);
 var
   S: TParseStringList;
   I: Integer;
@@ -385,7 +385,7 @@ begin
   inherited Destroy;
 end;
 
-function TMaskList.Matches(const AFileName: String): Boolean;
+function TMaskList.Matches(const AFileName: UTF8string): Boolean;
 var
   I: Integer;
 begin
diff --git a/packages/fpindexer/src/ireaderhtml.pp b/packages/fpindexer/src/ireaderhtml.pp
index e7e2d58719..513211fc25 100644
--- a/packages/fpindexer/src/ireaderhtml.pp
+++ b/packages/fpindexer/src/ireaderhtml.pp
@@ -28,17 +28,17 @@ type
 
   TIReaderHTML = class(TCustomFileReader)
   private
-    sLine: string;
+    sLine: UTF8String;
     StartPos: integer;
     Offset: integer;
     LinePos: integer;
     Tg, Tx: integer;
     FParser: THTMLParser; //our htmlparser class
-    procedure OnTag(NoCaseTag, ActualTag: string);
-    procedure OnText(Text: string);
+    procedure OnTag(NoCaseTag, ActualTag: String);
+    procedure OnText(Text: String);
   protected
-    function GetToken: string; override;
-    function AllowedToken(token: string): boolean; override;
+    function GetToken: UTF8String; override;
+    function AllowedToken(token: UTF8String): boolean; override;
   public
     procedure LoadFromStream(FileStream: TStream); override;
   end;
@@ -47,13 +47,13 @@ implementation
 
 { TIReaderHTML }
 
-procedure TIReaderHTML.OnTag(NoCaseTag, ActualTag: string);
+procedure TIReaderHTML.OnTag(NoCaseTag, ActualTag: String);
 begin
 end;
 
-procedure TIReaderHTML.OnText(Text: string);
+procedure TIReaderHTML.OnText(Text: String);
 var
-  token: string;
+  token: UTF8String;
   s: TSearchWordData;
   i : Integer;
 begin
@@ -78,10 +78,10 @@ begin
     end;
 end;
 
-function TIReaderHTML.GetToken: string;
+function TIReaderHTML.GetToken: UTF8String;
 var
-  s: string;
-  c: string;
+  s: UTF8String;
+  c: UTF8String;
 begin
   Result := '';
 
@@ -126,7 +126,7 @@ begin
   end;
 end;
 
-function TIReaderHTML.AllowedToken(token: string): boolean;
+function TIReaderHTML.AllowedToken(token: UTF8String): boolean;
 begin
   Result := (Length(token) > 1) and
             (token <> 'nbsp') and (token <> 'quot') and (token <> 'apos') and
diff --git a/packages/fpindexer/src/ireaderpas.pp b/packages/fpindexer/src/ireaderpas.pp
index 8b07216a51..f98f0abbad 100644
--- a/packages/fpindexer/src/ireaderpas.pp
+++ b/packages/fpindexer/src/ireaderpas.pp
@@ -28,7 +28,7 @@ type
   TIReaderPAS = class(TIReaderTXT)
   private
   protected
-    function AllowedToken(token: string): boolean; override;
+    function AllowedToken(token: UTF8String): boolean; override;
   public
     procedure LoadFromStream(FileStream: TStream); override;
   end;
@@ -37,7 +37,7 @@ implementation
 
 { TIReaderPAS }
 
-function TIReaderPAS.AllowedToken(token: string): boolean;
+function TIReaderPAS.AllowedToken(token: UTF8String): boolean;
 begin
   Result:=inherited AllowedToken(token);
 end;
diff --git a/packages/fpindexer/src/ireadertxt.pp b/packages/fpindexer/src/ireadertxt.pp
index cc47e34d4a..31f648a878 100644
--- a/packages/fpindexer/src/ireadertxt.pp
+++ b/packages/fpindexer/src/ireadertxt.pp
@@ -28,7 +28,7 @@ type
   TIReaderTXT = class(TCustomFileReader)
   private
   protected
-    function AllowedToken(token: string): boolean; override;
+    function AllowedToken(token: UTF8String): boolean; override;
   public
     procedure LoadFromStream(FileStream: TStream); override;
   end;
@@ -37,14 +37,14 @@ implementation
 
 { TIReaderTXT }
 
-function TIReaderTXT.AllowedToken(token: string): boolean;
+function TIReaderTXT.AllowedToken(token: UTF8String): boolean;
 begin
   Result := inherited AllowedToken(token) and (Length(token) > 1);
 end;
 
 procedure TIReaderTXT.LoadFromStream(FileStream: TStream);
 var
-  token: string;
+  token: UTF8String;
   p: TSearchWordData;
 begin
   inherited LoadFromStream(FileStream);
diff --git a/packages/fpindexer/src/memindexdb.pp b/packages/fpindexer/src/memindexdb.pp
index 03a820e3aa..a1694a7a7c 100644
--- a/packages/fpindexer/src/memindexdb.pp
+++ b/packages/fpindexer/src/memindexdb.pp
@@ -28,18 +28,18 @@ Type
 
   TDescrItem = Class(TCollectionItem)
   private
-    FDescription: String;
+    FDescription: UTF8string;
     FTheID: Integer;
   Protected
     Function BlockSize : Integer; virtual;
-    Procedure WriteStringToStream(Astream : TStream; S : String);
+    Procedure WriteStringToStream(Astream : TStream; S : UTF8string);
     Procedure WriteToStream(S : TStream); virtual;
     Procedure WriteRefToStream(AStream : Tstream; AItem : TDescrItem);
-    Function ReadStringFromStream(Astream : TStream) :  String;
+    Function ReadStringFromStream(Astream : TStream) :  UTF8string;
     Function ReadFromStream(S : TStream) : Integer; virtual;
   Published
     // The description
-    Property Description : String Read FDescription Write FDescription;
+    Property Description : UTF8string Read FDescription Write FDescription;
     // ID. Not used during work. Only used when loading/saving,
     // it then equals the Index. (index is slow, uses a linear search)
     Property TheID : Integer Read FTheID Write FTheID;
@@ -61,8 +61,8 @@ Type
     Procedure BeginLoading;
     Procedure EndLoading;
     Procedure AllocateIDS;
-    Function FindDescr(Const ADescription : String) : TDescrItem;
-    Function AddDescr(Const ADescription : String) : TDescrItem;
+    Function FindDescr(Const ADescription : UTF8string) : TDescrItem;
+    Function AddDescr(Const ADescription : UTF8string) : TDescrItem;
     Property Descr [AIndex : Integer] : TDescrItem Read GetD Write SetD; default;
   end;
 
@@ -115,8 +115,8 @@ Type
     FWord: TWordItem;
     FWordID : Integer;
     FURLID : Integer;
-    function GetContext: String;
-    procedure SetContext(AValue: String);
+    function GetContext: UTF8string;
+    procedure SetContext(AValue: UTF8string);
     procedure SetURL(AValue: TURLItem);
     procedure SetWord(AValue: TWordItem);
   protected
@@ -128,7 +128,7 @@ Type
     Property Word : TWordItem Read FWord Write SetWord;
     Property URL : TURLItem Read FURL Write SetURL;
     Property Position : Int64 Read FPosition Write FPosition;
-    Property Context : String Read GetContext Write SetContext;
+    Property Context : UTF8string Read GetContext Write SetContext;
   end;
 
   { TMatches }
@@ -151,7 +151,7 @@ Type
     FLanguages : TDescrCollection;
     FWords : TDescrCollection;
     FMatches : TMatches;
-    procedure GetMatches(AWord: String; SearchOptions: TSearchOptions;  AList: TFPList);
+    procedure GetMatches(AWord: UTF8string; SearchOptions: TSearchOptions;  AList: TFPList);
     procedure IntersectMatches(ListA, ListB: TFPList);
     procedure UnionMatches(ListA, ListB: TFPList);
   protected
@@ -166,7 +166,8 @@ Type
     procedure CommitTrans; override;
     procedure BeginTrans; override;
     procedure CompactDB; override;
-    procedure DeleteWordsFromFile(URL: string); override;
+    Procedure CreateDB; override;
+    procedure DeleteWordsFromFile(URL: UTF8string); override;
     procedure AddSearchData(ASearchData: TSearchWordData); override;
     procedure FindSearchData(SearchWord: TWordParser; FPSearch: TFPSearch; SearchOptions: TSearchOptions); override;
     procedure CreateIndexerTables; override;
@@ -177,7 +178,7 @@ Type
 
   TFileIndexDB = Class(TMemIndexDB)
   private
-    FFIleName: String;
+    FFIleName: UTF8string;
     FWriteOnCommit: Boolean;
   Protected
     Procedure LoadFromStream; override;
@@ -186,7 +187,7 @@ Type
     procedure Connect; override;
     procedure DisConnect; override;
     procedure CommitTrans; override;
-    Property FileName : String Read FFIleName Write FFileName;
+    Property FileName : UTF8string Read FFIleName Write FFileName;
     Property WriteOnCommit : Boolean Read FWriteOnCommit Write FWriteOnCommit;
   end;
 
@@ -247,7 +248,7 @@ begin
   Result:=Sizeof(Integer)+Length(FDescription)*SizeOf(Char);
 end;
 
-procedure TDescrItem.WriteStringToStream(Astream: TStream; S: String);
+procedure TDescrItem.WriteStringToStream(Astream: TStream; S: UTF8string);
 Var
   L : Integer;
 begin
@@ -277,7 +278,7 @@ begin
   AStream.WriteBuffer(I,SizeOf(I));
 end;
 
-function TDescrItem.ReadStringFromStream(Astream: TStream): String;
+function TDescrItem.ReadStringFromStream(Astream: TStream): UTF8string;
 Var
   L : Integer;
 begin
@@ -440,6 +441,11 @@ begin
   // Do nothing
 end;
 
+procedure TMemIndexDB.CreateDB;
+begin
+  Clear;
+end;
+
 procedure TMemIndexDB.BeginTrans;
 begin
   // Do nothing
@@ -494,7 +500,7 @@ begin
     end;
 end;
 
-procedure TMemIndexDB.DeleteWordsFromFile(URL: string);
+procedure TMemIndexDB.DeleteWordsFromFile(URL: UTF8string);
 begin
  // inherited DeleteWordsFromFile(URL);
 end;
@@ -563,7 +569,7 @@ begin
     end;
 end;
 
-procedure TMemIndexDB.GetMatches(AWord : String; SearchOptions: TSearchOptions; AList : TFPList);
+procedure TMemIndexDB.GetMatches(AWord : UTF8string; SearchOptions: TSearchOptions; AList : TFPList);
 
   Procedure AddMatches(W : TWordItem);
   Var
@@ -608,7 +614,7 @@ procedure TMemIndexDB.FindSearchData(SearchWord: TWordParser;
 
 Var
   L,W : TFPList;
-  S : String;
+  S : UTF8string;
   I : Integer;
   M : TMatch;
   WD : TSearchWordData;
@@ -685,12 +691,12 @@ begin
     FURL.AddMatch(Self);
 end;
 
-function TMatch.GetContext: String;
+function TMatch.GetContext: UTF8string;
 begin
   Result:=Description;
 end;
 
-procedure TMatch.SetContext(AValue: String);
+procedure TMatch.SetContext(AValue: UTF8string);
 begin
   Description:=AValue;
 end;
@@ -807,7 +813,7 @@ begin
     GetD(i).TheID:=I;
 end;
 
-function TDescrCollection.FindDescr(const ADescription: String): TDescrItem;
+function TDescrCollection.FindDescr(const ADescription: UTF8string): TDescrItem;
 begin
 
   If FHash=Nil then
@@ -816,7 +822,7 @@ begin
     Result:=TDescrItem(FHash.Find(ADescription));
 end;
 
-function TDescrCollection.AddDescr(const ADescription: String): TDescrItem;
+function TDescrCollection.AddDescr(const ADescription: UTF8string): TDescrItem;
 begin
   Result:=Add as TDescrItem;
   Result.Description:=ADescription;
diff --git a/packages/fpindexer/src/pgindexdb.pp b/packages/fpindexer/src/pgindexdb.pp
new file mode 100644
index 0000000000..80ef9f3492
--- /dev/null
+++ b/packages/fpindexer/src/pgindexdb.pp
@@ -0,0 +1,150 @@
+{
+    This file is part of the Free Component Library (FCL)
+    Copyright (c) 2012 by the Free Pascal development team
+
+    SQLDB Firebird based indexer
+    
+    See the file COPYING.FPC, included in this distribution,
+    for details about the copyright.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+
+ **********************************************************************}
+unit pgindexdb;
+
+{$mode objfpc}{$H+}
+
+interface
+
+uses
+  Classes, SysUtils, fpIndexer, sqldbIndexDB ,sqldb, pqconnection;
+
+type
+  { TPGIndexDB }
+
+  TPGIndexDB = class(TSQLDBIndexDB)
+  private
+    FGenQuery: Array[TIndexTable] of TSQLQuery;
+    Function GetGenQuery(ATable : TIndexTable) : TSQLQuery;
+  protected
+    procedure InsertMatch(AWordID, aFileID, aLanguageID: int64; const ASearchData: TSearchWordData); override;
+    function InsertWord(const AWord: UTF8String): int64; override;
+    function InsertURL(const URL: UTF8String; ATimeStamp: TDateTime; ALanguageID: int64): int64; override;
+    function InsertLanguage(const ALanguage: UTF8String): int64; override;
+    function CreateConnection: TSQLConnection; override;
+    function GetID(TableType: TIndexTable): int64;
+    procedure FinishCreateTable(const TableType: TIndexTable); override;
+    procedure FinishDropTable(const TableType: TIndexTable); override;
+  end;
+
+implementation
+
+{ TPGIndexDB }
+
+function TPGIndexDB.GetID(TableType: TIndexTable): int64;
+
+var
+  Q: TSQLQuery;
+
+begin
+  Q := GetGenQuery(TableType);
+  Q.Open;
+  try
+    if (Q.EOF and Q.BOF) then
+      raise Exception.CreateFmt('Could not get ID for table %s', [GetTableName(TableType)]);
+    Result := Q.Fields[0].AsLargeInt;
+  finally
+    Q.Close;
+  end;
+end;
+
+function TPGIndexDB.InsertLanguage(const ALanguage: UTF8String): int64;
+var
+  Q: TSQLQuery;
+begin
+  Result := getID(itLanguages);
+  Q := CreateCachedQuery(cqtInsertLanguage, InsertSQL(itLanguages));
+  Q.ParamByName(GetFieldName(ifLanguagesID)).AsLargeInt := Result;
+  Q.ParamByName(GetFieldName(ifLanguagesName)).AsString := ALanguage;
+  Q.ExecSQL;
+end;
+
+function TPGIndexDB.InsertWord(const AWord: UTF8String): int64;
+var
+  Q: TSQLQuery;
+begin
+  Result := getID(itWords);
+  Q := CreateCachedQuery(cqtInsertWord, InsertSQL(itWords));
+  Q.ParamByName(GetFieldName(ifWordsID)).AsLargeInt := Result;
+  Q.ParamByName(GetFieldName(ifWordsWord)).AsString := AWord;
+  Q.ExecSQL;
+end;
+
+function TPGIndexDB.InsertURL(const URL: UTF8String; ATimeStamp: TDateTime; ALanguageID: int64): int64;
+var
+  Q: TSQLQuery;
+begin
+  Result := getID(itFiles);
+  Q := CreateCachedQuery(cqtInsertFile, InsertSQL(itFiles));
+  Q.ParamByName(GetFieldName(ifFilesID)).AsLargeInt := Result;
+  Q.ParamByName(GetFieldName(ifFilesURL)).AsString := URL;
+  Q.ParamByName(GetFieldName(ifFilesTimeStamp)).AsDateTime := ATimeStamp;
+  Q.ParamByName(GetFieldName(ifFilesLanguageID)).AsLargeInt := ALanguageID;
+  Q.ParamByName(GetFieldName(ifFilesUpdated)).AsInteger := 0;
+  Q.ParamByName(GetFieldName(ifFilesReindex)).AsInteger := 0;
+  Q.ExecSQL;
+end;
+
+procedure TPGIndexDB.InsertMatch(AWordID, aFileID, aLanguageID: int64; const ASearchData: TSearchWordData);
+var
+  Q: TSQLQuery;
+begin
+  Q := CreateCachedQuery(cqtInsertMatch, InsertSQL(itMatches));
+  Q.ParamByName(GetFieldName(ifMatchesID)).AsLargeInt := GetID(itMatches);
+  Q.ParamByName(GetFieldName(ifMatchesLanguageID)).AsLargeInt := aLanguageID;
+  Q.ParamByName(GetFieldName(ifMatchesWordID)).AsLargeInt := aWordID;
+  Q.ParamByName(GetFieldName(ifMatchesFileID)).AsLargeInt := aFileID;
+  Q.ParamByName(GetFieldName(ifMatchesPosition)).AsLargeInt := ASearchData.Position;
+  Q.ParamByName(GetFieldName(ifMatchesContext)).AsString := Copy(ASearchData.Context,1,255);
+  Q.ExecSQL;
+end;
+
+function TPGIndexDB.CreateConnection: TSQLConnection;
+
+Var
+  T : TIndexTable;
+
+begin
+  // So they are recreated
+  for T in TIndexTable do
+    FreeAndNil(FGenQuery[T]);
+  Result := TPQConnection.Create(Self);
+end;
+
+procedure TPGIndexDB.FinishCreateTable(const TableType: TIndexTable);
+begin
+  Execute('CREATE SEQUENCE ' + DefaultGeneratorNames[TableType], True);
+end;
+
+procedure TPGIndexDB.FinishDropTable(const TableType: TIndexTable);
+begin
+  Execute('DROP SEQUENCE ' + DefaultGeneratorNames[TableType], True);
+end;
+
+
+function TPGIndexDB.GetGenQuery(ATable: TIndexTable): TSQLQuery;
+begin
+  If (FGenQuery[ATable]=Nil) then
+    begin
+    FGenQuery[ATable]:=CreateQuery(Format('select nextval(''%s'')', [DefaultGeneratorNames[ATable]]));
+    FGenQuery[ATable].ParseSQL:=False;
+    FGenQuery[ATable].Prepare;
+    end;
+  Result:=FGenQuery[ATable];
+end;
+
+
+end.
+
diff --git a/packages/fpindexer/src/sqldbindexdb.pp b/packages/fpindexer/src/sqldbindexdb.pp
index a812e38f8b..b53bc05dcc 100644
--- a/packages/fpindexer/src/sqldbindexdb.pp
+++ b/packages/fpindexer/src/sqldbindexdb.pp
@@ -26,10 +26,8 @@ type
   TCachedQueryType = (cqtInsertWord, cqtGetWordID, cqtInsertFile, cqtGetFileID,
     cqtInsertLanguage, cqtGetLanguageID, cqtInsertMatch);
 
-// Interbase specific
-const
-  {$note @MvC, TIndexTable is defined as: itWords, itLanguages, itFiles, itMatches, below order seems to be wrong}
-  DefaultGeneratorNames: array[TIndexTable] of string = ('GEN_WORDS', 'GEN_MATCHES', 'GEN_LANGUAGES', 'GEN_FILES');
+Const
+  DefaultGeneratorNames: array[TIndexTable] of string = ('GEN_WORDS','GEN_LANGUAGES', 'GEN_FILES', 'GEN_MATCHES');
 
 type
 
@@ -38,28 +36,32 @@ type
   TSQLDBIndexDB = class(TSQLIndexDB)
   private
     // SQLDB specific
-    db: TSQLConnection;
+    FDB: TSQLConnection;
     FLastURLID: int64;
     FLastURL: string;
     FLastLanguageID: int64;
     FLastLanguage: string;
     FLastWordID: int64;
     FLastWord: string;
+    FProps : Array [0..3] of UTF8String;
     FQueries: array [TCachedQueryType] of TSQLQuery;
+    function GetS(AIndex: integer): UTF8String;
+    procedure SetS(AIndex: integer; const AValue: UTF8String);
+    Procedure EnsureDB;
   protected
     // SQLDB Specific statements
-    procedure Execute(const sql: string; ignoreErrors: boolean = True); override;
-    function GetLanguageID(const ALanguage: string): int64;
-    function GetWordID(const AWord: string): int64;
-    function GetURLID(const URL: string; ATimeStamp: TDateTime; ALanguageID: int64; DoCreate: boolean = True): int64; override;
-    function CreateQuery(const ASQL: string): TSQLQuery;
-    function CreateCachedQuery(QueryType: TCachedQueryType; const ASQL: string): TSQLQuery;
+    procedure Execute(const sql: UTF8string; ignoreErrors: boolean = True); override;
+    function GetLanguageID(const ALanguage: UTF8string): int64;
+    function GetWordID(const AWord: UTF8String): int64;
+    function GetURLID(const URL: UTF8String; ATimeStamp: TDateTime; ALanguageID: int64; DoCreate: boolean = True): int64; override;
+    function CreateQuery(const ASQL: UTF8String): TSQLQuery;
+    function CreateCachedQuery(QueryType: TCachedQueryType; const ASQL: UTF8String): TSQLQuery;
     // Connection specific, need to be overridden
-    function GetConnection: TSQLConnection; virtual; abstract;
+    function CreateConnection: TSQLConnection; virtual; abstract;
     procedure InsertMatch(AWordID, aFileID, aLanguageID: int64; const ASearchData: TSearchWordData); virtual; abstract;
-    function InsertWord(const AWord: string): int64; virtual; abstract;
-    function InsertURL(const URL: string; ATimeStamp: TDateTime; ALanguageID: int64): int64; virtual; abstract;
-    function InsertLanguage(const ALanguage: string): int64; virtual; abstract;
+    function InsertWord(const AWord: UTF8String): int64; virtual; abstract;
+    function InsertURL(const URL: UTF8String; ATimeStamp: TDateTime; ALanguageID: int64): int64; virtual; abstract;
+    function InsertLanguage(const ALanguage: UTF8String): int64; virtual; abstract;
   public
     destructor Destroy; override;
     procedure Connect; override;
@@ -70,14 +72,53 @@ type
     procedure CompactDB; override;
     procedure AddSearchData(ASearchData: TSearchWordData); override;
     procedure FindSearchData(SearchWord: TWordParser; FPSearch: TFPSearch; SearchOptions: TSearchOptions); override;
-    procedure DeleteWordsFromFile(URL: string); override;
+    procedure DeleteWordsFromFile(URL: UTF8String); override;
+    Property NativeConnection : TSQLConnection Read FDB;
+  published
+    property DatabasePath: UTF8String Index 0 read GetS write SetS;
+    property UserName: UTF8String Index 1 read GetS write SetS;
+    property Password: UTF8String Index 2 read GetS write SetS;
+    property HostName : UTF8String Index 3 read GetS write SetS;
   end;
 
 implementation
 
 { TSQLDBIndexDB }
 
-function TSQLDBIndexDB.GetLanguageID(const ALanguage: string): int64;
+function TSQLDBIndexDB.GetS(AIndex: integer): UTF8String;
+begin
+  Result:=FProps[aIndex];
+end;
+
+procedure TSQLDBIndexDB.SetS(AIndex: integer; const AValue: UTF8String);
+begin
+  FProps[aIndex]:=aValue;
+  if Assigned(FDB) then
+    case Aindex of
+      0: FDB.DatabaseName := AValue;
+      1: FDB.UserName := AValue;
+      2: FDB.Password := AValue;
+      3: FDB.HostName := AValue;
+    end;
+end;
+
+procedure TSQLDBIndexDB.EnsureDB;
+begin
+  if FDB=Nil then
+    begin
+    FDB:=CreateConnection;
+    FDB.UserName:=UserName;
+    FDB.Password:=Password;
+    FDB.HostName:=HostName;
+    FDB.DatabaseName:=DatabasePath;
+    end;
+  if FDB.Transaction = nil then
+    FDB.Transaction := TSQLTransaction.Create(FDB);
+  FDB.LogEvents:=LogAllEventsExtra;
+end;
+
+
+function TSQLDBIndexDB.GetLanguageID(const ALanguage: UTF8String): int64;
 var
   Q: TSQLQuery;
 begin
@@ -101,7 +142,7 @@ begin
   end;
 end;
 
-function TSQLDBIndexDB.GetWordID(const AWord: string): int64;
+function TSQLDBIndexDB.GetWordID(const AWord: UTF8String): int64;
 var
   Q: TSQLQuery;
 begin
@@ -125,16 +166,18 @@ begin
   end;
 end;
 
-function TSQLDBIndexDB.CreateQuery(const ASQL: string): TSQLQuery;
+function TSQLDBIndexDB.CreateQuery(const ASQL: UTF8String): TSQLQuery;
 begin
   Result := TSQLQuery.Create(Self);
-  Result.Database := Self.db;
-  Result.Transaction := Self.db.Transaction;
+  Result.Database := Self.FDB;
+  Result.Transaction := Self.FDB.Transaction;
   Result.SQL.Text := ASQL;
+  Result.UsePrimaryKeyAsKey:=False;
+  Result.UniDirectional:=True;
   //Writeln('SQL  :',ASQL);
 end;
 
-function TSQLDBIndexDB.GetURLID(const URL: string; ATimeStamp: TDateTime; ALanguageID: int64; DoCreate: boolean = True): int64;
+function TSQLDBIndexDB.GetURLID(const URL: UTF8String; ATimeStamp: TDateTime; ALanguageID: int64; DoCreate: boolean = True): int64;
 var
   Q: TSQLQuery;
 begin
@@ -165,7 +208,7 @@ begin
   end;
 end;
 
-function TSQLDBIndexDB.CreateCachedQuery(QueryType: TCachedQueryType; const ASQL: string): TSQLQuery;
+function TSQLDBIndexDB.CreateCachedQuery(QueryType: TCachedQueryType; const ASQL: UTF8String): TSQLQuery;
 begin
   if FQueries[QueryType] = nil then
   begin
@@ -191,7 +234,7 @@ var
   Q: TSQLQuery;
   FN, FP, FD, FW, FC: TField;
   Res: TSearchWordData;
-  S,WW : String;
+  S,WW : UTF8String;
   I,L : Integer;
 
 begin
@@ -233,31 +276,32 @@ begin
   end;
 end;
 
-procedure TSQLDBIndexDB.DeleteWordsFromFile(URL: string);
+procedure TSQLDBIndexDB.DeleteWordsFromFile(URL: UTF8String);
 begin
   inherited DeleteWordsFromFile(URL);
   FLastURL := '';
 end;
 
-procedure TSQLDBIndexDB.Execute(const sql: string; ignoreErrors: boolean = True);
+procedure TSQLDBIndexDB.Execute(const sql: UTF8String; ignoreErrors: boolean = True);
 begin
   if SQL = '' then
     exit;
   try
-    DB.ExecuteDirect(sql);
+    FDB.ExecuteDirect(sql);
   except
-    if not IgnoreErrors then
-      raise;
+    on E : exception do
+      if not IgnoreErrors then
+        raise
+      else
+        Writeln(E.ClassName,' : ',E.Message);
+
   end;
 end;
 
 procedure TSQLDBIndexDB.Connect;
 begin
-  if (DB = nil) then
-    db := GetConnection;
-  if DB.Transaction = nil then
-    DB.Transaction := TSQLTransaction.Create(db);
-  DB.Connected := True;
+  EnsureDB;
+  FDB.Connected := True;
 end;
 
 procedure TSQLDBIndexDB.Disconnect;
@@ -267,14 +311,13 @@ Var
 begin
   For T:=Low(TCachedQueryType) to High(TCachedQueryType) do
     FreeAndNil(FQueries[T]);
-  FreeAndNil(DB);
+  FreeAndNil(FDB);
 end;
 
 procedure TSQLDBIndexDB.CreateDB;
 begin
-  if DB = nil then
-    DB := GetConnection;
-  DB.CreateDB;
+  EnsureDB;
+  FDB.CreateDB;
   Connect;
   CreateIndexerTables;
 end;
@@ -287,12 +330,12 @@ end;
 
 procedure TSQLDBIndexDB.BeginTrans;
 begin
-  DB.Transaction.StartTransaction;
+  FDB.Transaction.StartTransaction;
 end;
 
 procedure TSQLDBIndexDB.CommitTrans;
 begin
-  DB.Transaction.Commit;
+  FDB.Transaction.Commit;
 end;
 
 procedure TSQLDBIndexDB.CompactDB;
diff --git a/packages/fpindexer/src/sqliteindexdb.pp b/packages/fpindexer/src/sqliteindexdb.pp
index 3e1533e5a8..24381dde1f 100644
--- a/packages/fpindexer/src/sqliteindexdb.pp
+++ b/packages/fpindexer/src/sqliteindexdb.pp
@@ -24,7 +24,7 @@ uses
 type
   TDatabaseID = record
     ID: int64;
-    Name: string;
+    Name: UTF8string;
   end;
 
   { TSQLiteIndexDB }
@@ -32,24 +32,24 @@ type
   TSQLiteIndexDB = class(TSQLIndexDB)
   private
     db: Psqlite3;
-    FFileName: string;
+    FFileName: UTF8string;
     Frow: integer;
     FSearchClass: TFPSearch;
     LanguageID: TDatabaseID;
-    QueryResult: string;
+    QueryResult: UTF8string;
     SearchWordID: TDatabaseID;
     URLID: TDatabaseID;
     procedure CheckSQLite(Rc: cint; pzErrMsg: PChar);
   protected
     class function AllowForeignKeyInTable: boolean; override;
-    function GetFieldType(FieldType: TIndexField): string; override;
-    function GetLanguageID(const ALanguage: string): int64;
-    function GetURLID(const URL: string; ATimeStamp: TDateTime; ALanguageID: int64; DoCreate: boolean = True): int64; override;
-    function GetWordID(const AWord: string): int64; virtual;
-    function InsertLanguage(const ALanguage: string): int64; virtual;
-    function InsertURL(const URL: string; ATimeStamp: TDateTime; ALanguageID: int64): int64;
-    function InsertWord(const AWord: string): int64; virtual;
-    procedure Execute(const sql: string; ignoreErrors: boolean = True); override;
+    function GetFieldType(FieldType: TIndexField): UTF8string; override;
+    function GetLanguageID(const ALanguage: UTF8string): int64;
+    function GetURLID(const URL: UTF8string; ATimeStamp: TDateTime; ALanguageID: int64; DoCreate: boolean = True): int64; override;
+    function GetWordID(const AWord: UTF8string): int64; virtual;
+    function InsertLanguage(const ALanguage: UTF8string): int64; virtual;
+    function InsertURL(const URL: UTF8string; ATimeStamp: TDateTime; ALanguageID: int64): int64;
+    function InsertWord(const AWord: UTF8string): int64; virtual;
+    procedure Execute(const sql: UTF8string; ignoreErrors: boolean = True); override;
   public
     destructor Destroy; override;
     procedure AddSearchData(ASearchData: TSearchWordData); override;
@@ -58,10 +58,10 @@ type
     procedure CompactDB; override;
     procedure Connect; override;
     procedure CreateDB; override;
-    procedure DeleteWordsFromFile(URL: string); override;
+    procedure DeleteWordsFromFile(URL: UTF8string); override;
     procedure FindSearchData(SearchWord: TWordParser; FPSearch: TFPSearch; SearchOptions: TSearchOptions); override;
   published
-    property FileName: string read FFileName write FFileName;
+    property FileName: UTF8string read FFileName write FFileName;
   end;
 
 implementation
@@ -99,7 +99,7 @@ end;
 
 { TSQLiteIndexDB }
 
-procedure TSQLiteIndexDB.Execute(const sql: string; ignoreErrors: boolean = True);
+procedure TSQLiteIndexDB.Execute(const sql: UTF8string; ignoreErrors: boolean = True);
 var
   pzErrMsg: PChar;
   rc: cint;
@@ -111,9 +111,9 @@ begin
     CheckSQLite(rc, pzErrMsg);
 end;
 
-function TSQLiteIndexDB.GetURLID(const URL: string; ATimeStamp: TDateTime; ALanguageID: int64; DoCreate: boolean): int64;
+function TSQLiteIndexDB.GetURLID(const URL: UTF8string; ATimeStamp: TDateTime; ALanguageID: int64; DoCreate: boolean): int64;
 var
-  SQL: string;
+  SQL: UTF8string;
 begin
   if (URL = URLID.Name) then
     Result := URLID.ID
@@ -129,9 +129,9 @@ begin
   end;
 end;
 
-function TSQLiteIndexDB.GetLanguageID(const ALanguage: string): int64;
+function TSQLiteIndexDB.GetLanguageID(const ALanguage: UTF8string): int64;
 var
-  SQL: string;
+  SQL: UTF8string;
 begin
   if (ALanguage = LanguageID.Name) then
     Result := LanguageID.ID
@@ -147,9 +147,9 @@ begin
   end;
 end;
 
-function TSQLiteIndexDB.GetWordID(const AWord: string): int64;
+function TSQLiteIndexDB.GetWordID(const AWord: UTF8string): int64;
 var
-  SQL: string;
+  SQL: UTF8string;
 begin
   if (AWord = SearchWordID.Name) then
     Result := SearchWordID.ID
@@ -165,26 +165,26 @@ begin
   end;
 end;
 
-function TSQLiteIndexDB.InsertWord(const AWord: string): int64;
+function TSQLiteIndexDB.InsertWord(const AWord: UTF8string): int64;
 begin
   Execute(Format(InsertSQL(itWords, False), ['Null', QuoteString(AWord)]), False);
   Result := sqlite3_last_insert_rowid(db);
 end;
 
-function TSQLiteIndexDB.InsertURL(const URL: string; ATimeStamp: TDateTime; ALanguageID: int64): int64;
+function TSQLiteIndexDB.InsertURL(const URL: UTF8string; ATimeStamp: TDateTime; ALanguageID: int64): int64;
 begin
   // ifFilesID,ifFilesURL,ifFilesReindex,ifFilesUpdated,ifFilesTimeStamp,ifFilesLanguageID
   Execute(Format(InsertSQL(itFiles, False), ['Null', QuoteString(URL), '0', '0', QuoteString(DateToISO8601(ATimeStamp)), IntToStr(AlanguageID)]), False);
   Result := sqlite3_last_insert_rowid(db);
 end;
 
-function TSQLiteIndexDB.InsertLanguage(const ALanguage: string): int64;
+function TSQLiteIndexDB.InsertLanguage(const ALanguage: UTF8string): int64;
 begin
   Execute(Format(InsertSQL(itLanguages, False), ['Null', QuoteString(ALanguage)]), False);
   Result := sqlite3_last_insert_rowid(db);
 end;
 
-function TSQLiteIndexDB.GetFieldType(FieldType: TIndexField): string;
+function TSQLiteIndexDB.GetFieldType(FieldType: TIndexField): UTF8string;
 begin
   Result := inherited GetFieldType(FieldType);
   if (Result = PrimaryFieldType) then
@@ -196,7 +196,7 @@ begin
   Result := True;
 end;
 
-procedure TSQLiteIndexDB.DeleteWordsFromFile(URL: string);
+procedure TSQLiteIndexDB.DeleteWordsFromFile(URL: UTF8string);
 begin
   inherited DeleteWordsFromFile(URL);
 
@@ -247,7 +247,7 @@ end;
 procedure TSQLiteIndexDB.AddSearchData(ASearchData: TSearchWordData);
 var
   WID, LID, FID: int64;
-  SQL: string;
+  SQL: UTF8string;
 begin
   WID := GetWordID(ASearchData.SearchWord);
   LID := GetLanguageID(ASearchData.Language);
@@ -262,7 +262,7 @@ end;
 
 procedure TSQLiteIndexDB.CheckSQLite(Rc: cint; pzErrMsg: PChar);
 var
-  S: string;
+  S: UTF8string;
 begin
   if (rc <> SQLITE_OK) then
   begin
@@ -276,7 +276,7 @@ procedure TSQLiteIndexDB.FindSearchData(SearchWord: TWordParser; FPSearch: TFPSe
 var
   pzErrMsg: PChar;
   rc: cint;
-  sql: string;
+  sql: UTF8string;
 begin
   FSearchClass := FPSearch;
   Frow := 0;