unit Note_Lister;

{$define TOMBOY_NG}

{   Copyright (C) 2017-2024 David Bannon

    License:
    This code is licensed under MIT License, see the file License.txt
    or https://spdx.org/licenses/MIT.html  SPDX short identifier: MIT

    ------------------

    A class that knows how to read a directory full of notes. It keeps those list
	internally, sorted by date. Note details (
    Title, LastChange) can be updated (eg when a note is saved).

    This unit is all about maintaining a list (FPList) of all the notes. We use a threaded class,
    TIndexThread, to read files and initially populate the list. When saving, deleting or syncing
    in a note, we update the list.

    Templates are not added to the list.



     -------- Multithreaded Indexing ----------

  1. Only used for indexing all notes, when a single note is indexed, main thread.
  2. The IndexNotes() method will call ThreadIndex.Start (and therfore TIndexThread.Execute)
     four times, passing a set of the first chars [0..9, a..f, A..F] of file names to index.
     TIndexThread.Execute will call GetNoteDetails for each note that Execute finds
     GetNoteDetails reads note, builds a data structure and, subject to critical
     section, adds it to the main data structure.
  3. NoteBooks cleaned up etc in the IndexNotes method.
  4. Four theads on a multicore cpu gives some 2 - 3 times spead up. Little slower on single core.
  5. We use RTL CriticalSection code, LCL version is similar performance.
  6. Using FindFirst/Next is substantually faster than FindAllFiles.

  Search Modes
  ------------
  We have two Search Modes, SWYT and PressEnter. Both expect ListView to be OwnerData mode.
  SWYT - We have maintained at all time full content of all notes in the NoteLister unit. Two
         indexes, TitleSearchIndex and DataSearchIndex, all they hold (in the pointer FPList provides)
         is the index into NoteList. So we can sort them (and use them backwards) to get sorted
         data. When searching, on the first scan, we build new, probably shorter indexes and then
         subsquent scans remove invalid entries as user types a search term.
  PressEnter - is kinder on memory and not as CPU demanding. It effectivly just uses the SWYT
         first scan. But it has to add all note content to NoteList first and clear it when
         user appears to be finished searching. Uses TGetContentThread class to search.

  In addition, we maintain another index, DateAllIndex to be able to display notes in date
         listed order. Its always as long as NoteList and, like the two above, must be maintained
         when a note is saved, deleted, synced in etc.


	History
	2017/11/23 - added functions to save and retreive the Form when a note
    is open. Also added a function to turn a fullfilename, a filename or an ID
    into a filename of the form GID.note

	2017/11/29  Added FileName to "Note has no Title" error message.
	2017/11/29  check to see if NoteList is still valid before passing
				on updates to a Note's status. If we are quiting, it may not be.
	2017/11/29  Fixed a memory leak that occurred when Delete-ing a entry in the list
				Turns out you must dispose() that allocation before calling Delete.
	2017/12/28  Commented out unnecessary DebugLn
	2017/12/29  Added a debug line to ThisNoteIsOpen() to try and see if there is
				a problem there. Really don't think there is but ...
	2018/01/25  Changes to support Notebooks
    2018/02/14  Changed code that does Search All Notes stuff cos old code stopped on a tag
    2018/02/15  Can now search case sensitive or not and any combination or exact match
    2018/04/28  Set FixedRows to zero after Clean-ing strgrid, seems necessary in Trunk
    2018/06/26  Used E.Message in exception generated by bad XML - dah ....
    2018/07/04  Added a flag, XMLError, set if we found a note unable to index.    Why ?
    2018/08/24  Debugmode now set by calling process.
    2018/11/04  Added support for updating NoteList after a sync.
    2018/12/29  Small improvements in time to save a file.
    2019/04/13  Tweaks to overload to read help nodes
    2019/05/06  Support saving pos and open on startup in note.
    2020/01/03  When searching without AnyCombo ticked, string can be sub-grouped by double inverted commas "
    2020/01/29  Fix multiple notebook tags for same notebook in note file.
                Sort main list, added functions to populate MMenu Recent list.
                Tweek func that populates the main stringGrid avoiding initial sort
    2020/01/31  LoadStringGrid*() now uses the Lazarus column mode.
    2020/02/03  Make contents of strgrid look like it claims to be after new data
                Removed LoadSearchGrid, no use LoadStGrid in both modes.
    2020/02/19  XML Escape the notebook list sent back.
    2020/03/27  Better reporting on short lastchangedate string. But need an autofix.
    2020/04/01  Bug fix for code that auto fixes short last-change-date.
    2020/04/19  Missing $H+ caused  255 char default string, messed with RewriteBadChangeDate()
    2020/05/10  Multithreaded search
    2020/05/25  Don't read sett.checkcasesensitive in thread.
    2020/08/01  Disable code to rewrite short lcd.
    2021/01/03  LoadListView now uses TB_datetime, more tolerant of differing DT formats.
    2021/02/14  Notebook list now sorted, A->z
    2021/07/05  Changed a lot of "for X to 0" to "for 0 downto X" so searches start at end of list where current data is
    2021/08/30  Removed dependencies on Sett and SearchUnit.   Added Dump methods.
                Added function GetNotebooks(const ID: ANSIString): string; for GitHub
    2021/08/31  Added TheNoteLister to hold a ref to the NoteLister for any unit that 'uses' this unit.
    2021/09/06  GetNotebooks result now wrapped in square brackets, JSON style
    2022/01/12  Trapped out some errors that occur if XML element (field) is present but blank
    2022/04/14  GetNotebooks() now takes a StringArray instead of List
                TNoteLister.Count() removed, use TNoteLister.GetNoteCount() instead
    2022/09/05  ---- Substantial Changes ----
                Got rid of local global vars.
                Use either SWYT, Search While You Type, or PressEnter mode. All here in NoteLister
                SearchUnit.ListView now in ownerdata mode, faster with lots of notes.
                NoteList now a public var of NoteLister, other units can iterate over it.
    2022/10/29  Use test, "if TheMainNoteLister = self" to ensure Index are only used by TheMainNoteLister in IndexNotes()
    2022/11/09  Trap out a totally bad xml note, reasonably gracefully.


}

{$mode objfpc}  {$H+}

INTERFACE

uses
		Classes, SysUtils, Grids, ComCtrls, Forms, FileUtil;

type TLVSortMode = (smRecentUp, smRecentDown, smAATitleUp, smAATitleDown, smAllRecentUp);

type
   PNotebook=^TNotebook;
   TNotebook = record
       Name : ANSIString;      // Name of the notebook
       Template : ANSIString;  // The FName of the Template for this Notebook, inc .note
       Notes : TStringList;    // A list of the Fnames of notes that are members of this Notebook, inc .note.
   end;

type

   { TNoteBookList }

   TNoteBookList = class(TList)
   private
     	function Get(Index : integer) : PNoteBook;
		procedure RemoveNoteBook(const NBName: AnsiString);
   public
        destructor Destroy; Override;
                                { ID of Note to be added; Name of NoteBook it should be added to. Notebook rec
                                is created if necessary. But if IsTemplate ID is ID of a newly created Template  }
        procedure Add(const ID, ANoteBook : ANSIString; IsTemplate : boolean);
                                { Returns True if the passed note ID is in the passed Notebook }
        function IDinNotebook(const ID, Notebook : ANSIstring) : boolean;
                                // Returns a PNoteBook that has a name matching passed NoteBook.
        function FindNoteBook(const NoteBook : ANSIString) : PNoteBook;
                                { Removes any list entries that do not have a Template }
        procedure CleanList();
        property Items[Index : integer] : PNoteBook read Get; default;
   end;


type
  	PNote=^TNote;
  	TNote = record
        		{ will have 36 char GUI plus '.note' }
		ID : ANSIString;
        Title : ANSIString;
                { An all lower case version of Title for searching }
        TitleLow : string;
        		{ a 33 char date time string }
    	CreateDate : ANSIString;
                { a 33 char date time string, updateable }
    	LastChange : ANSIString;
        IsTemplate : boolean;
        OpenOnStart : boolean;
        OpenNote : TForm;           // If note is open, its in this TForm.
        Content : string;           // May contain note content, '' else.
        InSearch : boolean;         // indicates note 'passed' last filter, use again
	end;

type                                 { ---------- TNoteList ---------}
   //TNoteList = class(TList)
   TNoteList = class(TFPList)
   private

    	function Get(Index: integer): PNote;
    public
        destructor Destroy; override;
        function Add(ANote : PNote) : integer;
        function FindID(const ID: ANSIString): pNote;
        function FindID(out Index:integer; const ID: ANSIString): boolean;
        property Items[Index: integer]: PNote read Get; default;
    end;



type

   { SortList - Type for DateSortList and TitleSortList, indexes into NoteList }

   TSortList = class(TList)    // Provides a "revised index" into NoteList, sorted on either date or title
   private                     // ToDo : TFPList is supposed to be a bit faster.
    	function Get(Index: integer): integer;
    public
        destructor Destroy; override;
        function Add(ANumber : integer) : integer;
        //function FindName(const Name : ANSIString) : PNote;
        property Items[Index: integer]: integer read Get; default;
    end;


type

    { TNoteLister }

    TNoteLister = class
   private
    //DebugMode : boolean;
    CriticalSection: TRTLCriticalSection;   // we use RTL CriticalSection code, the LCL version is about the same

    TitleSearchIndex : TSortList;  // A list of Indexes into NoteList, filtered by search, sorted by Title
    DateSearchIndex  : TSortList;  // A list of Indexes into NoteList, filtered by search, sorted by Date
    DateAllIndex     : TSortList;  // An sorted on date index of all notes in NoteList (except templates)
    EnterDateSearchIndex  : TSortList;  // A list of Indexes into NoteList, filtered by Press Enter search, sorted by Date
    EnterTitleSearchIndex : TSortList;  // A list of Indexes into NoteList, filtered by Press Enter search, sorted by Title



//    SearchCount : integer;      // How many notes are active in search, or all notes if search not active

    OpenNoteIndex : integer;        // Used for Find*OpenNote(), points to last found one, -1 meaning none found
   	SearchNoteList : TNoteList;
                                { NoteBookList is a list of pointers. Each one points to a record
                                  containing Name, Template ID and a List (called Notes) of IDs of
                                  notes that are members of this Notebook. }
    NoteBookList : TNoteBookList;
                                // Passed a StringList containing 0 to n strings. Returns False if any (non empty)
                                // string is not present in NoteList[index]^.Content. True if all present or list empty.
    function CheckSearchTerms(const STermList : TStringList; const Index : integer;
        TitleOnly : boolean) : boolean;
                                { Returns a simple note file name, accepts simple filename or ID }
    function CleanFileName(const FileOrID: AnsiString): ANSIString;

                                // Indexes one note. Always multithread mode but sometimes its only one thread.
                                // Does require CriticalSection to be setup before calling.
                                // If note turns out to be a template, don't add it to main note list
                                // but still call Notebook.add to ensure its mentioned in notebook list.
                                // We might store Content and it may. or may not be all lower case.
    procedure GetNoteDetails(const Dir, FileName: ANSIString; DontTestName: boolean; TheLister : TNoteLister);



                                // Inserts a new item into the ViewList, always Title, DateSt, FileName
//    function NewLVItem(const LView: TListView; const Title, DateSt, FileName: string): TListItem;

                                { A Early ver of -ng wrote a bad date stamp, here we try to fix any we find. First
                                  just try to add missing bits, if that does not work, we replace the LCD with
                                  current, and known good date.}
	procedure RewriteBadChangeDate(const Dir, FileName, LCD: ANSIString);





   public
    DebugMode : boolean;
    ThreadLock : integer;          // -1 if unlocked, has value of thread when locked
    FinishedThreads : integer;     // There are here to allow the search threads to find them.

                            { NoteList is a list of pointers. Each one points to a record that contains data
                              about a particular note. Only Notebook info it has is whether or not its a
                              template. The ID is stored as a 36 char GUI plus '.note'. Dates must be 33 char. }
    NoteList : TNoteList;


    XMLError : Boolean;   // Indicates a note was found with an XML (or other) error, checked by calling process.
    ErrorNotes : TStringList;
                                        { The directory, with trailing seperator, that the notes are in }
   	WorkingDir : ANSIString;
   	SearchIndex : integer;
    procedure DumpNoteNoteList(WhereFrom: string);
                                        // Puts name of any note that contains (case insensitive) the passed string into
                                        // the passed stringlist. Does nothing to do with sorting, order etc.
                                        // Used to get backlinks.
                                        // ToDo : At present, assumes all content loaded.
    procedure SearchContent(const St: string; Stl: TstringList);
    procedure DumpNoteBookList(WhereFrom: String);

                                        { Returns true if there is a notebook of the passed title }
    function IsANotebookTitle(NBTitle : string) : boolean;
                                        { Returns the Notebook Name for a given filename or ID (of the template itself)}
    function GetNotebookName(FileorID: AnsiString): string;
                                        { returns a indexed pointer to a Notebookrecord }
    function GetNoteBook(Index: integer): PNoteBook;
                                        { returns the number items in the notebook list}
    function NotebookCount(): integer;
                                        {Returns the number of records in the Notelist, NOT the index lists. }
    function GetNoteCount() : integer;
                                        { Returns a pointer to PNote record, zero based, non sorted index }
    function GetNote(Index: integer): PNote;
                                        { Returns a pointer to PNote record, zero based index is adjusted for current search }
    function GetNote(Index: integer; mode : TLVSortMode): PNote;
                                        { Loads a TListView with note title, LCD and ID}
    //procedure LoadListView(const LView: TListView; const SearchMode: boolean);

    // ----------------- N O T E   B O O K   M E T H O D S ---------------------

                                        { Changes the name associated with a Notebook in the internal data structure }
    function AlterNoteBook(const OldName, NewName: string): boolean;
                                        { Returns a multiline string to use in writing a notes notebook membership,
                                          knows how to do a template too. String has special XML chars 'escaped'.
                                          Includes the <tags> tags but returns empty string if no NB membership.
                                          This function expects to be passed an ID + '.note'. }
    function NoteBookTags(const NoteID: string): ANSIString;
                                        { Returns true if it has returned with a pointer to a list with one or more Note Fnames
                                        that are members of NBName, it returns a pointer to the internal StList, do not create
                                        or free. FNames mean ID.note ! }
    function GetNotesInNoteBook(out NBIDList: TStringList; const NBName: string ): boolean;
                                       { Retuns the title of note at (zero based) index. }
    function GetTitle(Index: integer): string;
                                        { Returns the title for a given ID or Filename }
    function GetTitle(const ID: String): string;
                                        { Removes the Notebook entry with ID=Template from Notebook datastructure }
    procedure DeleteNoteBookwithID(FileorID : AnsiString);
                                        { Returns True if passed string is the ID or short Filename of a Template }
    function IsATemplate(FileOrID : AnsiString) : boolean;
                                        { ID of Note to be added; Name of NoteBook it should be added to. Notebook rec
                                        is created if necessary. But if IsTemplate ID is ID of a newly created Template  }
    procedure AddNoteBook(const ID, ANoteBook: ANSIString; IsTemplate: Boolean);
                                        { Sets the passed Notebooks as 'parents' of the passed note. Any pre existing membership
                                          will be cancelled. The list can contain zero to  many notebooks. }
    procedure SetNotebookMembership(const ID: ansistring; const MemberList: TStringList);

    procedure RemoveNoteFromNoteBooks(NoteID: string);
                                        { If ID is empty, always returns false, puts all Notebook names in NBArray. If ID is not
                                          empty, list is filtered for only notebooks that have that ID  and returns True iff the
                                          passed ID is that of a Template.  A Notebook Template will have only one Notebook name in
                                          its Tags and that will be added to strlist. The StartHere template won't have a Notebook
                                          Name and therefore wont get mixed up here ???? Expects ID.note }
    function GetNotebooks(out NBArray: TStringArray; const ID: ANSIString): boolean;
                                        { Rets a (JSON array like, escaped) string of Notebook names that this note is a member of.
                                        It returns an empty array if the note has no notebooks or cannot be found.
                                        If ID is a template, will send a two element array ["template', "notebook-name"].
                                        Expects an ID.note . Result is like this ["Notebook One", "Notebook2", "Notebook"]  }
    function NotebookJArray(const ID: ANSIString): string;
                                        { Loads the Notebook ListBox up with the Notebook names we know about. Add a bool to indicate
                                          we should only show Notebooks that have one or more notes mentioned in SearchNoteList. Call after
                                          GetNotes(Term) }
    procedure LoadListNotebooks(const NotebookItems: TStrings; SearchListOnly: boolean);

    // ---------------------------------

                                        { Returns the LastChangeDate string for ID in the Notes list, empty string
                                        if not found (empty string is its a notebook) }
    function GetLastChangeDate(const ID: String): string;
                                        { Adds details of odd note to NoteList, does NOT update Indexes. }
    procedure IndexThisNote(const ID : String);
                                        { Returns T is ID in current list, takes 36 char GUID or simple file name }
    function IsIDPresent(ID : string) : boolean;

                                        { Adds a note to main list, ie when user creates a new note }
    procedure AddNote(const FileName, Title, LastChange : ANSIString);
                                        { Read the metadata from all the notes into internal data structure,
                                        this is the main "go and do it" function. Note, it uses threads and FindFirst.
                                        Does NOT generate the note Indexes because its not always needed.}
   	function IndexNotes(DontTestName: boolean=false): longint;
                                        { Copy the internal Note data to the passed TStringGrid, empting it first.
                                          NoCols can be 2, 3 or 4 being Name, LastChange, CreateDate, ID.
                                          Special case only main List SearchMode True will get from the search list.
                                          Only used by Recover unit now. }
   	procedure LoadStGrid(const Grid: TStringGrid; NoCols: integer;  SearchMode: boolean=false);
                                        { Copy the internal Note Data to passed TStrings }
    procedure LoadStrings(const TheStrings : TStrings);
    		                            // Returns True if its updated the internal record as indicated,
                                        // will accept either an ID or a filename. Do NOT pass a Notebook ID !}
    function AlterNote(ID, Change : ANSIString; Title : ANSIString = '') : boolean;
                                        { True if the passed LOWERCASE string is a valid note Title }
    function IsThisATitle(const Title : ANSIString) : boolean;
                        		        { Returns the Form this note is open on, Nil if its not open. Take ID or FileName }
    function IsThisNoteOpen(const ID : ANSIString; out TheForm : TForm) : boolean;
                        		        { Tells the list that this note is open, pass NIL to indicate its now closed }
    function ThisNoteIsOpen(const ID: ANSIString; const TheForm: TForm): boolean;
                        		        { Returns true if it can find a FileName (ie ID.note) to Match this Title }
    function FileNameForTitle(const Title: ANSIString; out FileName : ANSIstring): boolean;
    procedure StartSearch();
    function NextNoteTitle(out SearchTerm : ANSIString) : boolean;
    		                            { removes note from int data, accepting either an ID or Filename. Because this
                                        alters the NoteList indexes, generate new Indexes from SearcUnit, not this method.}
    function DeleteNote(const ID : ANSIString) : boolean;
                                        { Copy the internal data about notes in passed Notebook to passed TListView
                                          for display. So, shown would be all the notes in the nominated notebook.}
    //procedure LoadNotebookViewList(const VL: TListView; const NotebookName: AnsiString);
                                        { Copy the internal data about notes in passed Notebook to passed TStringGrid
                                          for display. So, shown would be all the notes in the nominated notebook.}
    procedure LoadNotebookGrid(const Grid : TStringGrid; const NotebookName : AnsiString);
    		                            { Returns the ID (inc .note) of the notebook Template, if an empty string we did
                                          not find a) the Entry in NotebookList or b) the entry had a blank template. }
    function NotebookTemplateID(const NotebookName : ANSIString) : AnsiString;
                                        { Returns the Form of first open note and sets internal pointer to it, Nil if none found }
    function FindFirstOpenNote(out OpenIndex : integer) : TForm;
                                        { Call after FindFirstOpenNote(), it will return the next one or Nil if no more found }
    function FindNextOpenNote(out NoteIndex : integer) : TForm;
                                        { Returns the ID of first note that should be opened on startup internal pointer
                                          (which is same interger as FindFirstOpenNate) to it, '' if none found }
    function FindFirstOOSNote(out NTitle, NID: ANSIstring): boolean;
                                        { Call after FindFirstOOSNote(), it will return the next one or '' if no more found }
    function FindNextOOSNote(var NTitle, NID: ANSIstring): boolean;
                                        { Ret True if we need to either rerun search search or redisplay it.
                                        Called, typically, when a note is saved. May be a new note or a note
                                        that is being updated. Will always have a new LCD, might have a new Title.
                                        The note may or may not be displayed in SearchUnit. Depending on all that,
                                        we may update Indexes, return false if nothing needs to be done, if True
                                        we will refresh displayed list or, if ReRunSearch is true, we'll re-run the
                                        current search, thus updating Search Indexes. We always update DateAllIndex.}
    function AlterOrAddNote(out ReRunSearch : boolean; const FFName, LCD,
        Title : string) : boolean;


    // New Search methods

                                        // Continues a possible existing search with an extra char in STerm, rets number
                                        // of found items. Rewrites note indexes with only reference to Notes that pass test.
                                        // Does not do anything about Notebook, it may, or may not be already applied.
                                        // Calling process should trigger a redraw of Display.
    function RefineSearch(STermList: TstringList; TitleOnly : boolean): integer;
                                        // Clears any search, returns number of notes represented in list (not inc Templates)
                                        // Rewrites NoteIndexes using all Notes
                                        // Calling process should trigger a redraw of Display.
    function ClearSearch() : integer;
                                        // An overload, accepts a string rather than the StringList.
                                        // Triggers a new search, may have STerm or Notebook or both, rets number of found items.
                                         // Rebuilds and sorts DateSortIndex and TitleSortIndex.
                                         // Calling process should trigger a redraw of Display.
    function NewSearch(STerm : string; NoteBook : string; TitleOnly : boolean) : integer;
                                        // Triggers a new search, may have STerm or Notebook or both, rets number of found items.
                                        // Rebuilds and sorts DateSortIndex and TitleSortIndex.
                                        // Calling process should trigger a redraw of Display.
    function NewSearch(STermList: TstringList; NoteBook: string; TitleOnly : boolean): integer;
                                        // Returns the number of notes still active in SWYT, Search While You Type. Its
                                        // the number in NoteList or less. 0 is possible.
    function NoteIndexCount() : integer;
                                        // Unloads the note content from NoteList, thus saving some memory. Only used in
                                        // PressEnter search mode.
    procedure UnLoadContent();
                                        { This is only called when using the "Press Enter to search" mode -  triggers
                                        threads who's Execute add all the Note's content to NoteList. }
    function LoadContentForPressEnter(): longint;
                                        { Builds a new date sorted index refrencing all notes in NoteList for Menu builder }
    function BuildDateAllIndex(): integer;


    constructor Create;
    destructor Destroy; override;
   end;




Type   { ======================= GET CONTENT THREAD ========================== }

    TGetContentThread = class(TThread)
    private

    protected
        procedure Execute; override;
    public
        CaseSensitive : boolean;
        NoteLister : TNoteLister;   // Thats the note lister that called us
        TIndex : integer;           // Zero based count of threads
        ThreadBlockSize : integer;  // how many files each thread processes
        ResultsList1, ResultsList2 : TSortList;    // List to contain details of what we found, 1=date, 2=title
        WorkDir : String;           // Dir where notes files are
        Term_List : TStringList;    // Incoming list of terms to search for
        Constructor Create(CreateSuspended : boolean);
    end;

        { ======================= INDEX  THREAD ========================== }
type
    CharSet = set of char;

type    TGetNoteDetailsProc = procedure(const Dir, FileName: ANSIString; DontTestName: boolean; TheLister : TNoteLister) of Object;

Type

        TIndexThread = class(TThread)
        private

        protected
            procedure Execute; override;
        public
            GetNoteDetailsProc : TGetNoteDetailsProc;
            TIndex : integer;           // Zero based count of threads
            StartsWith : CharSet;
            WorkingDir : string;
            OneThread : boolean;        // indicates its not regular UUID based notes, do single thread index
            TheLister : TNoteLister;
            Constructor Create(CreateSuspended : boolean);
        end;

                                        // Not in Class so that Threads can find it.
function NoteContains(const TermList : TStringList; FullFileName: ANSIString; const CaseSensitive : boolean): boolean;


var
                        // This is a pointer to the MAIN notelister, its really, really global !
                        // Its set after lister is created in Search unit and must not be used by
                        // any of the other units thinking its the NoteListers made for their own use.
    TheMainNoteLister : TNoteLister = nil;

{ ------------------------------------------------------------------- }
{ -------------------------- IMPLEMENTATION ------------------------- }
{ ------------------------------------------------------------------- }


implementation

uses  laz2_DOM, laz2_XMLRead, LazFileUtils, LazUTF8, LazLogger, tb_utils, syncutils
        {, SearchUnit} {$ifdef TOMBOY_NG}, settings {$endif};         // project options -> Custom Options


{ Laz* are LCL packages, Projectinspector, double click Required Packages and add LCL }

// var                               // Look Mum, no Globals !
//    FinishedThreads : integer;     // There are here to allow the search threads to find them.
//    ThreadLock : integer;          // -1 if unlocked, has value of thread when locked
//    CriticalSection: TRTLCriticalSection;   // we use RTL CriticalSection code, the LCL version is about the same
    // NoteList : TNoteList;                // NO, not global !

 { -------------------------------- SortList --------------------------------- }

 { Several TSortLists are created. They are based on FPList, the only data they
   store is stored in the pointer itself, cast to an integer.  }

function TSortList.Get(Index: integer): integer;
begin
    {$push}
    {$hints off}
    result := PtrUInt(inherited get(Index));
    {$pop}
end;

destructor TSortList.Destroy;
begin
    // we have not allocated any memory for data, no need to dispose
    inherited Destroy;
end;

function TSortList.Add(ANumber: integer): integer;
begin
    // result := inherited Add(pointer(ANumber));            // warning
    {$push}
    {$hints off}
    result := inherited Add(pointer(PtrUInt(ANumber)));      // hint
    {$pop}
end;


// A sort function for TitleSortList
function SortOnTitle(Item1: Pointer; Item2: Pointer):Integer;  inline;  // BE VERY CAREFULL, usable ONLY by main NoteLister
var
   LItem1: SizeInt absolute Item1;                             // Superimpose an Int like thing over pointer to avoid warnings
   LItem2: SizeInt absolute Item2;
begin
    if TheMainNoteLister.NoteList[Litem1]^.TitleLow = TheMainNoteLister.NoteList[Litem2]^.TitleLow then
        Result := 0
    else if TheMainNoteLister.NoteList[LItem1]^.TitleLow > TheMainNoteLister.NoteList[LItem2]^.TitleLow then         // This gives alphabetical, AA at the top
        Result := 1
    else Result := -1;

(*    {$push}
    {$hints off}
    if TheMainNoteLister.NoteList[PtrUInt(item1)]^.TitleLow = TheMainNoteLister.NoteList[PtrUInt(item2)]^.TitleLow then
        Result := 0
    else if TheMainNoteLister.NoteList[PtrUInt(item1)]^.TitleLow > TheMainNoteLister.NoteList[PtrUInt(item2)]^.TitleLow then         // This gives alphabetical, AA at the top
        Result := 1
    else Result := -1;
    {$pop}    *)
end;

function SortOnDate(Item1, Item2 : Pointer):Integer; inline;  // BE VERY CAREFULL, usable ONLY by main NoteLister
var
   LItem1: SizeInt absolute Item1;
   LItem2: SizeInt absolute Item2;
begin
    if TheMainNoteLister.NoteList[LItem1]^.LastChange
                    = TheMainNoteLister.NoteList[LItem2]^.LastChange then
        Result := 0
    else if TheMainNoteLister.NoteList[LItem1]^.LastChange
                    > TheMainNoteLister.NoteList[LItem2]^.LastChange then         // ?? This gives most recent at the top
        Result := 1
    else Result := -1;

(*    {$push}
    {$hints off}
    if TheMainNoteLister.NoteList[PtrUInt(item1)]^.LastChange
                    = TheMainNoteLister.NoteList[PtrUInt(item2)]^.LastChange then
        Result := 0
    else if TheMainNoteLister.NoteList[PtrUInt(item1)]^.LastChange
                    > TheMainNoteLister.NoteList[PtrUInt(item2)]^.LastChange then         // ?? This gives most recent at the top
        Result := 1
    else Result := -1;
    {$pop}      *)
end;

{ ================ I N D E X  T H R E A D  ======================= }

// ToDo : much of the work here is done in GetNoteDetails, maybe it belongs in this Type ?

constructor TIndexThread.Create(CreateSuspended : boolean);
begin
    inherited Create(CreateSuspended);
    FreeOnTerminate := True;
end;

procedure TIndexThread.Execute;           // Might be only thread running or one of four.
var                                       // This tread will index all notes starting with Ch
      Ch : char;                          // The thread will clean up itself when terminated.

  procedure FindNoteFile(Mask : string);
  var
        Info : TSearchRec;
        Cnt : integer = 0;
  begin
    	if FindFirst(WorkingDir + Mask, faAnyFile, Info)=0 then
    		repeat
                inc(cnt);
                GetNoteDetailsProc(WorkingDir, Info.Name, OneThread, TheLister);
    		    //SearchForm.NoteLister.GetNoteDetails(WorkingDir, Info.Name, OneThread, TheLister);
    		until FindNext(Info) <> 0;
    	FindClose(Info);
  end;

begin
    {$ifdef FORCE_SINGLE_INDEX_THREAD}
       FindNoteFile('*.note');
    {$else}
    if OneThread then
        FindNoteFile('*.note')
    else
        for ch in StartsWith do
            FindNoteFile(Ch + '*.note');
    {$endif}
    InterLockedIncrement(TheLister.FinishedThreads);
end;

{ ========================== SEARCH THREAD =========================== }

constructor TGetContentThread.Create(CreateSuspended : boolean);
begin
    inherited Create(CreateSuspended);
    FreeOnTerminate := True;
end;

procedure TGetContentThread.Execute;
var
    EndBlock, I : integer;
//    NoteP : PNote;
    Doc : TXMLDocument;
	Node : TDOMNode;
begin
    EndBlock := (TIndex+1)*ThreadBlockSize;
    if EndBlock > NoteLister.NoteList.Count then
        EndBlock := NoteLister.NoteList.Count;
    if (NoteLister.NoteList.Count - EndBlock) < ThreadBlockSize then
        EndBlock := NoteLister.NoteList.Count;
    I := TIndex * ThreadBlockSize;
    {if EndBlock := FileList.Count then
        debugln('Last Thread Endblock=' + dbgs(EndBlock)); }
    while (not Terminated) and (I < EndBlock) do begin
        if not NoteLister.NoteList[i]^.IsTemplate then begin
          	if not FileExistsUTF8(WorkDir + NoteLister.NoteList[i]^.ID) then begin
                debugln('TNoteLister.TSearchThread.Execute ======== ERROR cannot find ' + WorkDir + NoteLister.NoteList[i]^.ID);
                exit;
            end;
            ReadXMLFile(Doc, WorkDir + NoteLister.NoteList[i]^.ID);           // requires free
            try
                Node := Doc.DocumentElement.FindNode('text');

                while InterlockedCompareExchange(NoteLister.ThreadLock, TIndex, -1) <> -1 do
                    if Terminated then break;          // cycle until its our turn
                if assigned(Node) then begin
                    {$ifdef TOMBOY_NG}
                    if Sett.SearchCaseSensitive then
                        NoteLister.NoteList[i]^.Content := Node.TextContent
                    else {$endif} NoteLister.NoteList[i]^.Content := lowercase(Node.TextContent);
                end
                else debugln('TNoteLister.TSearchThread.Execute ======== ERROR unable to find text in '
                                + WorkDir + NoteLister.NoteList[i]^.ID);
            finally
                InterlockedExchange(NoteLister.ThreadLock, -1);
                doc.free;
            end;
        end;
        inc(I);
    end;
    InterLockedIncrement(NoteLister.FinishedThreads);
end;


{ ========================= N O T E B O O K L I S T ======================== }

function TNoteBookList.Get(Index: integer): PNoteBook;
begin
    Result := PNoteBook(inherited get(Index));
end;

destructor TNoteBookList.Destroy;
var
    I : Integer;
begin
        for I := 0 to Count-1 do begin
          	Items[I]^.Notes.free;
    		dispose(Items[I]);
		end;
		inherited Destroy;
end;


procedure TNoteBookList.Add(const ID, ANoteBook: ANSIString; IsTemplate: boolean );
var
    NB : PNoteBook;
    NewRecord : boolean = False;
    I : integer;
begin
    NB := FindNoteBook(ANoteBook);
    if NB = Nil then begin
        NewRecord := True;
        new(NB);
    	NB^.Name:= ANoteBook;
        NB^.Template := '';
        NB^.Notes := TStringList.Create;
	end;
    if IsTemplate then begin
        NB^.Template:= ID         // should only happen if its a new template.
    end else begin
        // Check its not there already ....
        I := NB^.Notes.Count;
        while I > 0 do begin
                dec(I);
                if ID = NB^.Notes[i]
                    then exit;      // cannot be there if its a new entry so no leak here
        end;
        NB^.Notes.Add(ID);
	end;
	if NewRecord then inherited Add(NB);
end;

function TNoteBookList.IDinNotebook(const ID, Notebook: ANSIstring): boolean;
var
	Index : longint;
    TheNoteBook : PNoteBook;
begin
	Result := False;
    TheNoteBook := FindNoteBook(NoteBook);
    if TheNoteBook = Nil then exit();
    for Index := 0 to TheNoteBook^.Notes.Count-1 do
        if ID = TheNoteBook^.Notes[Index] then begin
            Result := True;
            exit();
		end;
end;

function TNoteBookList.FindNoteBook(const NoteBook: ANSIString): PNoteBook;
var
        Index : longint;
begin
        Result := Nil;
        for Index := 0 to Count-1 do begin
            if Items[Index]^.Name = NoteBook then begin
                Result := Items[Index];
                exit()
    	    end;
    	end;
end;

function TNoteLister.IsANotebookTitle(NBTitle: string): boolean;
var
    P : PNoteBook;
begin
    P := NoteBookList.FindNoteBook(NBTitle);
    result := P <> nil;
end;

procedure TNoteBookList.CleanList;
var
	Index : integer = 0;
begin
	while Index < Count do begin
        if Items[Index]^.Template = '' then begin
          	Items[Index]^.Notes.free;
    		dispose(Items[Index]);
            Delete(Index);
		end else
        	inc(Index);
	end;
end;

		// Don't think we use this method  ?
procedure TNoteBookList.RemoveNoteBook(const NBName: AnsiString);
var
	Index : integer;
begin
	for Index := 0 to Count-1 do
    	if Items[Index]^.Name = NBName then begin
          	Items[Index]^.Notes.free;
    		dispose(Items[Index]);
            Delete(Index);
            break;
		end;
    debugln('ERROR, asked to remove a note book that I cannot find.');
end;

// =================== DEBUG PROC ======================================

procedure TNoteLister.DumpNoteBookList(WhereFrom : String);
var
    P : PNotebook;
    I : integer;
begin
    debugln('------------ ' + WhereFrom + ' -----------');
    for P in NoteBookList do begin
        debugln('Name=' + P^.Name);
        for I := 0 to P^.Notes.Count -1 do
            debugln('     ' + P^.Notes[I]);
    end;
    debugln('-----------------------');
end;

procedure TNoteLister.DumpNoteNoteList(WhereFrom : string);
var
    P : PNote;
    Pnb : PNotebook
//    I : integer;
;begin
    debugln('-----------' + WhereFrom + '------------');
    for P in NoteList do begin
        debugln('ID=' + P^.ID + '   ' +  P^.Title);
        debugln('CDate=' + P^.CreateDate + ' template=' + booltostr(P^.IsTemplate, true));
    end;
    debugln('-----------------------------------------------');
    for Pnb in NoteBookList do
        debugln('Template ID=' + Pnb^.Template + '  NB Name='+Pnb^.Name + ' and Notes are ' + Pnb^.Notes.Text);
    debugln('-----------------------------------------------');
end;



function TNoteLister.GetNoteCount(): integer;
begin
    result :=  NoteList.Count;
end;

{ ============================== NoteLister ================================ }

{ -------------  Things relating to NoteBooks ------------------ }


function TNoteLister.NotebookCount(): integer;
begin
    Result := NoteBookList.Count;
end;


function TNoteLister.GetNoteBook(Index : integer) : PNoteBook;
begin
    Result := NoteBookList[Index];
end;

function TNoteLister.NoteBookTags(const NoteID : string): ANSIString;
var
    NBArray : TStringArray;
    Index : Integer;
begin
    Result := '';
    if GetNotebooks(NBArray, NoteID) then begin         // its a template
   		Result := '  <tags>'#10'    <tag>system:template</tag>'#10;
        if length(NBArray) > 0 then
        	Result := Result + '    <tag>system:notebook:' + RemoveBadXMLCharacters(NBArray[0], True) + '</tag>'#10'  </tags>'#10;
    end else
   		if length(NBArray) > 0 then begin				// its a Notebook Member
        	Result := '  <tags>'#10;
        	for Index := 0 to High(NBArray) do		    // here, we auto support multiple notebooks.
        		Result := Result + '    <tag>system:notebook:' + RemoveBadXMLCharacters(NBArray[Index], True) + '</tag>'#10;
        	Result := Result + '  </tags>'#10;
		end;
end;

function TNoteLister.NotebookJArray(const ID: ANSIString): string;
var
    NBArray : TStringArray;
    Index : Integer;
begin
    Result := '';
    if GetNotebooks(NBArray, ID) then               // its a template
    		Result := '"template", "' + EscapeJSON(NBArray[0]) + '"'
    else begin                                      // maybe its a Notebook Member
        for Index := 0 to high(NBArray) do		    // here, we auto support multiple notebooks.
            Result := Result + '"' + EscapeJSON(NBArray[Index]) + '", ';
        if Result <> '' then                        // will be empty if note is not member of a notebook
            delete(Result, length(Result)-1, 2);    // remove trailing comma and space
    end;
    Result := '[' + Result + ']';                   // Always return the brackets, even if empty
    //debugln('TNoteLister.NotebookJArray returning Notebooks jArray = ' + Result);
end;

function TNoteLister.GetNotesInNoteBook(out NBIDList : TStringList; const NBName : string) : boolean;
var
    NB : PNoteBook;
begin

    { It appears this returns the ID of a note that was added to this notebook
    but, maybe, not saved, not added to main TnoteLister yet ?  #312
    }
    // debugln('TNoteLister.GetNotesInNoteBook - 1 - NBName=[' + NBName + '] and NBIDList nil=' + booltostr(NBIDList=nil, true));
    Result := True;
    NB := NoteBookList.FindNoteBook(NBName);
    if NB <> Nil then
        NBIDList := NB^.Notes
    else begin
        Result := False;
        NBIDList := nil;
    end;
    // debugln('TNoteLister.GetNotesInNoteBook - 2 - NBName=[' + NBName + '] and NBIDList nil=' + booltostr(NBIDList=nil, true));
end;

function TNoteLister.AlterNoteBook(const OldName, NewName : string) : boolean;
var
    NB : PNoteBook;
begin
    Result := True;
    NB := NoteBookList.FindNoteBook(OldName);
    if NB <> nil then
        NB^.Name:= NewName
    else Result := False;
end;

procedure TNoteLister.AddNoteBook(const ID, ANoteBook: ANSIString; IsTemplate : Boolean);
begin
    NoteBookList.Add(ID, ANoteBook, IsTemplate);
    //DumpNoteBookList('After TNoteLister.AddNoteBook');
end;
(*
procedure TNoteLister.LoadNotebookViewList(const VL : TListView; const  NotebookName: AnsiString);
var
    Index : integer;
    LCDst : string;
begin
    VL.Clear;
    Index := NoteList.Count;
    while Index > 0 do begin
        dec(Index);
        if NotebookList.IDinNotebook(NoteList.Items[Index]^.ID, NoteBookName) then begin
            LCDst := NoteList.Items[Index]^.LastChange;
            if length(LCDst) > 11 then  // looks prettier, dates are stored in ISO std
                LCDst[11] := ' ';       // with a 'T' between date and time
            NewLVItem(VL, NoteList.Items[Index]^.Title, LCDst, NoteList.Items[Index]^.ID);
        end;
	end;
end;     *)

procedure TNoteLister.LoadNotebookGrid(const Grid: TStringGrid; const NotebookName: AnsiString);
var
    Index : integer;
begin
    while Grid.RowCount > 1 do Grid.DeleteRow(Grid.RowCount-1);
    Index := NoteList.Count;
    while Index > 0 do begin
        dec(Index);
        if NotebookList.IDinNotebook(NoteList.Items[Index]^.ID, NoteBookName) then begin
        	Grid.InsertRowWithValues(Grid.RowCount, [NoteList.Items[Index]^.Title,
        			NoteList.Items[Index]^.LastChange]);
		end;
	end;
end;

function TNoteLister.NotebookTemplateID(const NotebookName: ANSIString): AnsiString;
var
    Index : integer;
    //St : string;
begin
    for Index := 0 to NotebookList.Count - 1 do begin
        //St := NotebookList.Items[Index]^.Name;
        if NotebookName = NotebookList.Items[Index]^.Name then begin
            Result := NotebookList.Items[Index]^.Template;
            exit();
		end;
	end;
    debugln('ERROR - asked for the template for a non existing Notebook');
    debugln('NotebookName = ' + Notebookname);
    for Index := 0 to NotebookList.Count - 1 do begin
        if NotebookName = NotebookList.Items[Index]^.Name then
            debugln('Match [' + NotebookList.Items[Index]^.Name + ']')
        else debugln('NO - Match [' + NotebookList.Items[Index]^.Name + ']')
	end;
    Result := '';
end;

function TNoteLister.GetNotebookName(FileorID: AnsiString) : string;
var
    Index : integer;
begin
    for Index := 0 to NotebookList.Count - 1 do
        if CleanFileName(FileorID) = NotebookList.Items[Index]^.Template then
            exit(NotebookList.Items[Index]^.Name);
    //debugln('TNoteLister.GetNotebookName ALERT - asked to find a notebook name but cannot find it : ' + FileorID);
    // thats not an error, sometimes sync systems asks, just in case .....
    result := '';
end;

procedure TNoteLister.DeleteNoteBookwithID(FileorID: AnsiString);
var
    Index : integer;
begin
    for Index := 0 to NotebookList.Count - 1 do begin
        if CleanFileName(FileorID) = NotebookList.Items[Index]^.Template then begin
          	NotebookList.Items[Index]^.Notes.free;
    		dispose(NotebookList.Items[Index]);
            NotebookList.Delete(Index);
            exit();
		end;
	end;
    debugln('TNoteLister.DeleteNoteBookwithID ERROR - asked to delete a notebook by ID but cannot find it : '
        + FileorID);
end;


function TNoteLister.IsATemplate(FileOrID: AnsiString): boolean;
var
    NBArray : TStringArray;
begin
    Result := GetNotebooks(NBArray, CleanFileName(FileOrID));
end;

// Removes all mention of a particular note from the whole NoteBookList
// Useful when a note is being deleted. Pass just ID or Filename.
// No problems if ID is not present anywhere.
procedure TNoteLister.RemoveNoteFromNoteBooks(NoteID : string);
var
    Index, BookIndex : integer;
begin
    if not NoteID.EndsWith('.note') then     // accept just ID or FileName
        NoteID := NoteID + '.note';
    for Index := 0 to NotebookList.Count - 1 do begin                   // Check every notebook
        if NoteBookList[Index]^.Notes <> nil then begin
            BookIndex := NoteBookList[Index]^.Notes.IndexOf(NoteID);
            if BookIndex > -1 then                                      // working directly with NoteBookList entry here.
                NoteBookList[Index]^.Notes.Delete(BookIndex);
        end;
    end;
end;

procedure TNoteLister.SetNotebookMembership(const ID : ansistring; const MemberList : TStringList);
var
    Index, BookIndex : integer;
begin
    // First, remove any mention of this ID from data structure
	for Index := 0 to NotebookList.Count - 1 do begin
        BookIndex := 0;
        while BookIndex < NotebookList.Items[Index]^.Notes.Count do begin
            if ID = NotebookList.Items[Index]^.Notes[BookIndex] then
            	NotebookList.Items[Index]^.Notes.Delete(BookIndex);
            inc(BookIndex);
        end;
	end;
	// Now, put back the ones we want there.
    for BookIndex := 0 to MemberList.Count -1 do
        for Index := 0 to NotebookList.Count - 1 do
            if MemberList[BookIndex] = NotebookList.Items[Index]^.Name then begin
                NotebookList.Items[Index]^.Notes.Add(ID);
                break;
            end;
end;

procedure TNoteLister.LoadListNotebooks(const NotebookItems : TStrings; SearchListOnly : boolean);
var
    Index : integer;

    function FindInSearchList(NB : PNoteBook) : boolean;
    var  X : integer = 0;
    begin
        result := true;
        if Nil = SearchNoteList then
            exit;
        while X < NB^.Notes.Count do begin
            if Nil <> SearchNoteList.FindID(NB^.Notes[X]) then
                exit;
            inc(X);
        end;
        result := false;
    end;

begin
    NoteBookItems.Clear;
    for Index := 0 to NotebookList.Count - 1 do begin
        if (not SearchListOnly) or FindInSearchList(NotebookList.Items[Index]) then begin
            NotebookItems.Add(NotebookList.Items[Index]^.Name);
            //NotebookGrid.InsertRowWithValues(NotebookGrid.RowCount, [NotebookList.Items[Index]^.Name]);
        end;
	end;

end;

function TNoteLister.GetNotebooks(out NBArray: TStringArray; const ID: ANSIString): boolean;
var
    Index, I : Integer;
    Cnt : Integer = 0;
begin
	Result := false;
    Setlength(NBArray, 0);
    Setlength(NBArray, NoteBookList.Count);               // Cannot be more than that
 	for Index := 0 to NoteBookList.Count -1 do begin      // look at each NoteBook, one by one
      	if ID = '' then
            NBArray[Index] := NotebookList.Items[Index]^.Name
        else begin
            if ID = NotebookList.Items[Index]^.Template then begin
                // The passed ID is the ID of a Template itself, not a note.
                // debugln('Looks like we asking about a template ' + ID);
                //if length(NBArray) > 0 then
                //    debugln('Error, seem to have more than one Notebook Name for template ' + ID);
                Setlength(NBArray, 1);                   // truncate after first entry
                NBArray[0] := NotebookList.Items[Index]^.Name;
                exit(True);
			end;
            // OK, if its not a Template, its a note, what notebooks is it a member of ?
            // Each NotebookList item has a list of the notes that are members of that item.
            // if the ID is mentioned in the items note list, copy name to Array.
            // Iterate over the Notes list associated with this particular Notebook entry.
			for I := 0 to NotebookList.Items[Index]^.Notes.Count -1 do
            	if ID = NotebookList.Items[Index]^.Notes[I] then begin
                    NBArray[Cnt] := NotebookList.Items[Index]^.Name;
                    inc(Cnt);
                    // debugln('TNoteLister.GetNotebooks Insert ' + NotebookList.Items[Index]^.Name);
                end;
        end;
	end;
    // if we are still here, its either ID='' or ID is that of a note, not a notebook
    if ID <> '' then
        setlength(NBArray, Cnt);    // almost certainly less than we set above.
(*
    debugln('TNoteLister.GetNotebooks ID = ' +  ID);
    debugln('TNoteLister.GetNotebooks LENGTH ' +  inttostr(length(NBArray)));
    debugln('TNoteLister.GetNotebooks HIGH ' +  inttostr(high(NBArray)));
    for i := 0 to high(NBArray) do
        debugln('TNoteLister.GetNotebooks Array ' + NBArray[i]);       *)
end;


{ -------------- Things relating to Notes -------------------- }

// Address of this function is passed to note list sort. Newest notes at end of list.
(*function LastChangeSorter( Item1: Pointer;   Item2: Pointer) : Integer;
begin                                                                            //  test its way of comparing before trashing !
    // Also ANSICompareStr but we are just looking at date numbers here
	result := CompareStr(PNote(Item1)^.LastChange, PNote(Item2)^.LastChange);
end;                 *)


function NotebookSorter( Item1 : pointer; Item2 : pointer) : integer;
begin
    result := CompareStr(PNoteBook(Item1)^.Name, PNoteBook(Item2)^.Name);
end;

procedure TNoteLister.RewriteBadChangeDate(const Dir, FileName, LCD : ANSIString);
var
    InFile, OutFile: TextFile;
    InString, NewLCD : String;
    {$ifdef WINDOWS}
    ErrorMsg : ANSIString;
    {$endif}
begin
    // Bad format looks like this 2020-03-06 21:25:18
    // But it Should be like this 2020-02-15T12:07:41.0000000+00:00
    AssignFile(InFile, Dir + FileName);
    AssignFile(OutFile, Dir + Filename + '-Dated');
    try
        try
            Reset(InFile);
            Rewrite(OutFile);
            while not eof(InFile) do begin
                readln(InFile, InString);
                if (Pos('<last-change-date>', InString) > 0) then
                    if length(LCD) = 19 then begin
                        NewLCD := LCD + copy(TB_GetLocalTime(), 20, 14);
                        NewLCD[11] := 'T';
                        writeln(OutFile, '  <last-change-date>' + NewLCD + '</last-change-date>');
                    end else begin
                        writeln(OutFile, '  <last-change-date>'
                                + TB_GetLocalTime()
                                + '</last-change-date>');
					end
				else  writeln(OutFile, InString);
		    end;
        finally
            CloseFile(OutFile);
            CloseFile(InFile);
        end;
    except
        on E: EInOutError do begin
                debugln('File handling error occurred updating clean note location. Details: ' + E.Message);
                exit;
            end;
    end;
    {$ifdef WINDOWS}
        if FileExists(Dir + FileName) then    // will not be there if its a new note.
            if not SafeWindowsDelete(Dir + FileName, ErrorMsg) then         // In syncutils, maybe over kill but ......
               exit;
    {$endif}
    RenameFileUTF8(Dir + Filename + '-Dated', Dir + FileName);    // Unix ok to over write, windows is not !
end;


procedure TNoteLister.GetNoteDetails(const Dir, FileName: ANSIString; DontTestName : boolean; TheLister : TNoteLister);
			// This is how we search for XML elements, attributes are different.
            // Note : we used to do seaching here as well as indexing, now just indexing
		    // Note that this method is not Multithread aware, calling method must setup
		    // CriticalSection even if it is single threaded.
var
    NoteP : PNote;
    Doc : TXMLDocument;
	Node : TDOMNode;
    J : integer;
    PossibleError : string = '';
begin
    // writeln('TNoteLister.GetNoteDetails Start');    // debugln gets confused by threading ?
//    if FileName[1] = '1' then exit;       // ToDo : remove
    if DebugMode then debugln('TNoteLister.GetNoteDetails - indexing ' + Dir + FileName);
    if not DontTestName then
        if not IDLooksOK(copy(FileName, 1, 36)) then begin      // In syncutils !!!!
            EnterCriticalSection(CriticalSection);
            try
                ErrorNotes.Append(FileName + ', ' + 'Invalid ID in note filename');
                XMLError := True;
            finally
                LeaveCriticalSection(CriticalSection);
            end;
            exit;
        end;
  	if FileExistsUTF8(Dir + FileName) then begin
        new(NoteP);
        NoteP^.IsTemplate := False;
        NoteP^.ID:=FileName;
        try
            ReadXMLFile(Doc, Dir + FileName);
        except on E: EXMLReadError do begin
                //debugln('GetNoteDetails - XMLReadError ' + E.ErrorMessage);
                //debugln('GetNoteDetails - Invalid XML in ' + Dir + FileName);
                EnterCriticalSection(CriticalSection);
                try
                    ErrorNotes.Append(FileName + ', ' + E.ErrorMessage);
                    XMLError := True;
                finally
                    LeaveCriticalSection(CriticalSection);
                end;
                Doc.Free;
                Dispose(NoteP);
                exit;
            end;
        end;
        try
	        try
	  	        Node := Doc.DocumentElement.FindNode('title');
                if not assigned(Node.FirstChild) then
                   PossibleError := 'XML ERROR, blank Title';               // Catch it as an EObjectException further down
                // If title is blank, next line will trigger a EObjectCheck Exception.
	      	    NoteP^.Title := Node.FirstChild.NodeValue;          // This restores & etc.
                NoteP^.TitleLow := lowercase(NoteP^.Title);
	            //if DebugMode then Debugln('Title is [' + Node.FirstChild.NodeValue + '] ID is ' + FileName);
	            Node := Doc.DocumentElement.FindNode('last-change-date');
                if not assigned(Node.FirstChild) then
                   PossibleError := 'XML ERROR, blank last-change-date';   // Catch it as an EObjectException further down
	            NoteP^.LastChange := Node.FirstChild.NodeValue;
	                {if (length(NoteP^.LastChange) <> 33) or (length(NoteP^.LastChange) <> 27) then begin
	                    RewriteBadChangeDate(Dir, FileName, NoteP^.LastChange);
                        inc(TryCount);
                        if TryCount > 2 then begin
                            debugln('Failed to fix bad last-change-date in ' +  NoteP^.Title);
                            break;     // sad but life must go on.
						end;
                        Doc.free;
					end else
                        break;
				    until false;          }
                if DontTestName or (not Sett.AutoSearchUpdate) then NoteP^.Content := ''       // silly to record content for, eg, help notes.
                else begin
                    NoteP^.Content := '';
                    Node := Doc.DocumentElement.FindNode('text');
                    if assigned(Node) then begin
                        {$ifdef TOMBOY_NG}
                        if Sett.SearchCaseSensitive then
                            NoteP^.Content := Node.TextContent
                        else {$endif} NoteP^.Content := lowercase(Node.TextContent);
                    end
                    else begin
                        EnterCriticalSection(CriticalSection);
                        {$ifdef LINUX}
                        writeln('WRITELN - TNoteLister.GetNoteDetails ======== ERROR unable to find text in ' + FileName);
                        {$endif}
                        TheLister.ErrorNotes.Append(FileName + ', Unable to find text block');
                        freeandnil(Doc);
                        dispose(NoteP);
                        LeaveCriticalSection(CriticalSection);
                    end;
                    if NoteP^.Content = '' then begin
                        EnterCriticalSection(CriticalSection);
                        {$ifdef LINUX}                           // Don't use debugln, unhappy inside a thread.
                        debugln('WRITELN - TNoteLister.GetNoteDetails ERROR, note has no content : Filename : ' + FileName);
                        {$endif}
                        XMLError := True;
                        TheLister.ErrorNotes.Append(FileName + ', This note has no content');
                        freeandnil(Doc);
                        dispose(NoteP);
                        LeaveCriticalSection(CriticalSection);
                        exit();
                        { A completely empty note is an error, we auto insert a newline on a real note so, if empty, something has gone wrong }
                    end;
                end;

                NoteP^.OpenNote := nil;
                NoteP^.InSearch := True;
                Node := Doc.DocumentElement.FindNode('create-date');
                if not assigned(Node.FirstChild) then
                    PossibleError := 'XML ERROR, blank create-date';    // Catch it as an EObjectException further down
                NoteP^.CreateDate := Node.FirstChild.NodeValue;
                try                                                     // this because GNote leaves out 'open-on-startup' !
                    Node := Doc.DocumentElement.FindNode('open-on-startup');
                    if Node = nil then NoteP^.OpenOnStart:= False
                    else NoteP^.OpenOnStart:= (Node.FirstChild.NodeValue = 'True');
                except on E: EObjectCheck do
                    NoteP^.OpenOnStart:= False;
                end;
                Node := Doc.DocumentElement.FindNode('tags');
                if Assigned(Node) then begin
                    for J := 0 to Node.ChildNodes.Count-1 do
                        if UTF8pos('system:template', Node.ChildNodes.Item[J].TextContent) > 0 then
                                NoteP^.IsTemplate := True;
                    for J := 0 to Node.ChildNodes.Count-1 do
                        if UTF8pos('system:notebook', Node.ChildNodes.Item[J].TextContent) > 0 then begin
                            EnterCriticalSection(CriticalSection);
                            try
                                TheLister.NoteBookList.Add(Filename, UTF8Copy(Node.ChildNodes.Item[J].TextContent, 17, 1000), NoteP^.IsTemplate);
                            finally
                                LeaveCriticalSection(CriticalSection);
                            end;
                            // debugln('Notelister #691 ' +  UTF8Copy(Node.ChildNodes.Item[J].TextContent, 17,1000));
                        end;
                                // Node.ChildNodes.Item[J].TextContent) may be something like -
                                // * system:notebook:DavosNotebook - this note belongs to DavosNotebook
                                // * system:template - this note is a template, if does not also have a
                                // Notebook tag its the StartHere note, otherwise its the Template for
                                // for the mentioned Notebook.
		        end;
            except 	on E: EXMLReadError do begin                                // Invalid XML
                                EnterCriticalSection(CriticalSection);
                                try
                                    DebugLn('XML ERROR ' + E.Message);
                                    Debugln('Offending File ' + Dir + FileName);
                                    XMLError := True;
                                    dispose(NoteP);
                                    TheLister.ErrorNotes.Append(FileName + ', ' + E.Message);
								finally
                                    LeaveCriticalSection(CriticalSection);
								end;
                                exit();
							end;
                        on E: EObjectCheck do begin                             // Triggered by, valid xml but empty field accessed
                                EnterCriticalSection(CriticalSection);
                                try
                                    DebugLn('XML ERROR ' + E.Message + ' - ' + PossibleError);
                                    Debugln('Offending File ' + Dir + FileName);
                                    XMLError := True;
                                    dispose(NoteP);
                                    TheLister.ErrorNotes.Append(FileName + ', ' + E.Message + ' - ' + PossibleError);
                        		finally
                                    LeaveCriticalSection(CriticalSection);
                        		end;
                                exit();
                        	end;
            		    on E: EAccessViolation do begin                         // I don't think we see this happen, but just in case
                            EnterCriticalSection(CriticalSection);
                            try
//                                DebugLn('Access Violation ' + E.Message);
//                                Debugln('Offending File ' + Dir + FileName);
                                XMLError := True;
                                dispose(NoteP);
                                TheLister.ErrorNotes.Append(FileName + ', ' + E.Message);
                    		finally
                                LeaveCriticalSection(CriticalSection);
                    		end;
                            exit();
                        end;
            end;
                if NoteP^.IsTemplate then begin    // Don't show templates in normal note list
                    dispose(NoteP);
                    exit();
			    end;
                EnterCriticalSection(CriticalSection);
                try
                    NoteList.Add(NoteP);
				finally
                    LeaveCriticalSection(CriticalSection);
				end;
		finally
      	        Doc.free;
  	    end;
    end else DebugLn('Error, found a note and lost it ! ' + Dir + FileName);
end;


procedure TNoteLister.IndexThisNote(const ID: String);
// While not using threads, this method must init critical section because GetNoteDetails expects it.
// This is used to index imported, newly download synced notes and newly recovered (from backup) notes.
begin
    //DebugMode := True;
    //debugln('TNoteLister.IndexThisNote');
    InitCriticalSection(CriticalSection);                          // could be done in background thread but rarely called
    GetNoteDetails(WorkingDir, CleanFileName(ID), false, self);    // while single call, must setup critical
    // this should trigger a regeneration of Index Lists, otherwise,
    DoneCriticalSection(CriticalSection);
    //DebugMode := False;
end;


function TNoteLister.GetLastChangeDate(const ID: String) : string;
var
    index : integer;
    FileName : string;
    eStr : string = '';
begin
    Result := '';
    if not assigned(NoteList) then exit('');
    FileName := CleanFileName(ID);
    //for Index := 0 to NoteList.Count -1 do
    for Index := NoteList.Count -1 downto 0 do
        if NoteList.Items[Index]^.ID = FileName then begin
            exit(NoteList.Items[Index]^.LastChange);
	    //	debugln('NoteLister #759 from list '  + NoteList.Items[Index]^.LastChange);
        end;
    // if to here, did not find that ID in Notes List. I wonder if its a Notebook ?
    if FileExists(WorkingDir + ID + '.note') then begin
        Result := GetNoteLastChangeSt(WorkingDir + ID + '.note', EStr);
        if EStr <> '' then
                  DebugLn('TGithubSync.LocalLastChangeDate - detected error in ' + ID);
    end;
end;

function TNoteLister.GetTitle(const ID: String) : string;
var
    index : integer;
    FileName : string;
begin
    Result := '';
    if not assigned(NoteList) then exit('');
    FileName := CleanFileName(ID);
    for Index := NoteList.Count -1 downto 0 do
    //for Index := 0 to NoteList.Count -1 do
        if NoteList.Items[Index]^.ID = FileName then
            exit(NoteList.Items[Index]^.Title);
end;

function TNoteLister.IsIDPresent(ID: string): boolean;
var
    FileName : string;
    index : integer;
begin
    Result := False;
    FileName := CleanFileName(ID);
    for Index := NoteList.Count -1 downto 0 do
    //for Index := 0 to NoteList.Count -1 do
        if NoteList.Items[Index]^.ID = FileName then
            exit(True);
end;

function TNoteLister.FindFirstOpenNote(out OpenIndex : integer): TForm;
begin
    OpenNoteIndex:=0;
    while OpenNoteIndex < NoteList.Count do
        if NoteList.Items[OpenNoteIndex]^.OpenNote <> nil then begin
            OpenIndex := OpenNoteIndex;
            exit(NoteList.Items[OpenNoteIndex]^.OpenNote)
        end else inc(OpenNoteIndex);
    result := nil;
    OpenNoteIndex := -1;
    OpenIndex := OpenNoteIndex;
end;

function TNoteLister.FindNextOpenNote(out NoteIndex : integer): TForm;
begin
    if OpenNoteIndex < 0 then exit(Nil);
    inc(OpenNoteIndex);
    while OpenNoteIndex < NoteList.Count do begin
        if NoteList.Items[OpenNoteIndex]^.OpenNote <> nil then begin
            NoteIndex := OpenNoteIndex;
            exit(NoteList.Items[OpenNoteIndex]^.OpenNote)
        end
        else inc(OpenNoteIndex);
    end;
    result := nil;
    OpenNoteIndex := -1;
    NoteIndex := OpenNoteIndex;
end;

function TNoteLister.FindFirstOOSNote(out NTitle, NID : ANSIstring): boolean;
begin
    OpenNoteIndex:=0;
    while OpenNoteIndex < NoteList.Count do
        if NoteList.Items[OpenNoteIndex]^.OpenOnStart then begin
            NTitle := NoteList.Items[OpenNoteIndex]^.Title;
            NID := NoteList.Items[OpenNoteIndex]^.ID;
            exit(True)
        end
        else inc(OpenNoteIndex);
    result := False;
    OpenNoteIndex := -1;
end;

function TNoteLister.FindNextOOSNote(var NTitle, NID : ANSIstring): boolean;
begin
    if OpenNoteIndex < 0 then exit(False);
    inc(OpenNoteIndex);
    while OpenNoteIndex < NoteList.Count do
        if NoteList.Items[OpenNoteIndex]^.OpenOnStart then begin
            NTitle := NoteList.Items[OpenNoteIndex]^.Title;
            NID := NoteList.Items[OpenNoteIndex]^.ID;
            exit(True)
        end
        else inc(OpenNoteIndex);
    result := False;
    OpenNoteIndex := -1;
end;


// ----------------- Search Related Methods -----------------------------------

{ The two indexes are sorted by title or date. We iterate over both removing any
  entry that points to note that does not have a match for search term(s). Then the
  listview is redrawn using the rebuilt index.
}

function TNoteLister.RefineSearch(STermList: TstringList; TitleOnly : boolean): integer;
//var
//    T1, T2, T3, T4, T5 : qword;
begin
    // We remove from the indexes any entries that correspond to NoteList entries
    // that don't match the passed search term.
    //T1 := GetTickCount64();
    result := 0;
    // Iterate over the Index, for each entry, the value stored is the index into NoteList
    while result < TitleSearchIndex.Count do begin
        if not CheckSearchTerms(STermList, TitleSearchIndex[result], TitleOnly) then
            TitleSearchIndex.Delete(result)
        else inc(result);
    end;
    result := 0;
    //T2 := GetTickCount64();
    while result < DateSearchIndex.count do begin
        if not CheckSearchTerms(STermList, DateSearchIndex[result], TitleOnly) then
            DateSearchIndex.Delete(result)
        else inc(result);
    end;
    //T3 := GetTickCount64();
    //debugln('TNoteLister.RefineSearch() ' + inttostr(T2-T1) + 'mS ' + inttostr(T3-T2) + 'mS');
end;

function TNoteLister.BuildDateAllIndex() : integer;
var
    i : integer;
begin
    Result := 0;
    if DateAllIndex = nil then DateAllIndex := TSortList.Create
    else DateAllIndex.Clear;
    for i := 0 to NoteList.Count-1 do begin
        if not NoteList[i]^.IsTemplate then begin
            DateAllIndex.Add(i);
            inc(Result);
        end;
    end;
    DateAllIndex.sort(@SortOnDate);
end;

{x$define PROFILEINDEX}

{ This should ret the number of items, not the zero based index of the last item.
So, if we do '0', one pass, it should ret 1
If the list is empty, ret 0;
}
function TNoteLister.ClearSearch(): integer;
var
    i : integer;
    {$ifdef PROFILEINDEX} T1, T2, T3, T4, T5, T6 : qword; {$endif}
begin
    {$ifdef PROFILEINDEX}T1 := GetTickCount64();{$endif}
    //DumpNoteNoteList('Called from ClearSearch');
    result := 0;
    if TitleSearchIndex = nil then TitleSearchIndex := TSortList.Create
    else TitleSearchIndex.Clear;
    if DateSearchIndex = nil then DateSearchIndex := TSortList.Create
    else DateSearchIndex.Clear;
    for i := 0 to NoteList.Count-1 do begin
        if not NoteList[i]^.IsTemplate then begin
            TitleSearchIndex.add(i);
            DateSearchIndex.Add(i);
            inc(Result);
        end;
    end;
    {$ifdef PROFILEINDEX}T2 := GetTickCount64();{$endif}
    TitleSearchIndex.sort(@SortOnTitle);                     // My 2017 Dell, 1mS, 2K notes
    {$ifdef PROFILEINDEX}T3 := GetTickCount64();{$endif}
    DateSearchIndex.sort(@SortOnDate);           // Ends up with the most recent at the bottom of list
    {$ifdef PROFILEINDEX}T4 := GetTickCount64();{$endif}     // My 2017 Dell, < 1mS, 2K notes
    {$ifdef PROFILEINDEX}debugln('TNoteLister.ClearSearch() build=' + inttostr(T2-T1)
                + 'mS Tsort=' + inttostr(T3-T2) + 'mS DSort=' + inttostr(T4-T3) );
    debugln('Top ' + ' ' + NoteList[TitleSearchIndex[0]]^.LastChange);
    debugln('Bot ' + ' ' + NoteList[TitleSearchIndex[TitleSearchIndex.Count-1]]^.LastChange);  {$endif}
    // debugln('TNoteLister.ClearSearch() returning ' + inttostr(REsult) + ' and ' + inttostr(NoteList.Count));
end;

function TNoteLister.CheckSearchTerms(const STermList : TStringList; const Index : integer; TitleOnly : boolean) : boolean; inline;
var
    St : string;
begin
//    if NoteList[index]^.Title = 'tomboy-ng Release Process' then
//        debugln('TNoteLister.CheckSearchTerms - found tomboy-ng Release Process');
    for St in STermList do begin
        if St = '' then continue;
        if TitleOnly then begin
             if pos(St, lowercase(NoteList[index]^.Title)) = 0 then exit(False)
        end
        else if pos(St, NoteList[index]^.Content) = 0 then exit(False);
    end;
    result := true;
end;
// OK, above is where we decide to search on note title instead of note content.

function TNoteLister.NewSearch(STerm : string; NoteBook: string; TitleOnly : boolean): integer;
var
    STL : TStringList;
begin
    STL := TStringList.Create;
    if not ((STerm = '') {or (STerm = rsMenuSearch)}) then              // else we pass an empty list if no valid search term
        if Sett.SearchCaseSensitive then
            STL.AddDelimitedtext(STerm, ' ', false)
        else STL.AddDelimitedtext(lowercase(STerm), ' ', false);
    result := NewSearch(STL, NoteBook, TitleOnly);
    STL.Free;
end;

{ NewSearch can be called in three modes -
  Just STermList, Just a Notebook, or Both. Not neither.     }

function TNoteLister.NewSearch(STermList : TstringList; NoteBook: string; TitleOnly : boolean): integer;
var
    NBStrL : TStringList = nil;        // gets set to a pre-existing list, do not create or free !
    //T1, T2, T3, T4, T5 : qword;
    // St : string;

    function SearchNoteBook() : integer;
    var
        i, j : integer;
    begin
        result := 0;
        j:= 0;
        if GetNotesInNoteBook(NBStrL, NoteBook) then begin         // check each line in NBStrL for this note instance
            //writeln('SearchNoteBook NB=' + NoteBook + ' NBStrL.text=[' + NBStrL.Text + ']');
            while j < NBStrL.Count do begin
                if NoteList.FindID(i, NBStrL[j]) then begin        // this the time consuming part. 4mS 2000 notes
                    if CheckSearchTerms(STermList, i, TitleOnly) then begin
                        TitleSearchIndex.add(i);
                        DateSearchIndex.Add(i);
                        inc(Result);
                    end;
                    inc(j);
                end;
            end;
        end;
    end;

    function OnlySTerm() : integer;
    var
        ii : integer;
    begin
        result := 0;;
        for ii := 0 to NoteList.Count-1 do begin
            if not CheckSearchTerms(STermList, ii, TitleOnly) then continue;
            if not NoteList[ii]^.IsTemplate then begin
                TitleSearchIndex.add(ii);
                DateSearchIndex.Add(ii);
                inc(Result);
            end;
        end;
    end;


begin
    // debugln('TNoteLister.NewSearch() STerm=' + STermList.Text + ' >> ' + NoteBook);
    // T1 := GetTickCount64();
    TitleSearchIndex.Clear;
    DateSearchIndex.Clear;
    TitleSearchIndex.Capacity := NoteList.Count;
    DateSearchIndex.Capacity :=  NoteList.Count;
    //T2 := GetTickCount64();
    if (NoteBook <> '')  then
        Result := SearchNoteBook()
    else Result := OnlySTerm();

//    TitleSearchIndex.pack();                                    // Causes an unidentified crash after GTK loop resumes....
//    DateSearchIndex.pack();                                     // Pack removes nil pointers in list. I don't think we have any.

    //T3 := GetTickCount64();
    TitleSearchIndex.sort(@SortOnTitle);                          // Note : running each sort in its own thread ?  Only two
    //T4 := GetTickCount64();                                     // and we have to wait here until they end anyway, I think not.
    DateSearchIndex.sort(@SortOnDate);

    //T5 := GetTickCount64();
    {debugln('TNoteLister.NewSearch() clear=' + inttostr(T2-T1) + 'mS Search='
            + inttostr(T3-T2) + 'mS ' + ' TSort=' + inttostr(T4-T3)
            + ' mS DSort=' + inttostr(T5-T4) + ' ' + inttostr(TitleSearchIndex.Count));   }

(*    debugln('as above -- ' + STermList.text + ' -- ' + Notebook);
    for St in STermList do
        debugln('TNoteLister.NewSearch() STermList Item [' + St + ']');
    debugln('as above -- content >> ' + copy(NoteList[0]^.Content, 50, 1));       *)

{    debugln(' TNoteLister.NewSearch()  Top Index=' + NoteList[DateSearchIndex[0]]^.LastChange);
    debugln(' TNoteLister.NewSearch Botton Index=' + NoteList[DateSearchIndex[DateSearchIndex.Count - 1]]^.LastChange);
    }
end;

function TNoteLister.NoteIndexCount(): integer;
begin
    if TitleSearchIndex = nil then
        result := 0
    else result := TitleSearchIndex.Count;
end;

// Pass this function a TStringList each line of which must be matched for a 'hit'
// Moved out of class so that the threaded search can find and use it.

function NoteContains(const TermList : TStringList; FullFileName: ANSIString; const CaseSensitive : boolean): boolean;
var
    SLNote : TStringList;
    I, Index : integer;
begin
    Result := False;
    SLNote := TStringList.Create;
    SlNote.LoadFromFile(FullFileName);
    for Index := 0 to SLNote.Count - 1 do
        SLNote.Strings[Index] := RemoveXML(SLNote.Strings[Index]);
    for I := 0 to TermList.Count -1 do begin      // Iterate over search terms
        Result := False;
        for Index := 0 to SLNote.Count - 1 do begin // Check each line of note for a match against current word.
            if  CaseSensitive then begin
                if (UTF8Pos(TermList.Strings[I], SLNote.Strings[Index]) > 0) then begin
                    Result := True;
                    break;
                end;
            end else
                if (UTF8Pos(UTF8LowerString(TermList.Strings[I]), UTF8LowerString(SLNote.Strings[Index])) > 0) then begin
                    Result := True;
                    break;
                end;
        end;
        if not Result then break;  // if failed to turn Result on for first word, no point in continuing
    end;
    // when we get here, if Result is true, run finished without a fail.
    FreeandNil(SLNote);
end;

{ With 2000 notes, on my Dell, linux, search for 'and'.
  Before multithreading - 250mS - 280mS
  With Multithreading, cthreads and cmem -
  6 : 90ms to 110ms; 4 : 100mS - 134ms; 3 : 110ms - 130mS; 2 : 155ms - 180ms; 1 : 255ms - 280mS
  However, noted on Windows Vista (!), significent slow down !  Windows10, similar to Linux
  Under new search model, August 2022, down to 68mS !
}

const ThreadCount = 3;  // The number of extra threads set searching. 3 seems reasonable...


procedure TNoteLister.UnLoadContent();
var
    P : PNote;
begin
    for P in NoteList do
        P^.Content := '';
end;

function TNoteLister.LoadContentForPressEnter() : longint;
var
    //P : PNote;
    ThreadIndex : integer = 0;
    SearchThread : TGetContentThread;
begin
    if EnterDateSearchIndex = Nil then
        EnterDateSearchIndex := TSortList.Create
    else EnterDateSearchIndex.Clear;
    if EnterTitleSearchIndex = Nil then
        EnterTitleSearchIndex := TSortList.Create
    else EnterTitleSearchIndex.clear;
//    for P in NoteList do begin
        FinishedThreads := 0;
        ThreadLock := -1;
        while ThreadIndex < ThreadCount do begin
            SearchThread := TGetContentThread.Create(True);        // Threads clean themselves up.
            SearchThread.NoteLister := self;
            SearchThread.ThreadBlockSize := NoteList.Count div ThreadCount;
            // SearchThread.Term_List := Terms;
            SearchThread.WorkDir := WorkingDir;
            //SearchThread.ResultsList1 := EnterDateSearchIndex;
            //SearchThread.ResultsList2 := EnterTitleSearchIndex;
            SearchThread.TIndex := ThreadIndex;
            {$ifdef TOMBOY_NG}
            //SearchThread.CaseSensitive := Sett.SearchCaseSensitive;
            {$endif}
            SearchThread.start();
            inc(ThreadIndex);
        end;
        while FinishedThreads < ThreadCount do sleep(1);       // ToDo : some sort of 'its taken too long ..."
        // SearchNoteList.Sort(@LastChangeSorter);

//    end;
    EnterDateSearchIndex.sort(@SortOnDate);
    EnterTitleSearchIndex.sort(@SortOnTitle);
	result := EnterDateSearchIndex.Count;
end;


procedure TNoteLister.SearchContent(const St : string; Stl : TstringList);
// St is Title of note asking for search. We seach all note's content for St but it must
// surrounded by suitable deliminators. Hard to see this method used for anything else
var
    i, TitleL, Offset, FoundIndex, CLen : integer;
    Title : string;
begin
    TitleL := length(St);
    for i := 0 to NoteList.Count -1 do begin                                    // Loop over whole notelist
        Title := NoteList.Items[i]^.TitleLow;
        if St = Title then
            Continue;
        CLen := length(NoteList.Items[i]^.Content);
        FoundIndex := 0;                                                        // Note : NoteContent is NOT always lowercase, do need to lowercase() it here.
        Offset := 0;                                                            // Could check Sett.SearchCaseSensitive but seems fast enough at present.
        while ((Offset+TitleL) < CLen) and (FoundIndex > -1) do begin           // loop over Content, break out if found a hit or got to end
            FoundIndex := (lowercase(NoteList.Items[i]^.Content)).IndexOf(St, Offset);                      // Zero base result. -1 if not present
            if FoundIndex > -1 then begin                                       // FoundIndex cannot be zero, that would be title, checked above
                if  ((NoteList.Items[i]^.Content)[FoundIndex] in [' ', #10, ',', '.'])                      // char before potential target
                    and (((FoundIndex+TitleL) = CLen)                                                       // nothing in note after link content
                    or  ((NoteList.Items[i]^.Content)[FoundIndex+TitleL+1] in [#10, ' ', ',', '.'])) then   // char after potential target
                        begin
                            Stl.Add(NoteList.Items[i]^.Title);
                            FoundIndex := -1;           // force our way out of inner loop
                            continue;                   // On to the next title
                        end;
            end;                                        // Nothing to see here folks.
            OffSet := Offset + FoundIndex + 1;          // +1 to move past just tested one
        end;
    end;
end;

procedure TNoteLister.AddNote(const FileName, Title, LastChange : ANSIString);
var
    NoteP : PNote;
begin
    new(NoteP);
    NoteP^.IsTemplate := False;
    NoteP^.ID := CleanFilename(FileName);
    NoteP^.LastChange := LastChange; {copy(LastChange, 1, 19); }
    //NoteP^.LastChange[11] := ' ';
    NoteP^.CreateDate := LastChange; {copy(LastChange, 1, 19); }
    //NoteP^.CreateDate[11] := ' ';
    NoteP^.Title:= Title;
    NoteP^.TitleLow:= lowercase(Title);
    NoteP^.OpenNote := nil;
    NoteList.Add(NoteP);
    // We don't need to re-sort here, the new note is added at the end, and our
    // list is sorted, newest towards the end. All good.
end;

function TNoteLister.GetTitle(Index : integer) : string;
begin
    Result := PNote(NoteList.get(Index))^.Title;
end;

function TNoteLister.GetNote(Index : integer) : PNote;          // ToDo : Used by sync, might be faster to just give it access to NoteList
begin
    Result := nil;
    if (Index > -1) and (Index < NoteList.Count) then
        Result := NoteList[Index];
end;

function TNoteLister.GetNote(Index: integer; mode: TLVSortMode): PNote;
begin
    Result := Nil;
    if NoteList.Count <= Index then exit;
    case Mode of
        smRecentDown  : result := NoteList[DateSearchIndex[Index]];
        smRecentUp    : result := NoteList[DateSearchIndex[DateSearchIndex.Count - Index -1]];
        smAATitleUp   : result := NoteList[TitleSearchIndex[Index]];
        smAATitleDown : result := NoteList[TitleSearchIndex[TitleSearchIndex.Count - Index -1]];
        smAllRecentUp :
                result := NoteList[DateAllIndex[DateAllIndex.Count - Index -1]];

    end;
end;


function TNoteLister.IndexNotes(DontTestName : boolean = false): longint;
var
    //Info : TSearchRec;
    cnt : integer = 4;
    IndexThread : TIndexThread;
    //T1, T2, T3, T4, T5 : qword;
    //i : integer;
begin
    // DebugMode := true;                           // ToDo : remove
    //T1 := gettickcount64();
    XMLError := False;
    if DontTestName then begin
        cnt := 1;      // Just one thread.
        FinishedThreads := 3;
	end else FinishedThreads := 0;
    if NoteList <> nil then NoteList.Free;
    NoteList := TNoteList.Create;
    if NoteBookList <> nil then NoteBookList.Free;
    NoteBookList := TNoteBookList.Create;
    if DebugMode then debugln('Empty Note and Book Lists created');
    FreeandNil(ErrorNotes);
    ErrorNotes := TStringList.Create;
    if DebugMode then debugln('IndexNotes : Looking for notes in [' + WorkingDir + ']');

    InitCriticalSection(CriticalSection);                  // +++++++++++
    {$ifdef FORCE_SINGLE_INDEX_THREAD}                     // This is a testing thing.
    Cnt := 1;
    FinishedThreads := 3;
    debugln('TNoteLister.IndexNotes running one thread mode');
    {$endif}
//    debugln('');                                           // This may be necessary to initialise logger BEFORE going multithread
    while Cnt > 0 do begin
        //debugln('Making thread ' + inttostr(Cnt));
        IndexThread := TIndexThread.Create(True);          // Threads clean themselves up.
        IndexThread.GetNoteDetailsProc := @GetNoteDetails; // pass the address of the proc to the Thread class.
        IndexThread.WorkingDir := WorkingDir;
        IndexThread.OneThread := DontTestName;
        IndexThread.TheLister := self;
        case Cnt of                                        // This is ignored in OneThread mode
            {$ifdef WINDOWS}
            1 : IndexThread.StartsWith:= ['0', '1', '2', '3'];
            2 : IndexThread.StartsWith:= ['4', '5', '6', '7'];
            3 : IndexThread.StartsWith:= ['8', '9', 'A', 'B'];
            4 : IndexThread.StartsWith:= ['C', 'D', 'E', 'F'];
            {$else}
            1 : IndexThread.StartsWith:= ['0', '1', '2', '3', '4'];
            2 : IndexThread.StartsWith:= ['5', '6', '7', '8', '9'];
            3 : IndexThread.StartsWith:= ['a', 'B', 'c', 'D', 'e', 'F'];
            4 : IndexThread.StartsWith:= ['A', 'b', 'C', 'd', 'E', 'f'];
            {$endif}
		end;
        IndexThread.start();
        dec(Cnt);
    end;
    while FinishedThreads < 4 do sleep(1);       // ToDo : some sort of 'its taken too long ..."
    DoneCriticalSection(CriticalSection);                  // ++++++++++++
    if ErrorNotes.Count > 0 then begin
        debugln('TNoteLister.IndexNotes One or more notes have error.');
        debugln(ErrorNotes.Text);
        debugln('-------------------------------------------');
    end;
    if DebugMode then begin
        debugLn('Finished indexing notes');
        DumpNoteNoteList('TNoteLister.IndexNotes');
    end;
    NotebookList.CleanList();
    Result := NoteList.Count;
    //T2 := gettickcount64();
    NoteBookList.Sort(@NotebookSorter);
    //T3 := gettickcount64();
    //debugln('TNoteLister.IndexNotes() called for ' + WorkingDir + ' count=' + inttostr(NoteList.Count));
    if TheMainNoteLister = self then
        BuildDateAllIndex();
    //if TheMainNoteLister <> nil then BuildDateAllIndex();                           // Initialized below, set in Searchform, must be the MAIN notelister
    //T4 := gettickcount64();
    //debugln('TNoteLister.IndexNotes read=' + inttostr(T2-T1) + 'mS, NoteBook Sort= ' + inttostr(T3-T2) + ' DateAllIndex=' + inttostr(T4-T3));
end;

procedure TNoteLister.LoadStGrid(const Grid : TStringGrid; NoCols : integer; SearchMode : boolean = false);
var
    Index : integer;                                       // used by Snapshot manager. Sigh ....
    TheList : TNoteList;
    LCDst : string;
    CDst  : string;
    //T1, T2, T3 : qword;
begin
    //T1 := gettickcount64();
    if SearchMode then
        TheList := SearchNoteList
    else TheList := NoteList;
    while Grid.RowCount > 1 do Grid.DeleteRow(Grid.RowCount-1);
    //T2 := gettickcount64();
    Index := TheList.Count;
    while Index > 0 do begin
        dec(Index);
        LCDst := TheList.Items[Index]^.LastChange;
        if length(LCDst) > 11 then  // looks prettier, dates are stored in ISO std
            LCDst[11] := ' ';       // with a 'T' between date and time
        if length(LCDst) > 16 then
            LCDst := copy(LCDst, 1, 16);    // we only want hours and minutes

        CDst := TheList.Items[Index]^.CreateDate;
        if length(CDst) > 11 then
            CDst[11] := ' ';
        if length(CDst) > 16 then
            CDst := copy(CDst, 1, 16);

        case NoCols of
            2 : Grid.InsertRowWithValues(Grid.RowCount, [TheList.Items[Index]^.Title, LCDst]);
            3 : Grid.InsertRowWithValues(Grid.RowCount, [TheList.Items[Index]^.Title,
        	    LCDst, CDst]);
            4 : Grid.InsertRowWithValues(Grid.RowCount, [TheList.Items[Index]^.Title,
                LCDst, CDst, TheList.Items[Index]^.ID]);
        end;
    end;

    if Grid.SortColumn > -1 then
        Grid.SortColRow(True, Grid.SortColumn);
    // T3 := gettickcount64();
    // debugln('Note_Lister - LoadStGrid ' + inttostr(T2 - T1) + ' ' + inttostr(T3 - T2));
end;
(*
function TNoteLister.NewLVItem(const LView : TListView; const Title, DateSt, FileName: string): TListItem;
var
    TheItem : TListItem;
    DT : TDateTime;
begin
   TheItem := LView.Items.Add;
   TheItem.Caption := Title;
   if MyTryISO8601ToDate(DateSt, DT) then
        TheItem.SubItems.Add(MyFormatDateTime(DT, True) + ' ')
   else TheItem.SubItems.Add('ERROR bad date string ');
   TheItem.SubItems.Add(FileName);
   Result := TheItem;
end;

procedure TNoteLister.LoadListView(const LView : TListView; const SearchMode : boolean);
var
    Index : integer;
    TheList : TNoteList;
    //LCDst : string;
    //T1, T2, T3 : qword;
    // Full list mode, 2000 notes, Dell 7mS to clear, 20-40mS to load.
begin
    //T1 := gettickcount64();
    LView.Clear;
    if SearchMode then
        TheList := SearchNoteList
    else TheList := NoteList;
    //T2 := gettickcount64();
    Index := TheList.Count;
    while Index > 0 do begin
        dec(Index);
        NewLVItem(LView, TheList.Items[Index]^.Title, TheList.Items[Index]^.LastChange, TheList.Items[Index]^.ID);
    end;
    //T3 := gettickcount64();
    //debugln('TNoteLister.LoadListView Clear=' + dbgs(T2 - T1) + ' Fill=' + dbgs(T3 - T2));
end;
*)

procedure TNoteLister.LoadStrings(const TheStrings: TStrings);
var
    Index : integer;
begin
    Index := NoteList.Count;
    while Index > 0 do begin
        dec(Index);
        TheStrings.AddObject(NoteList.Items[Index]^.Title, tObject(NoteList.Items[Index]^.ID));
    end;
end;

{ New model, Aug 2022, now need to manage the Index Files.
  We will deal with what ever is needed here, alter existing, create new.
  We need to ret a value that tells SearchUnit it needs to -
  1. Do nothing
  2. update display.
  3. rerun last search (and, update display)
  Ret value is either do nothing (false) or do something, True.
  We pass an Out Var, that will be false if the something is just update display
  and True if its a full rerun of search.  ReRunSearch : boolean.

  This function first looks for the note ID in NoteList, if its there,
  we should update LCD and then look to see if its in Title has changed.
  If it has, rebuild DateAllIndex and return True. SearchUnit will trigger a
  repeat search.

  If its just a LCD job, we look to see if its mentioned in DateSearchIndex, if
  not, ret false. Its its there, ret true and set ReRunSearch to False.

  If its a new note, add to NoteList, update DateAllIndex, return True and set
  ReRunSearch=True.

                Present  Ret  ReRun  Actions
  Just LCD      No       f    f      Update NoteList, update DateAllIndex
  Just LCD      Yes      T    f      Update NoteList, update DateAllIndex and DateSearchIndex
  LCD + Title   No       f    f      Update NoteList, update all 3 indexes
  LCD + Title   Yes      T    T      Update NoteList
  New Note               T    T      Add to NoteList (still might not be displayed but too hard to tell)

}
            // Note that FFname is always needed, LCD and Title note needed if JustIndex is true
function TNoteLister.AlterOrAddNote(out ReRunSearch : boolean; const FFName, LCD, Title : string) : boolean;
var
    ID : string;
    i : integer = 0;                   // Is an index into NoteList

    // If it finds the i entry in Index, moves it to last entry, that being most recent.
    function UpdateIndex(TheIndex : TSortList) : boolean;    // i is an index to NoteList
    var  j : integer;
    begin
        (* j := 0; Result := False;                          // False, if returned, means i does not exist in Index, reindex is needed
         while j < TheIndex.Count do begin
            if TheIndex[j] = i then begin
                TheIndex.Delete(j);
                TheIndex.add(i);
                exit(True);                                 // We found and entry in the index and updated it.
            end;
            inc(j);
        end;    *)
        j := TheIndex.Count; Result := False;               // False, if returned, means i does not exist in Index, reindex is needed
        while j > 0 do begin
            dec(j);
            if TheIndex[j] = i then begin
                TheIndex.Delete(j);
                TheIndex.add(i);
                exit(True);                                 // We found an entry in the index and updated it.
            end;
        end;
    end;

    { This gets called every time we update the datestamp on a note, so, as a user edits
    a note, they constently trigger a save. But the note is at the end of the Index after
    the first call, assume its called several time and start searching at end of list }

begin
    Result := False;
    ReRunSearch := True;
    ID := CleanFilename(FFName);

    while i < NoteList.Count do begin                   // try to find ID in NoteList
        if ID = NoteList.Items[i]^.ID then break;       // Found it
        inc(i);
    end;
    if i = NoteList.count then begin                    // Drop through, must be a new note.
        AddNote(ID, Title, LCD);                        // Add to NoteList
        DateAllIndex.Add(NoteList.Count -1);            // Added at the end of DateAllIndex, most recent.
        result := true;                                 // ReRunSearch is set to True so calling process must re run an existing search
    end else begin                                      // Else its already in NoteList, maybe just a LCD but possible also Title change
        NoteList.Items[i]^.LastChange := LCD;
        if Title = NoteList.Items[i]^.Title then begin  // Its just a LCD update, update DateSearchIndex and advise a re-Display
            ReRunSearch := False;
            result := UpdateIndex(DateSearchIndex);     // False if not currently DateSearchIndex, its not being displayed anyway
        end else begin                                  // Ah, a new title, must rerun search.
            NoteList.Items[i]^.Title := Title;
            NoteList.Items[i]^.TitleLow := lowercase(Title);
        end;
    end;
    UpdateIndex(DateAllIndex);                      // Always update DateAllIndex, its not done by Search methods
end;

function TNoteLister.AlterNote(ID, Change: ANSIString; Title: ANSIString): boolean;
var
    Index : integer;
begin
	result := False;
    for Index := NoteList.Count -1 downto 0 do begin
    //for Index := 0 to NoteList.Count -1 do begin
        if CleanFilename(ID) = NoteList.Items[Index]^.ID then begin
        	if Title <> '' then begin
            	NoteList.Items[Index]^.Title := Title;
                NoteList.Items[Index]^.TitleLow := lowercase(Title);
            end;
            if Change <> '' then begin
                NoteList.Items[Index]^.LastChange := Change;  {copy(Change, 1, 19);}
                // check if note is already at the bottom of the list, don't need to re-sort.
(*                if (Index < (NoteList.Count -1)) then
                    NoteList.Sort(@LastChangeSorter);     *)                    // we don't do this any more .....
            end;
            exit(True);
		end;
	end;
end;

function TNoteLister.IsThisATitle(const Title: ANSIString): boolean;
var
    Index : integer;
begin
  	Result := False;
    for Index := NoteList.Count -1 downto 0 do begin
	//for Index := 0 to NoteList.Count -1 do begin
        if Title = NoteList.Items[Index]^.TitleLow then begin
        	Result := True;
            break;
		end;
	end;
end;

function TNoteLister.CleanFileName(const FileOrID : AnsiString) : ANSIString;
begin
  	if length(ExtractFileNameOnly(FileOrID)) = 36 then
        Result := ExtractFileNameOnly(FileOrID) + '.note'
    else
        Result := ExtractFileNameOnly(FileOrID);
end;

function TNoteLister.IsThisNoteOpen(const ID: ANSIString; out TheForm : TForm): boolean;
var
    Index : integer;
begin
  	Result := False;
    TheForm := Nil;
    for Index := NoteList.Count -1 downto 0 do begin
	//for Index := 0 to NoteList.Count -1 do begin
        if CleanFileName(ID) = NoteList.Items[Index]^.ID then begin
        	TheForm := NoteList.Items[Index]^.OpenNote;
            Result := not (NoteList.Items[Index]^.OpenNote = Nil);
            break;
		end;
	end;
end;

function TNoteLister.ThisNoteIsOpen(const ID : ANSIString; const TheForm: TForm) : boolean;
var
    Index : integer;
    //cnt : integer;
    JustID : string;
begin
    result := false;
    if NoteList = NIl then
        exit;
    JustID := CleanFileName(ID);
    if NoteList.Count < 1 then begin
        //DebugLn('Called ThisNoteIsOpen() with empty but not NIL list. Count is '
        //		+ inttostr(NoteList.Count) + ' ' + ID);
        // Occasionally I think we see a non reproducable error here.
        // I believe is legal to start the for loop below with an empty list but ....
        // When we are creating the very first note in a dir, this happens. Count should be exactly zero.
	end;
	//cnt := NoteList.Count;
    for Index := NoteList.Count -1 downto 0 do begin
	// for Index := 0 to NoteList.Count -1 do begin
      	//writeln('ID = ', ID, ' ListID = ', NoteList.Items[Index]^.ID);
        if JustID = NoteList.Items[Index]^.ID then begin
            NoteList.Items[Index]^.OpenNote := TheForm;
            exit(true);
		end;
	end;
    // if Index = (NoteList.Count -1) then DebugLn('Failed to find ID in List ', ID);
end;

function TNoteLister.FileNameForTitle(const Title: ANSIString; out FileName : ANSIstring): boolean;
var
    Index : integer;
begin
    FileName := '';
  	Result := False;
    for Index := NoteList.Count -1 downto 0 do begin
	//for Index := 0 to NoteList.Count -1 do begin
        if lowercase(Title) = lowercase(NoteList.Items[Index]^.Title) then begin
            FileName := NoteList.Items[Index]^.ID;
        	Result := True;
            break;
		end;
	end;
end;

procedure TNoteLister.StartSearch();
begin
	SearchIndex := 0;
end;

function TNoteLister.NextNoteTitle(out SearchTerm: ANSIString): boolean;
begin
  	Result := False;
	if SearchIndex < NoteList.Count then begin
    	SearchTerm := NoteList.Items[SearchIndex]^.Title;
    	inc(SearchIndex);
        Result := True;
	end;
end;

function TNoteLister.DeleteNote(const ID: ANSIString): boolean;
var
    Index : integer;
    JustID : string;
begin
	result := False;
    JustID := CleanFileName(ID);
    //DebugLn('TNoteLister.DeleteNote - asked to delete ', ID);
    for Index := NoteList.Count -1 downto 0 do begin
    //for Index := 0 to NoteList.Count -1 do begin
        if JustID = NoteList.Items[Index]^.ID then begin
        	dispose(NoteList.Items[Index]);
        	NoteList.Delete(Index);
        	Result := True;
        	break;
		end;
	end;
    if Result = false then
        DebugLn('Failed to remove ref to note in NoteLister ', ID);
end;

constructor TNoteLister.Create;
begin
    SearchNoteList := nil;
    NoteList := nil;
    NoteBookList := Nil;
    ErrorNotes := Nil;
    DateSearchIndex := Nil;
    TitleSearchIndex := Nil;
    DateAllIndex := nil;
    EnterDateSearchIndex  := Nil;
    EnterTitleSearchIndex := Nil;

end;


destructor TNoteLister.Destroy;
begin
    SearchNoteList.Free;
    SearchNoteList := Nil;

    FreeAndNil(NoteBookList);
    FreeAndNil(NoteList);
    FreeAndNil(ErrorNotes);
    FreeAndNil(DateAllIndex);
    FreeAndNil(DateSearchIndex);
    FreeAndNil(TitleSearchIndex);
    FreeAndNil(EnterDateSearchIndex);
    FreeAndNil(EnterTitleSearchIndex);
	inherited Destroy;
end;

{  =========================  TNoteList ====================== }


destructor TNoteList.Destroy;
var
  I : integer;
begin
	for I := 0 to Count-1 do begin
    	dispose(Items[I]);
    end;
    inherited Destroy;
end;

function TNoteList.Add(ANote: PNote): integer;
begin
    result := inherited Add(ANote);
end;

function TNoteList.FindID(out Index:integer; const ID: ANSIString): boolean;
begin
    Result := False;
    Index := 0;
    while Index < Count do begin
        if Items[Index]^.ID = ID then begin
            Result := True;
            exit();
		end;
        inc(Index);
    end;
end;


function TNoteList.FindID(const ID: ANSIString): PNote;
var
    Index : longint;
begin
    Result := Nil;
    for Index := Count-1 downto 0 do begin
    //for Index := 0 to Count-1 do begin
        if Items[Index]^.ID = ID then begin
            Result := Items[Index];
            exit()
		end;
	end;
end;

function TNoteList.Get(Index: integer): PNote;
begin
    Result := PNote(inherited get(Index));
end;

//initialization                           // done in SearchUnit
//    TheMainNoteLister := Nil;            // A global that points to the main note list.

end.

