index.hpp

Go to the documentation of this file.

//==================================================================================================
/// \file
/// This header-file is part of \dxl - A doxygen post-processor that allows to define smarter
/// <b>Doxygen</b>-links.
///
/// \emoji :copyright: 2025-2026 A-Worx GmbH, Germany.
/// Published under \ref mainpage_license "Boost Software License".
//==================================================================================================
#ifndef HPP_DXL_INDEX
#define HPP_DXL_INDEX
#pragma once
#include "target.hpp"
#include "threadpool.hpp"
#include "ALib.Strings.StdIOStream.H"
#include "ALib.ThreadModel.H"
#include "ALib.Files.H"
#include <fstream>
 
/// A preprocessor constant defined in the file #"F;index.hpp".
#define TEST_CONSTANT  5
#undef  TEST_CONSTANT
 
namespace dxl {
class XLink;
class DoxygenXLinks;
 
/// Denotes whether to find inherited members..
extern bool     FindInheritedMembers;
 
// TODO(251222 17:52): ^^^^ make configurable
//    And: if all is done, disable with ALib dox and instead change links everywhere
//         to XLink to base types and insert the word "inherited" into the documentation.
//         This is much nicer for the reader of a manual.
//    Then: Add a chapter to the DXL manual that it is recommended to switch this off.
//    Or: Change display to automatically display first parent of the path.
 
 
/// This class holds an #"StringTree;alib::StringTree" of all tags found in a doxygen tag-file.
/// In addition, a non-unique hash-map is created that allows to find entities in O(1).
class Index : public alib::containers::StringTree< alib::MonoAllocator,
                                                   Target*,
                                                   alib::containers::StringTreeNamesAlloc<> > {
  friend class DoxygenXLinks;
 
  protected:
    /// The type of the string tree that holds the tag-file.
    using TreeType     = StringTree;
 
    /// The allocator used for the parent #"StringTree" and for the hashtable in the field #map.
    alib::MonoAllocator ma;
 
    /// A pod-like struct providing the key for the hash table found in the field #"Index::anchorMap;2".
    struct AnchorKey {
        const alib::String&  file;   ///< The HTML file name that the anchor is defined in.
        const alib::String&  anchor; ///< The anchor name.
 
        /// Equality operator for AnchorKey.
        /// @param other The anchor to compare with.
        /// @return The result of the comparison.
        bool operator==(const AnchorKey& other) const {
            return file == other.file && anchor == other.anchor;
        }
 
        /// Hash functor for AnchorKey.
        struct Hash {
            /// The hash function. Combines the hashes of the two strings using the XOR operator.
            /// @param key The key to hash. 
            /// @return The hash value.
            std::size_t operator()(const AnchorKey& key) const {
                return   std::hash<alib::String>()(key.file)
                       ^ std::hash<alib::String>()(key.anchor);
            }
        };
    };
 
    /// The hashmap providing an index to tree nodes by their name.
    /// For some compound types, for example, those that represent \e overloaded methods in C++,
    /// the tree node name string contains additional information like parameter types or
    /// method qualifiers. Those are stripped before inserted into this map. Thus, this table
    /// contains multiple entries and has to be fed and queried using the according set of
    /// interface methods.
    alib::containers::HashMap<alib::MonoAllocator,
                              alib::String,
                              CursorHandle >       entityMap;
 
    /// Maps HTML-files found to their corresponding tree nodes.
    alib::containers::HashMap<alib::MonoAllocator,
                              const alib::String&,
                              CursorHandle,
                              std::hash<alib::String>,
                              std::equal_to<const alib::String>,
                              alib::lang::Caching::Disabled,
                              alib::Recycling::None                                     >   fileMap;
 
    /// Maps HTML-files found to their corresponding tree nodes.
    alib::containers::HashMap<alib::MonoAllocator,
                              AnchorKey,
                              CursorHandle,
                              AnchorKey::Hash,
                              std::equal_to<AnchorKey>,
                              alib::lang::Caching::Enabled    >                           anchorMap;
 
  public:
    /// The cursor type of the #"StringTree".
    struct Node : protected Cursor {
 
        /// Defaulted default constructor. Creates an invalid node.
        Node() = default;
 
        /// Move constructor from underlying cursor type.
        /// @param other The cursor to move.
        Node(      Cursor&& other) : Cursor(std::move(other)) {}
 
        /// Copy constructor from underlying cursor type.
        /// @param other The cursor to copy.
        Node(      Cursor&  other) : Cursor(other) {}
 
        /// Gets the parent node by calling #"TCursor::Parent;2".
        /// @return The parent node.
        Node    Parent() { return Cursor::Parent(); }
 
        /// Gets the parent node by calling #"TCursor::Parent;2".
        /// @return The parent node.
        Node    Parent() const { return Cursor::Parent(); }
 
        /// Comparison operator.
        /// @param other  The object to compare ourselves to.
        /// @return \c    true if this and the given cursor are equal, \c false
        ///               otherwise.
        bool operator==(const Node &other)                                                const {
            return    node == other.node
                   && tree == other.tree;
        }
 
        /// Comparison operator.
        /// @param other  The object to compare ourselves to.
        /// @return \c    false if this and the given cursor are equal, \c true otherwise.
        bool operator!=(const Node &other)                   const { return !((*this) == other); }
 
        /// Calls the parent method and then replaces the separator characters accordingly.
        /// @param targetString  The string buffer to append the path to.
        /// @param targetData    Denotes whether \p{target} should be cleared before
        ///                      appending the path. Defaults to \b CurrentData::Clear.
        /// @return The given \b AString to allow concatenated operations.
        alib::strings::TAString<cmTree::CharacterType, alib::lang::HeapAllocator>&
        Path( alib::strings::TAString<cmTree::CharacterType>&  targetString,
              alib::lang::CurrentData                                   targetData=
                                                            alib::lang::CurrentData::Clear ) const {
            alib::integer prevLength= targetData == alib::lang::CurrentData::Keep
                                      ? targetString.Length()
                                      : 0;
            Cursor::AssemblePath(targetString, targetData);
            Index().ReplaceFromTreeSeparator(targetString, Kind(), prevLength);
            return targetString;
        }
 
        /// Calls the parent method and then replaces the separator charachters accordingly.
        /// @param targetString The string buffer to append the path to.
        /// @param parent       Denotes the parent node to start a relative path from.
        /// @return The given \b AString to allow concatenated operations.
        alib::strings::TAString<cmTree::CharacterType, alib::lang::HeapAllocator>&
        Path( alib::strings::TAString<cmTree::CharacterType>&  targetString,
              Node&                                            parent       ) const {
 
            alib::integer prevLength= targetString.Length();
            Cursor::AssemblePath(targetString, parent, alib::lang::CurrentData::Keep);
            Index().ReplaceFromTreeSeparator(targetString, Kind(), prevLength);
            return targetString;
        }
 
        /// Returns the #"Index" that this #"%Index::Node" belongs to.
        /// @return The index that this object refers to.
        Index&          Index() const { dbgCheckTree(); return *static_cast<class Index*>(tree); }
 
        /// @return The target that this node refers to.
        const Target*   Target() const { return Value(); }
 
        /// @return The target that this node refers to.
        class Target*   Target()      { return Value(); }
 
        /// Returns the line number in the tag-file that this node created.
        /// @return The line number in the tag-file.
        int LineNo() const { return Target()->LineNo; }
 
        /// The HTML file that this node links to (or into).
        /// @return The HTML file name.
        const alib::String& HTMLFile() const { return Target()->HTMLFile; }
 
        /// Returns the kind of this target. The result may be used to cast a pointer to the
        /// corresponding descendant
        /// @return The kind of this target.
        Target::Kinds   Kind() const { return Target()->Kind(); }
 
        /// Tests if this node holds a target is of a given \p{aKind}.
        /// @param kind The kind to test this instance for.
        /// @return The result of #"Target::IsA;Target()->IsA".
        bool IsA(Target::Kinds kind) const { return Target()->IsA(kind); }
 
 
        /// While usually targets are never replaced, this method is only exclamatory used.
        /// For example, with the method #"tryResolveHTMLTargetFile;2".
        /// @param newTarget The new target to set.
        void              ReplaceTarget(class Target* newTarget)   { Cursor::Value()= newTarget; }
 
        /// Shortcut to #"Cast(const Target*);(" passing this's #"TCursor::Value;value".
        /// @tparam TGT    The requested target type.
        /// @tparam TCheck If #"alib::NC" is given, no check whether the right kind is contained in
        ///                this node is performed. Defaults to #"alib::CHK".
        /// @return The cast type or \c nullptr if the \p{node}'s target is not of the requested type.
        template<typename TGT, typename TCheck= alib::CHK>
        requires std::derived_from<TGT, dxl::Target>
        const TGT* Cast() const { return dxl::Cast<TGT,TCheck>(Value()); }
        
        // Allowed functions
        using Cursor::Name;
        using Cursor::IsRoot;
        using Cursor::IsValid;
        using Cursor::IsInvalid;
        using Cursor::Export;
 
        /// Cast to to protected parent #"StringTree::Cursor;2".
        /// @return The cast cursor.
        Cursor& AsCursor() { return static_cast<Cursor&>(*this); }
    };
 
    /// The path to the doxygenTagFile.
    alib::system::PathString FilePath;
 
    /// The base name component of #FilePath.
    alib::system::PathString FileName;
 
    /// The URL of the HTML-output created with this tag-file.
    const alib::String& BaseURL;
 
    /// A shared lock to this index. While most of the time (during link resolve) read-only
    /// access is needed, for a few operations a write-lock must be set. This is, for example,
    /// done in method #"tryResolveHTMLTargetFile", which  modifies the index.
    alib::SharedLock SLock;
 
    /// Indicates whether this index is for the main tag file or an imported tag-file of a
    /// different project. Set with construction.
    const bool IsMainTagFile;
 
  protected:
    /// A list of parts of the #FileName separated by dots. This is used by the methods
    /// #ScopeHintMatch and #Search to find scope-hints and parent-scopes.<br>
    /// This list is created once with construction.
    alib::String  fileNameDotComponents[10];
 
    /// The number of components in #fileNameDotComponents.
    int           fileNameDotComponentsSize;
    //--------------------------------------- tag-file reading ---------------------------------------
protected:
    /// Internal struct that parses a tag-file. Used by the method #loadTagFile.
    struct Parser {
        /// our parent
        Index&                     index;
 
        /// our parent's parent
        DoxygenXLinks&             dxl;
 
        /// A referenced to the allocator of outer class #Index.
        alib::MonoAllocator&        ma;
 
        /// The log verbosity of the parser (used for performance optimization)
        alib::lox::Verbosity        verbosity;
 
        /// A referenced to the \alib text file reader tool.
        alib::IStreamReader&        reader;
 
        /// The path to the doxygenTagFile.
        alib::system::PathString    FilePath;
 
        /// The current line number when reading
        int                         lineNo= 0;
 
        /// The buffer of the line currently read
        alib::String2K              lineBuf;
 
        /// A parser for the currently read line.
        alib::Substring             line;
 
        /// The current tag attributes. Set by #parseTag.
        alib::Substring             tagAttr;
 
        /// The maximum line width found.
        alib::integer               maxWidth= 0;
 
        /// Set by nextLine() in case it finds a <docanchor>.
        alib::ListMA<TGTDocAnchor*> actDocAnchors;
 
        /// Constructor
        /// @param parent    The index this instance belongs to.
        /// @param pReader   The input data.
        /// @param pFilePath The file path of the tag-file.
        Parser(Index& parent, alib::IStreamReader& pReader, const alib::String& pFilePath);
 
        /// parses an XML-attribute with the given \p{key} from field #tagAttr.
        /// @param key The key of the attribute to parse.
        /// @return The attribute value.
        alib::String getAttr(const alib::String& key) {
            return alib::Substring( tagAttr.Substring(
                                       tagAttr.IndexOf(alib::String32(key)._("=")) + key.Length() + 2 ))
                         .ConsumeToken('"', alib::lang::Inclusion::Exclude);
        }
 
        /// Reads the next line of the file into #lineBuf. Increases #lineNo and sets #line.
        /// If it reads a \b Doxygen <em><docanchor></em> text, that anchor is added to
        /// the \p{actCompound}.
        /// \note This might not be the nicest design, but it's an effective solution. Doc-Anchors
        ///       can just appear in any component.
        void nextLine();
 
        /// Parses the given tag from the actual line. If the tag has attributes, that string
        /// is copied to the field #tagAttr. Otherwise, that field is \e nulled.
        /// @param tagName The name of the tag to parse.
        /// @return the contents of the tag.
        alib::String parseTag(const alib::String& tagName);
 
        /// Calls #parseTag and allocates the result in our \b MonoAllocator.
        /// @param tagName The name of the tag to parse.
        /// @return The allocated copy of the parsed tag.
        alib::String allocTag(const alib::String& tagName)
        { return alib::String(ma, parseTag(tagName)); }
 
        /// Parses the given attribute from the current string in #tagAttr.
        /// If not found, an exception is thrown.
        /// @param attrName   The name of the attribute to parse.
        /// @param isOptional if \c false if given, an error is logged if the attribute was not
        ///                   found.
        /// @return the contents of the tag.
        alib::String parseAttr(const alib::String& attrName, bool isOptional= false);
 
        /// Calls #nextLine and then parseTag.
        /// @param tagName The name of the tag to parse.
        /// @return The allocated copy of the parsed tag.
        alib::String nextTag(const alib::String& tagName) { nextLine(); return parseTag(tagName); }
 
        /// Calls #nextTag and allocates the result in our \b MonoAllocator.
        /// @param tagName The name of the tag to parse.
        /// @return The allocated copy of the parsed tag.
        alib::String allocNextTag(const alib::String& tagName)
        { return alib::String(ma, nextTag(tagName)); }
 
 
        /// Tries to identify \p{tagName} in the current #line. If found, #parseTag is called.
        /// @param tagName The name of the tag to try to parse.
        /// @return On success the parsed tag, a \e nulled string otherwise.
        alib::String tryTag(const alib::String& tagName) {
            if (line.Substring(1).StartsWith(tagName))
                return parseTag(tagName);
            return alib::NULL_STRING;
        }
 
        /// Reads a member at the actual position.
        /// The returned key equals the member name.
        /// With #"TGTFunction;function-members", in addition, the argument list is added by
        /// using the method #"FunctionArguments::PARSE" and likewise the field #"TGT Qualifiers".
        ///
        /// If the \p{compoundKind} indicates a \b Doxygen group, then parsing is very tolerant
        /// because most members are just references to entities defined elsewhere.
        ///
        /// @param[out] key          The key to use for the string tree.
        /// @param      compoundKind The kind of the compound actually parsed.
        /// @return A member struct of a type derived of type #"TGTMember".
        TGTMember* readMember(alib::String512& key, Target::Kinds compoundKind);
 
        /// Adds all anchors in #actDocAnchors to the given parent node and clears the list.
        /// @param  parent          The parent node that receives the anchors as children.
        void      addDocAnchors(Cursor& parent);
    };
 
    /// Loads the tag-file. This is called by #Load.
    void loadTagFile();
 
  public:
    /// Statistics on the number of entities found per kind in #"Target::Kinds;2".
    Target::KindStats   KindStats;
 
    /// Statistics on the number of lines in the tag-file
    int                 StatCtdLines;
 
    /// An entry in the vector of search results generated by the method #Search.
    struct SearchResult {
        /// The cursor pointing to the result.
        Node          Node;
 
        /// Set if the user has given some hint to match the qualifiers.
        /// Note that in the case of qualifiers, they are all added to the display, even if just a
        /// hint was given.
        alib::String    HTMLBaseURL;
 
        /// The name of the target HTML-file. If empty, the #HTMLAnchor resides in the same file as
        /// it was found
        alib::String    HTMLFile;
 
        /// The HTML anchor hash. Set only with members.
        alib::String    HTMLAnchor;
 
        /// The result of #"Target::FunctionArguments::MATCH".
        /// Note that function arguments are only added to the display, if they precisely match.
        int             FunctionArgumentMatch;
 
        /// Set if the user has given some hint to match the qualifiers.
        /// Note that in the case of qualifiers, they are all added to the display, even if just a
        /// hint was given.
        bool            QualifiersHintMatch;
 
        /// Set if a member was found in a base type and not in the originally given one.
        bool            IsIndirectByInheritance;
 
        /// Set if the given outer scope is a type definition and the member was found in its
        /// target type.
        bool            IsIndirectByTypeDef;
 
        /// Set the \xl targets a type-definition and requested to resolve that.
        bool            IsResolvedTypeDef;
 
        /// Set if a link targets file that is not in indexed (in the tag-file) but evaluated
        /// by the file-scanner of \dxl. (Done with the method #"tryResolveHTMLTargetFile".)
        bool            LinksToAScannedHTMLFile;
 
        /// Set if the method #DoxygenXLinks::tryResolveHTMLTargetFile" is needed and succeeds.
        bool            IsIndirectLinkToScannedHTMLSourceFile;
 
        /// Shortcut to #"Cast(const Target*);(" passing the #"Index::Node"'s #"TCursor::Value;value".
        /// @tparam TGT    The requested target type.
        /// @return The cast type or \c nullptr if the #"%Index::Node"'s target is not of the
        ///         requested type.
        template<typename TGT>
        requires std::derived_from<TGT, Target>
        const TGT*        Cast()                                                              const;
 
    }; // struct SearchResult
 
  protected:
    /// Sub-function called by method #Search in case flag #"XLink::KindSpec" equals
    /// #"Target::Dir", hence a file is searched by <c>"!d"</c>-specification.
    /// @param xLink The link found in the HTML-files.
    void   searchFile( XLink& xLink );
 
    /// Sub-function by #Search which checks various things, like function arguments, template
    /// parameters, etc, and applies internal disambiguation rules.
    /// @param xLink                    The link that is processed.
    /// @param node                    The node to evaluate.
    /// @param isIndirectByInheritance Set if \p{node} is inherited member
    /// @param isIndirectByTypeDef     Set if \p{node} is indirectly found through a type definiton.
    void   evaluateEntity( XLink& xLink, Node node,
                           bool isIndirectByInheritance, bool isIndirectByTypeDef );
 
    /// Sub-function called at the end of the search-methods to remove group entries.
    /// @param targets The vector to remove group entries from.
    void   dedupGroupEntries( alib::StdVectorMA<SearchResult>& targets );
 
 
  public:
    /// A timestamp set when loaded.
    alib::Ticks::Duration   LoadTime;
 
    /// Constructor.
    /// @param tagFilePath   The path to the tag-file.
    /// @param baseURL       The URL that the HTML files that this tag-files represent are found.
    /// @param isMainTagFile Sets field #".IsMainTagFile".
    Index(const alib::system::PathString& tagFilePath, const alib::String& baseURL,
          bool isMainTagFile);
 
    /// Destructor.
    ~Index();
 
    /// After all source code and html files have been scanned, \dxl continues single-threadded.
    /// By calling this method from #"DoxygenXLinks::Run", we can omit locks in the rest of the
    /// code without getting warnings. However, when option <c>--doxyfy</c> is given, we
    /// later enable them again.
    /// @param sWitch Determines to enable or disable the checks.
    void   DbgCriticalSectionCheck(alib::lang::Switch sWitch) {
        (void) sWitch;
        #if ALIB_DEBUG_CRITICAL_SECTIONS
        alib::lang::DbgCriticalSections::AssociatedLock* lockp=  sWitch == alib::lang::Switch::On
                                                                 ? &SLock : nullptr;
            nodeTable.dcs.DCSLock= lockp;
            entityMap.dcs.DCSLock= lockp;
            fileMap  .dcs.DCSLock= lockp;
            anchorMap.dcs.DCSLock= lockp;
        #endif
    }
 
    /// Imports a node (cursor) previously exported with #"TCursor Export".
    /// @param handle The handle value.
    /// @return The imported node instance.
    Node                ImportNode( CursorHandle handle )           { return ImportCursor(handle); }
 
    /// Loads the tag-file and creates the index.
    void                Load();
 
    /// Returns the #"StringTree"-node of the entity that is described in the given HTML file.
    /// @param htmlFile The HTML file node.
    /// @return A cursor pointing to the local entity.
    ConstCursorHandle   GetHTMLFileEntity(const alib::File& htmlFile);
 
    /// Searches entries in this doxygen tag-file index.
    /// The given \p{results} vector might be filled already by other instances of this type
    /// in the case that multiple tag-files are provided with the doxygen INI-file.
    /// Also, a call to this function might insert more than one result.
    /// @param xLink             The link found in the HTML-files.
    void                Search(XLink& xLink);
 
    /// Tests if the given node in the Index's #"StringTree" sits in a path that matches the
    /// #"scopeHintsSize;path hints" of the given \p{searchLink}.
    /// @param entry      The node in the "dxl #Index" to test.
    /// @param searchLink The search definition to test against.
    /// @return \c true if the path ints given with this search specification are matching this
    ///         the path to the \p{entry}
    bool ScopeHintMatch(Node entry, XLink& searchLink);
 
    /// Replaces the tree separator with the character(s) appropriate for the given kind.
    /// @param buffer   The string buffer to modify.
    /// @param kind     The kind of target for which the separator should be replaced.
    /// @param startPos The starting position within the buffer where the replacement should begin.
    ///                 Defaults to \c 0.
    void ReplaceFromTreeSeparator( alib::AString& buffer, Target::Kinds kind,
                                   alib::integer startPos= 0 );
 
    /// Replaces character(s) separating the scope of the given kind with the tree separator.
    /// @param buffer   The string buffer to modify.
    /// @param kind     The kind of target for which the separator should be replaced.
    /// @param startPos The starting position within the buffer where the replacement should begin.
    ///                 Defaults to \c 0.
    void ReplaceToTreeSeparator( alib::AString& buffer, Target::Kinds kind,
                                 alib::integer startPos= 0 );
 
    /// Dumps the contents of the tag-file branch to the Lox.
    /// @param branch The branch to dump.
    void  DumpTree(Node branch);
}; // class Index
 
/// Shortcut to #"Cast(const Target*);(" passing the given \p{node}'s #"TCursor::Value;value".
/// @tparam TGT    The requested target type.
/// @tparam TCheck If #"alib::NC" is given, no check whether the right kind is contained in
///                this node is performed. Defaults to #"alib::CHK".
/// @param  node   The index-node.
/// @return The cast type or \c nullptr if the \p{node}'s target is not of the requested type.
template<typename TGT, typename TCheck= alib::CHK>
requires std::derived_from<TGT, Target>
const TGT* Cast(const Index::Node& node) { return node.Cast<TGT,TCheck>(); }
 
template<typename TGT>
requires std::derived_from<TGT, Target>
const TGT*  Index::SearchResult::Cast()                        const { return dxl::Cast<TGT>(Node); }
 
 
} //namespace [dxl]
 
 
#endif // HPP_DXL_INDEX