GatewaySupervisor.h

#ifndef _ots_GatewaySupervisor_h
#define _ots_GatewaySupervisor_h
#include <atomic>
#include <condition_variable>
#include <mutex>
 
#include "otsdaq/CoreSupervisors/ConfigurationSupervisorBase.h"
#include "otsdaq/CoreSupervisors/CorePropertySupervisorBase.h"
#include "otsdaq/FiniteStateMachine/RunControlStateMachine.h"
#include "otsdaq/GatewaySupervisor/Iterator.h"
#include "otsdaq/SOAPUtilities/SOAPMessenger.h"
#include "otsdaq/SupervisorInfo/AllSupervisorInfo.h"
#include "otsdaq/WebUsersUtilities/WebUsers.h"
#include "otsdaq/WorkLoopManager/WorkLoopManager.h"
 
#include "otsdaq/CodeEditor/CodeEditor.h"
#include "otsdaq/TablePlugins/DesktopIconTable.h"
 
#include "otsdaq/NetworkUtilities/TransceiverSocket.h"  // for UDP state changer
 
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
#include <xdaq/Application.h>
#pragma GCC diagnostic pop
#include "otsdaq/Macros/XDAQApplicationMacros.h"
 
#include <toolbox/task/WorkLoop.h>
#include <xdata/String.h>
#include <xgi/Method.h>
#include "otsdaq/GatewaySupervisor/PixelHistoPicGen.h"
 
#include <pthread.h>  // for pthread_setcancelstate in broadcastMessageThread
#include <set>
#include <sstream>
#include <string>
 
// clang-format off
 
// #define FSM_LAST_CONFIGURED_GROUP_ALIAS_FILE             std::string("FSMLastConfiguredGroupAlias.hist")
// #define FSM_LAST_STARTED_GROUP_ALIAS_FILE                std::string("FSMLastStartedGroupAlias.hist")
 
// #define FSM_CONFIGURED_GROUP_ALIASES_FILE                std::string("FSMConfiguredGroupAliases.hist")
// #define FSM_STARTED_GROUP_ALIASES_FILE                   std::string("FSMStartedGroupAlias.hist")
 
// #define FSM_CONFIGURED_OR_STARTED_GROUP_ALIASES_FILE     std::string("FSMConfiguredOrStartedGroupAlias.hist")
 
// #define FSM_CONFIGURED_CONTEXTS_FILE                     std::string("FSMConfiguredContexts.hist")
// #define FSM_STARTED_CONTEXTS_FILE                        std::string("FSMStartedContexts.hist")
// #define FSM_CONFIGURED_OR_STARTED_CONTEXTS_FILE      std::string("FSMConfiguredOrStartedContexts.hist")
 
// #define FSM_CONFIGURED_BACKBONES_FILE                    std::string("FSMConfiguredBackbones.hist")
// #define FSM_STARTED_BACKBONES_FILE                       std::string("FSMStartedBackbones.hist")
// #define FSM_CONFIGURED_OR_STARTED_BACKBONES_FILE         std::string("FSMConfiguredOrStartedBackbones.hist")
 
// #define FSM_CONFIGURED_ITERATORS_FILE                    std::string("FSMConfiguredIterators.hist")
// #define FSM_STARTED_ITERATORS_FILE                       std::string("FSMStartedIterators.hist")
// #define FSM_CONFIGURED_OR_STARTED_ITERATORS_FILE         std::string("FSMConfiguredOrStartedIterators.hist")
 
 
namespace ots
{
class ConfigurationManager;
class TableGroupKey;
class WorkLoopManager;
 
 
    class GatewaySupervisor : public xdaq::Application,
        public SOAPMessenger,
        public RunControlStateMachine,
        public CorePropertySupervisorBase,
        public ConfigurationSupervisorBase
    {
        friend class WizardSupervisor;
        friend class Iterator;
 
        static const std::string COMMAND_PARAM_LOG_ENTRY_PREAMBLE;
        static const std::string COMMAND_PARAM_SUBSYSTEM_COMMON_PREAMBLE;
        static const std::string COMMAND_PARAM_SUBSYSTEM_COMMON_OVERRIDE_PREAMBLE;
        static const std::string COMMAND_PARAM_ITERATION_INDEX_PREAMBLE;
 
    public:
        XDAQ_INSTANTIATOR();
 
                                    GatewaySupervisor               (xdaq::ApplicationStub* s);
        virtual                     ~GatewaySupervisor              (void);
 
        void                        init                            (void);
 
        void                        Default                         (xgi::Input* in, xgi::Output* out);
 
        void                        loginRequest                    (xgi::Input* in, xgi::Output* out);
        void                        request                         (xgi::Input* in, xgi::Output* out);
        void                        tooltipRequest                  (xgi::Input* in, xgi::Output* out);
        void                        XGI_Turtle                      (xgi::Input* in, xgi::Output* out);
 
        void                        addStateMachineStatusToXML      (HttpXmlDocument& xmlOut, const std::string& fsmName, bool getRunNumber = true);
        void                        addFilteredConfigAliasesToXML   (HttpXmlDocument& xmlOut, const std::string& fsmName);
        void                        addRequiredFsmLogInputToXML     (HttpXmlDocument& xmlOut, const std::string& fsmName);
 
        // State Machine requests handlers
        void                        stateMachineXgiHandler(xgi::Input* in, xgi::Output* out);
        void                        stateMachineIterationBreakpoint(xgi::Input* in, xgi::Output* out);
 
        static std::string          getIconHeaderString(void);
        static bool                 handleAddDesktopIconRequest(const std::string& author, cgicc::Cgicc& cgiIn, HttpXmlDocument& xmlOut, std::vector<DesktopIconTable::DesktopIcon>* newIcons = nullptr);
        static void                 handleGetApplicationIdRequest(AllSupervisorInfo* applicationInfo, cgicc::Cgicc& cgiIn, HttpXmlDocument& xmlOut, std::map<std::string /* requestOrigin */, std::map<std::string /* requestUrlHostPort */, std::string /* translatedHostPort */>>* portTranslationMap = nullptr);
 
        xoap::MessageReference      stateMachineXoapHandler(xoap::MessageReference msg);
 
        bool                        stateMachineThread(toolbox::task::WorkLoop* workLoop);
 
        // Status requests handlers
        void                        statusRequest(xgi::Input* in, xgi::Output* out);
        void                        infoRequestResultHandler(xgi::Input* in, xgi::Output* out);
        bool                        infoRequestThread(toolbox::task::WorkLoop* workLoop);
 
        // External GatewaySupervisor XOAP handlers
        xoap::MessageReference      supervisorCookieCheck(xoap::MessageReference msg);
        xoap::MessageReference      supervisorGetActiveUsers(xoap::MessageReference msg);
        xoap::MessageReference      supervisorSystemMessage(xoap::MessageReference msg);
        xoap::MessageReference      supervisorGetUserInfo(xoap::MessageReference msg);
        xoap::MessageReference      supervisorSystemLogbookEntry(xoap::MessageReference msg);
        xoap::MessageReference      supervisorLastTableGroupRequest(xoap::MessageReference msg);
 
        // Finite State Machine States
        void                        stateInitial(toolbox::fsm::FiniteStateMachine& fsm) override;
        void                        statePaused(toolbox::fsm::FiniteStateMachine& fsm) override;
        void                        stateRunning(toolbox::fsm::FiniteStateMachine& fsm) override;
        void                        stateHalted(toolbox::fsm::FiniteStateMachine& fsm) override;
        void                        stateConfigured(toolbox::fsm::FiniteStateMachine& fsm) override;
        void                        inError(toolbox::fsm::FiniteStateMachine& fsm) override;
 
        void                        transitionConfiguring(toolbox::Event::Reference e) override;
        void                        transitionHalting(toolbox::Event::Reference e) override;
        void                        transitionInitializing(toolbox::Event::Reference e) override;
        void                        transitionPausing(toolbox::Event::Reference e) override;
        void                        transitionResuming(toolbox::Event::Reference e) override;
        void                        transitionStarting(toolbox::Event::Reference e) override;
        void                        transitionStopping(toolbox::Event::Reference e) override;
        void                        transitionShuttingDown(toolbox::Event::Reference e) override;
        void                        transitionStartingUp(toolbox::Event::Reference e) override;
        void                        enteringError(toolbox::Event::Reference e) override;
 
        void                        makeSystemLogEntry(const std::string& entryText, const std::string& subjectText = "");
        static void                 addSystemMessage(std::string toUserCSV, std::string message);
 
        void                        checkForAsyncError(void);
 
        // CorePropertySupervisorBase override functions
        virtual void                    setSupervisorPropertyDefaults                   (void) override;  
        virtual void                    forceSupervisorPropertyValues                   (void) override;  
 
 
    private:
        unsigned int                    getNextRunNumber                                (const std::string& fsmName = "");
        void                            setNextRunNumber                                (unsigned int runNumber, const std::string& fsmName = "");
        std::string                     getLastLogEntry                                 (const std::string& logType, const std::string& fsmName = "");
        void                            setLastLogEntry                                 (const std::string& logType, const std::string& logEntry, const std::string& fsmName = "");
 
 
        static xoap::MessageReference   lastTableGroupRequestHandler                    (const SOAPParameters& parameters);
        static void                     launchStartOTSCommand                           (const std::string& command, ConfigurationManager* cfgMgr);
        static void                     launchStartOneServerCommand                     (const std::string& command, ConfigurationManager* cfgMgr, const std::string& contextName);
 
        static void                     indicateOtsAlive                                (const CorePropertySupervisorBase* properties = 0);
        xoap::MessageReference          TRACESupervisorRequest                          (xoap::MessageReference message);
 
        static void                     StateChangerWorkLoop                            (GatewaySupervisor* supervisorPtr);
        static void                     AppStatusWorkLoop                               (GatewaySupervisor* supervisorPtr, const bool doDisconnected = false);
 
        std::string                     attemptStateMachineTransition                   (HttpXmlDocument* xmldoc,
                                                                                        std::ostringstream* out,
                                                                                        const std::string& command,
                                                                                        const std::string& fsmName,
                                                                                        const std::string& fsmWindowName,
                                                                                        const std::string& username,
                                                                                        const std::vector<std::string>& parameters,
                                                                                        std::string logEntry = "");
        void                            broadcastMessage                                (xoap::MessageReference msg);
        void                            broadcastMessageToRemoteGateways                (const xoap::MessageReference msg, unsigned int iteration = 0);
        void                            broadcastMessageToRemoteGatewaysComplete        (const xoap::MessageReference msg, unsigned int iterationIndex = 0);
        void                            signalAndWaitForBroadcastThreads                (unsigned int numberOfThreads);
 
        struct BroadcastMessageIterationsDoneStruct
        {
            // Creating std::vector<std::vector<bool>>
            //  because of problems with the standard library
            //  not allowing passing by reference of bool types.
            // Broadcast thread implementation requires passing by reference.
            ~BroadcastMessageIterationsDoneStruct()
            {
                for (auto& arr : iterationsDone_)
                    delete[] arr;
                iterationsDone_.clear();
                arraySizes_.clear();
            }  // end destructor
 
            void push(const unsigned int& size)
            {
                iterationsDone_.push_back(new bool[size]);
                arraySizes_.push_back(size);
 
                // initialize to false
                for (unsigned int i = 0; i < size; ++i)
                    iterationsDone_[iterationsDone_.size() - 1][i] = false;
            }  // end push()
 
            bool* operator[](unsigned int i) { return iterationsDone_[i]; }
            const bool* operator[](unsigned int i) const { return iterationsDone_[i]; }
            unsigned int size(unsigned int i = -1)
            {
                if (i == (unsigned int)-1)
                    return iterationsDone_.size();
                return arraySizes_[i];
            }
 
        private:
            std::vector<bool*>        iterationsDone_;
            std::vector<unsigned int> arraySizes_;
        };  // end BroadcastMessageIterationsDoneStruct definition
 
        struct BroadcastThreadStruct
        {
            //===================
            BroadcastThreadStruct()
                : threadIndex_(-1)
                , exitThread_(false)
                , working_(true)
                , workToDo_(false)
                , error_(false)
            {
            }  // end BroadcastThreadStruct constructor()
 
            //===================
            BroadcastThreadStruct(BroadcastThreadStruct &&b)
                : threadIndex_(b.threadIndex_)
                , exitThread_(b.exitThread_.load())
                , working_(b.working_.load())
                , workToDo_(b.workToDo_.load())
                , error_(b.error_.load())
            {
            }  // end BroadcastThreadStruct move constructor()
 
 
            struct BroadcastMessageStruct
            {
                //===================
                BroadcastMessageStruct(const SupervisorInfo& appInfo,
                    xoap::MessageReference message,
                    const std::string& command,
                    const unsigned int& iteration,
                    bool& iterationsDone,
                    std::shared_ptr<BroadcastMessageIterationsDoneStruct> iterationsDoneOwner)
                    : appInfo_(appInfo)
                    , message_(message)
                    , command_(command)
                    , iteration_(iteration)
                    , iterationsDone_(iterationsDone)
                    , iterationsDoneOwner_(iterationsDoneOwner)
                {
                }
 
                const SupervisorInfo& appInfo_;
                xoap::MessageReference message_;
                const std::string command_;
                const unsigned int iteration_;
                bool& iterationsDone_;
                // Keep the BroadcastMessageIterationsDoneStruct alive while this message
                // is in use by a thread, preventing UAF even if broadcastMessage() returns
                // early (e.g. on timeout or exception) before the thread finishes.
                std::shared_ptr<BroadcastMessageIterationsDoneStruct> iterationsDoneOwner_;
 
                std::string reply_;
            };  // end BroadcastMessageStruct definition
 
            //===================
            void setMessage(const SupervisorInfo& appInfo,
                xoap::MessageReference message,
                const std::string& command,
                const unsigned int& iteration,
                bool& iterationsDone,
                std::shared_ptr<BroadcastMessageIterationsDoneStruct> iterationsDoneOwner)
            {
                messages_.clear();
                messages_.push_back(BroadcastThreadStruct::BroadcastMessageStruct(
                    appInfo, message, command, iteration, iterationsDone, iterationsDoneOwner));
                workToDo_ = true;
            }  // end setMessage()
 
            const SupervisorInfo& getAppInfo() { return messages_[0].appInfo_; }
            xoap::MessageReference getMessage() { return messages_[0].message_; }
            const std::string& getCommand() { return messages_[0].command_; }
            const unsigned int& getIteration() { return messages_[0].iteration_; }
            std::string& getReply() { return messages_[0].reply_; }
            bool& getIterationsDone() { return messages_[0].iterationsDone_; }
 
            // each thread accesses these members
            std::mutex           threadMutex_;
            unsigned int         threadIndex_;
            std::atomic<bool>    exitThread_, working_, workToDo_, error_;
            // always just 1 message (for now)
            std::vector<BroadcastThreadStruct::BroadcastMessageStruct> messages_;
 
        };  // end BroadcastThreadStruct declaration
    static void broadcastMessageThread(
        GatewaySupervisor* supervisorPtr,
        std::shared_ptr<GatewaySupervisor::BroadcastThreadStruct> threadStruct);
    bool handleBroadcastMessageTarget(const SupervisorInfo& appInfo,
        xoap::MessageReference message,
        const std::string& command,
        const unsigned int& iteration,
        std::string& reply,
        unsigned int           threadIndex = 0,
        const std::atomic<bool>* exitFlag  = nullptr);
 
 
        // Member Variables -----------------------
 
        bool                supervisorGuiHasBeenLoaded_;  
        static WebUsers     theWebUsers_;
        std::map<std::string /* requestOrigin */, std::map<std::string /* requestUrlHostPort */,
            std::string /* translatedHostPort */>>
                            portTranslationMap_;  
 
        WorkLoopManager     stateMachineWorkLoopManager_;
        toolbox::BSem       stateMachineSemaphore_;
 
        std::string         activeStateMachineName_;  
        std::string         activeStateMachineWindowName_;
        std::string         activeStateMachineDumpFormatOnRun_, activeStateMachineDumpFormatOnConfigure_; 
        std::string         activeStateMachineSystemDumpOnRun_, activeStateMachineSystemDumpOnConfigure_; 
        bool                activeStateMachineSystemDumpOnRunEnable_, activeStateMachineSystemDumpOnConfigureEnable_; 
        std::string         activeStateMachineSystemDumpOnRunFilename_, activeStateMachineSystemDumpOnConfigureFilename_; 
        bool                activeStateMachineRequireUserLogOnRun_, activeStateMachineRequireUserLogOnConfigure_; 
        std::string         activeStateMachineRunInfoPluginType_; 
        std::map<std::string /* fsmName */, std::string /* logEntry */>
                            stateMachineConfigureLogEntry_, stateMachineStartLogEntry_, stateMachineStopLogEntry_;
        std::string         activeStateMachineRunNumber_, activeStateMachineRunAlias_, activeStateMachineConfigurationAlias_;
        bool                activeStateMachineRollOverLogOnConfigure_, activeStateMachineRollOverLogOnStart_;
        std::chrono::steady_clock::time_point
                            activeStateMachineRunStartTime;
        int                 activeStateMachineRunDuration_ms; 
        unsigned int        activeStateMachineConfigureConditionID_, activeStateMachineRunConditionID_;
        std::string         activeStateMachineSubsystemCommonList_, activeStateMachineSubsystemCommonOverrideList_; 
 
        std::mutex          systemStatusMutex_;
        std::string         lastLogbookEntry_;
        time_t              lastLogbookEntryTime_ = 0;
 
        std::string         lastConsoleErr_, lastConsoleWarn_, lastConsoleInfo_, lastConsoleErrTime_, lastConsoleWarnTime_, lastConsoleInfoTime_;
        std::string         firstConsoleErr_, firstConsoleWarn_, firstConsoleInfo_, firstConsoleErrTime_, firstConsoleWarnTime_, firstConsoleInfoTime_;
        size_t              systemConsoleErrCount_ = 0, systemConsoleWarnCount_ = 0, systemConsoleInfoCount_ = 0;
 
        std::pair<std::string /*group name*/, TableGroupKey>
                            theConfigurationTableGroup_;  
        std::string         stateMachineTransitionUsername_;  
 
        Iterator            theIterator_;
        std::mutex          stateMachineAccessMutex_;  
        std::string         stateMachineLastCommandInput_;
        enum
        {
            VERBOSE_MUTEX = 0
        };
 
        CodeEditor          codeEditor_;
 
        std::mutex          broadcastCommandMessageIndexMutex_;
        unsigned int        broadcastCommandMessageIndex_;
        std::atomic<bool>   broadcastIterationsDone_{true};
        std::mutex          broadcastIterationBreakpointMutex_;
        unsigned int        broadcastIterationBreakpoint_;  
        std::mutex          broadcastCommandStatusUpdateMutex_;
        std::string         broadcastCommandStatus_;
 
        std::mutex              remoteIterationMutex_;
        std::condition_variable remoteIterationCV_;
        unsigned int            remoteIterationIndex_ = 0;
        std::atomic<bool>       isRemoteSubsystemIteration_{false}; 
 
        static std::vector<std::shared_ptr<GatewaySupervisor::BroadcastThreadStruct>> broadcastThreadStructs_; 
 
        std::string         securityType_;
        PixelHistoPicGen    picGen_;
 
        //Variable used by the RunInfo plugin
        unsigned int        conditionID_;
 
public: //used by remote subsystem control and status
 
        struct RemoteGatewayInfo {
            SupervisorInfo::SubappInfo          appInfo;
 
            enum class ConfigDumpTypes 
            {
                Text,
                JSON_all,
                Unknown
            };
 
            std::string                         command, fsmName; 
          private: //make error private to connect to set timestamp
            std::string                     error;
            time_t                          errorTimestamp = 0;
          public:
            void                                setError(const std::string& err) { error = err; errorTimestamp = time(0); }
            void                                clearError() { error = ""; errorTimestamp = 0; }
            void                                copyError(const RemoteGatewayInfo& r) { error = r.error; errorTimestamp = r.errorTimestamp; }
            const std::string&                  getError() const { return error; }
            const std::string                   getErrorTimestamp() const { return StringMacros::getTimestampString(errorTimestamp); }
 
            std::string                         config_dump;
            ConfigDumpTypes                     config_dump_type = ConfigDumpTypes::Unknown;
 
            size_t                              ignoreStatusCount = 0; 
 
            size_t                              consoleErrCount = 0, consoleWarnCount = 0;
 
            std::string                         fullName;
            std::string                         user_data_path_record; 
            std::string                         setupType, instancePath, instanceHost, instanceUser; 
 
            std::string                         selected_config_alias; 
            std::set<std::string>               config_aliases; 
            std::string                         iconString, parentIconFolderPath, landingPage, permissionThresholdString; 
 
            std::string                         usernameWithLock;
 
            enum class FSM_ModeTypes 
            {
                Follow_FSM,
                DoNotHalt, 
                OnlyConfigure, 
            };
            FSM_ModeTypes                       fsm_mode = FSM_ModeTypes::Follow_FSM; 
            bool                                fsm_included = true;
 
            std::string                         getFsmMode() const {
                switch(fsm_mode)
                {
                    case FSM_ModeTypes::Follow_FSM: return "Follow FSM";
                    case FSM_ModeTypes::DoNotHalt: return "Do Not Halt";
                    case FSM_ModeTypes::OnlyConfigure: return "Only Configure";
                    default: return "Impossible";
                }
            } //end getFsmMode()
 
            std::string                         getConfigDumpType() const {
                switch(config_dump_type)
                {
                    case ConfigDumpTypes::Text: return "Text";
                    case ConfigDumpTypes::JSON_all: return "JSON all";
                    default: return "Unknown";
                }
            } //end getFsmMode()
 
            std::map<std::string, SupervisorInfo::SubappInfo>   subapps; 
            bool iterationsDone = false; 
        }; //end GatewaySupervisor::RemoteGatewayInfo struct
 
        std::vector<GatewaySupervisor::RemoteGatewayInfo>   remoteGatewayApps_;
        std::mutex                                          remoteGatewayAppsMutex_;
        std::map<std::string /* appName */,
            bool /* lastStatusGood */>                      appLastStatusGood_;
        std::mutex                                          dualStatusThreadMutex_;
 
        std::string                                         ipAddressForStateChangesOverUDP_; 
        int                                                 portForReverseLoginOverUDP_ = 0;  
 
        std::map<unsigned int /* lid */, SupervisorInfo>    localAllSupervisorInfo_; 
 
 
        std::mutex                                          latestGatewayIconsMutex_;
        std::vector<DesktopIconTable::DesktopIcon>          latestGatewayIcons_; 
        std::pair<std::string /* latestIconContext group */, TableGroupKey>
                                                            latestGatewayIconsContextGroup_; 
 
        std::string                                         latestGatewayRemoteIconsString_; 
        std::pair<std::string /* latestIconContext group */, TableGroupKey>
                                                            latestGatewayRemoteIconsContextGroup_; 
 
        static void                 CheckRemoteGatewayStatus                    (GatewaySupervisor::RemoteGatewayInfo& remoteGatewayApp, const std::unique_ptr<TransceiverSocket>& remoteGatewaySocket, const std::string& ipForReverseLoginOverUDP, int portForReverseLoginOverUDP);
        static void                 SendRemoteGatewayCommand                    (GatewaySupervisor::RemoteGatewayInfo& remoteGatewayApp, const std::unique_ptr<TransceiverSocket>& remoteGatewaySocket);
        static void                 GetRemoteGatewayIcons                       (GatewaySupervisor::RemoteGatewayInfo& remoteGatewayApp, const std::unique_ptr<TransceiverSocket>& remoteGatewaySocket);
        void                        loadRemoteGatewaySettings                   (std::vector<GatewaySupervisor::RemoteGatewayInfo>& remoteGateways, bool onlyNotFound = false) const;
        void                        saveRemoteGatewaySettings                   (void) const;
        static std::string          translateURLForRequestOrigin                (const std::string& url, const std::string& requestOrigin, std::map<std::string /* requestOrigin */, std::map<std::string /* requestUrlHostPort */, std::string /* translatedHostPort */>>& portTranslationMap);
        static std::string          translateRemoteIconStringForRequestOrigin   (const std::string& iconString, const std::string& requestOrigin, std::map<std::string /* requestOrigin */, std::map<std::string /* requestUrlHostPort */, std::string /* translatedHostPort */>>& portTranslationMap);
 
    };
// clang-format on
 
}  // namespace ots
 
#endif