SharedDummyBackend.h

Go to the documentation of this file.

// SPDX-FileCopyrightText: Deutsches Elektronen-Synchrotron DESY, MSK, ChimeraTK Project <chimeratk-support@desy.de>
// SPDX-License-Identifier: LGPL-3.0-or-later
#pragma once
 
#include "DummyBackendBase.h"
 
#include <boost/filesystem.hpp>
#include <boost/function.hpp>
#include <boost/interprocess/allocators/allocator.hpp>
#include <boost/interprocess/containers/vector.hpp>
#include <boost/interprocess/managed_shared_memory.hpp>
#include <boost/interprocess/sync/interprocess_semaphore.hpp>
#include <boost/interprocess/sync/named_mutex.hpp>
#include <boost/move/unique_ptr.hpp>
#include <boost/unordered_set.hpp>
 
#include <list>
#include <map>
#include <thread>
#include <utility>
#include <vector>
 
// Define shared-memory compatible vector type and corresponding allocator
using ShmemAllocator =
    boost::interprocess::allocator<int32_t, boost::interprocess::managed_shared_memory::segment_manager>;
using SharedMemoryVector = boost::interprocess::vector<int32_t, ShmemAllocator>;
using PidSet = boost::interprocess::vector<int32_t, ShmemAllocator>;
 
namespace ChimeraTK {
 
  // max. allowed SharedDummyBackend instances using common shared mem segment (global count, over all processes)
  const int SHARED_MEMORY_N_MAX_MEMBER = 10;
 
  class SharedDummyBackend : public DummyBackendBase {
   public:
    SharedDummyBackend(
        size_t instanceIdHash, const std::string& mapFileName, const std::string& dataConsistencyKeyDescriptor = "");
    ~SharedDummyBackend() override;
 
    void open() override;
    void closeImpl() override;
 
    using DummyBackendBase::read;  // use the 32 bit version from the base class
    using DummyBackendBase::write; // use the 32 bit version from the base class
    void read(uint64_t bar, uint64_t address, int32_t* data, size_t sizeInBytes) override;
    void write(uint64_t bar, uint64_t address, int32_t const* data, size_t sizeInBytes) override;
 
    std::string readDeviceInfo() override;
 
    static boost::shared_ptr<DeviceBackend> createInstance(
        std::string address, std::map<std::string, std::string> parameters);
 
    VersionNumber triggerInterrupt(uint32_t interruptNumber) override;
 
   private:
    std::string _mapFile;
 
    // Bar contents with shared-memory compatible vector type. Plain pointers are
    // used here since this is what we get from the shared memory allocation.
    std::map<uint64_t, SharedMemoryVector*> _barContents;
 
    // Bar sizes
    std::map<uint64_t, size_t> _barSizesInBytes;
 
    // Naming of bars as shared memory elements
    const char* SHARED_MEMORY_BAR_PREFIX = "BAR_";
 
    class InterruptDispatcherInterface;
 
    // Helper class to manage the shared memory: automatically construct if
    // necessary, automatically destroy if last using process closes.
    class SharedMemoryManager {
      friend class SharedDummyBackend;
 
     public:
      SharedMemoryManager(SharedDummyBackend&, std::size_t instanceIdHash, const std::string&);
      ~SharedMemoryManager();
 
      SharedMemoryVector* findOrConstructVector(const std::string& objName, size_t size);
 
      std::pair<size_t, size_t> getInfoOnMemory();
 
     private:
      // Constants to take overhead of managed shared memory into respect
      // (approx. linear function, meta data for memory and meta data per vector)
      // Uses overestimates for robustness.
      static const size_t SHARED_MEMORY_CONST_OVERHEAD = 1000;
      static const size_t SHARED_MEMORY_OVERHEAD_PER_VECTOR = 160;
 
      const char* SHARED_MEMORY_PID_SET_NAME = "PidSet";
      const char* SHARED_MEMORY_REQUIRED_VERSION_NAME = "RequiredVersion";
 
      SharedDummyBackend& sharedDummyBackend;
 
      // the name of the segment
      std::string name;
 
      // the shared memory segment
      boost::interprocess::managed_shared_memory segment;
 
      // the allocator instance
      const ShmemAllocator sharedMemoryIntAllocator;
 
      // Pointers to the set of process IDs and the required version
      // specifier in shared memory
      PidSet* pidSet{nullptr};
      // Version number is not used for now, but included in shared memory
      // to facilitate compatibility checks later
      unsigned* requiredVersion{nullptr};
 
      size_t getRequiredMemoryWithOverhead();
      bool checkPidSetConsistency();
      void reInitMemory();
      std::vector<std::string> listNamedElements();
 
     protected:
      // interprocess mutex, has to be accessible by SharedDummyBackend class
      boost::interprocess::named_mutex interprocessMutex;
 
      boost::movelib::unique_ptr<InterruptDispatcherInterface> intDispatcherIf;
    }; /* class SharedMemoryManager */
 
    // Managed shared memory object
    std::unique_ptr<SharedMemoryManager> sharedMemoryManager;
 
    // Setup register bars in shared memory
    void setupBarContents();
 
    // Helper routines called in init list
    size_t getTotalRegisterSizeInBytes() const;
 
    static void checkSizeIsMultipleOfWordSize(size_t sizeInBytes);
 
    static std::string convertPathRelativeToDmapToAbs(std::string const& mapfileName);
 
    /****************** definitions for across-instance triggering ********/
 
    // We are using the process id as an id of the semaphore which is to be triggered for the interrupt dispatcher
    // thread. Since there is only one interrupt dispatcher thread per mapped shared memory region in a process, and
    // the semaphore is set inside the shared memory, this means we can identify all semaphores per shared memory
    // this way.
    // However, this implies the restriction that you must not create more than one backend instance per shared memory
    // region inside a process. E.g. if you wanted to write a test by tricking the backend factory into creating more
    // than one backend instance for the same process and shared memory, you will have a problem.
    using SemId = std::uint32_t;
 
    struct SemEntry {
      SemEntry() = default;
      boost::interprocess::interprocess_semaphore s{0};
      SemId semId{};
      bool used = false;
    };
 
    struct InterruptEntry {
      int _controllerId{0};
      int _intNumber{};
      std::uint32_t _counter = 0;
      bool used = false;
    };
 
    static const int maxInterruptEntries = 1000;
 
    struct ShmForSems {
      // In addition to the semaphores themselves, shm stores a vector of
      // interrupt numbers and their current counts.
      // Vector entries are not moved and marked as unused when no longer needed.
      // We need
      // - a find function which returns list of appropriate  semaphores to be triggered
      //   current concept does not use interrupt number for that
      // - add/remove functions to add/remove a semaphore
      // - functions to update interrupt counts
 
      using Sem = boost::interprocess::interprocess_semaphore;
 
      ShmForSems() = default;
      ShmForSems(const ShmForSems&) = delete;
 
      Sem* addSem(SemId semId);
      bool removeSem(SemId semId);
      void cleanup(PidSet* pidSet);
 
      void addInterrupt(uint32_t interruptNumber);
 
      std::list<Sem*> findSems(uint32_t interruptNumber = {}, bool update = false);
 
      void print();
 
      SemEntry semEntries[SHARED_MEMORY_N_MAX_MEMBER];
      InterruptEntry interruptEntries[maxInterruptEntries];
    };
 
    struct InterruptDispatcherThread;
 
    class InterruptDispatcherInterface {
     public:
      InterruptDispatcherInterface(SharedDummyBackend& backend, boost::interprocess::managed_shared_memory& shm,
          boost::interprocess::named_mutex& shmMutex);
 
      ~InterruptDispatcherInterface();
      static void cleanupShm(boost::interprocess::managed_shared_memory& shm);
      static void cleanupShm(boost::interprocess::managed_shared_memory& shm, PidSet* pidSet);
 
      void triggerInterrupt(uint32_t intNumber);
      boost::interprocess::named_mutex& _shmMutex;
      SemId _semId;
      ShmForSems* _semBuf;
      boost::movelib::unique_ptr<InterruptDispatcherThread> _dispatcherThread;
      SharedDummyBackend& _backend;
    };
 
    struct InterruptDispatcherThread {
      explicit InterruptDispatcherThread(InterruptDispatcherInterface* dispatcherInterf);
      InterruptDispatcherThread(const InterruptDispatcherThread&) = delete;
      ~InterruptDispatcherThread();
 
      void run();
      void stop() noexcept;
      void handleInterrupt(uint32_t interruptNumber);
 
     private:
      // plain pointer, because of cyclic dependency
      InterruptDispatcherInterface* _dispatcherInterf;
      SemId _semId;
      ShmForSems* _semShm;
      ShmForSems::Sem* _sem = nullptr;
      // This must not be boost::thread since we don't want interruption points.
      // The problem is that boost::thread::join() is an interruption point for the calling thread. The
      // InterruptDispatcherThread does not rely on boost interruption points to control its tear down, it uses the
      // _stop flag for this purpose.
      std::thread _thr;
      std::atomic_bool _started{false};
      std::atomic_bool _stop{false};
    };
  };
} // namespace ChimeraTK