pixel_detector_calibration.h

Go to the documentation of this file.

// Copyright (C) 2013 Lutz Foucar

/**
 * @file pixel_detector_calibration.h processors that do calibrations on pixel
 *                                    detector data
 *
 * @author Lutz Foucar
 */

#ifndef _PIXELDETECTORCALIBRATION_H_
#define _PIXELDETECTORCALIBRATION_H_

#include <valarray>

#include "processor.h"

#include "statistics_calculator.hpp"

namespace cass
{
//forward declaration
class CASSEvent;

/** pixel detector calibrations
 *
 * @PPList "330": pixel detector calibrations
 *
 * @see Processor for a list of all commonly available cass.ini
 *      settings.
 *
 * @cassttng Processor/\%name\%/{RawImage} \n
 *           the raw image of the pixel detector
 * @cassttng Processor/\%name\%/{OutputFilename} \n
 *           the name of the file where the calibration will be written to.
 *           Default is "NotSet", in which case a filename will be automatically
 *           generated from the time and name of the Processor and a link to
 *           the autogenerated file name will be created which has the same name
 *           but without the time.
 * @cassttng Processor/\%name\%/{WriteCal} \n
 *           Flag to tell whether the calibration should be written. Default is
 *           true.
 * @cassttng Processor/\%name\%/{Train} \n
 *           Flag to tell whether training should be done. Default is true
 * @cassttng Processor/\%name\%/{NbrTrainingImages} \n
 *           The Number of images used for the training. Default is 200
 * @cassttng Processor/\%name\%/{SNR} \n
 *           The signal to noise value that indicates whether a pixel is an
 *           outlier of the distribution of pixels in the trainingset and
 *           should not be considered. Default is 4.
 * @cassttng Processor/\%name\%/{SNRNoiseAutoBoundaries} \n
 *           The signal to noise ratio for determining the boundary values of
 *           bad pixels from the noise map. If this value is negative the manual
 *           set values will be taken from the values below. Otherwise the mean
 *           value and the standart deviation of all noise values will be
 *           calculated and boundaries are set to
 *           \f$ bound_{upper} = mean + (SNRNoiseAutoBoundaries * stdv) \f$
 *           \f$ bound_{lower} = mean - (SNRNoiseAutoBoundaries * stdv) \f$
 *           Default is 4.
 * @cassttng Processor/\%name\%/{SNRNoiseAutoBoundariesStat} \n
 *           In case of using auto generation of the boundaries this signal to \
 *           noise ratio determines which values will be taken when calculating
 *           the mean and stdv of the distribution. Default is 4.
 * @cassttng Processor/\%name\%/{NoiseLowerBoundary|NoiseUpperBoundary} \n
 *           In case the SNRNoiseAutoBoundaries is negative these values will be
 *           taken to determine the bad pixels from the noise map.
 *           Default is 0|3.
 * @cassttng Processor/\%name\%/{SNROffsetAutoBoundaries} \n
 *           The signal to noise ratio for determining the boundary values of
 *           bad pixels from the offset map. If this value is negative the manual
 *           set values will be taken from the values below. Otherwise the mean
 *           value and the standart deviation of all noise values will be
 *           calculated and boundaries are set to
 *           \f$ bound_{upper} = mean + (SNROffsetAutoBoundaries * stdv) \f$
 *           \f$ bound_{lower} = mean - (SNROffsetAutoBoundaries * stdv) \f$
 *           Default is -1.
 * @cassttng Processor/\%name\%/{SNROffsetAutoBoundariesStat} \n
 *           In case of using auto generation of the boundaries this signal to \
 *           noise ratio determines which values will be taken when calculating
 *           the mean and stdv of the distribution. Default is 4.
 * @cassttng Processor/\%name\%/{OffsetLowerBoundary|OffsetUpperBoundary} \n
 *           In case the SNROffsetAutoBoundaries is negative these values will
 *           be taken to determine the bad pixels from the offset map.
 *           Default is -1e20|1e20.
 * @cassttng Processor/\%name\%/{MinNbrPixels} \n
 *           The minimum amount of pixels of the trainingset that are required
 *           for the individual pixel. The value is given in percent of the total
 *           amount of pixels available. If the amount is lower than this value,
 *           the pixel will be marked as bad. Default is 90
 * @cassttng Processor/\%name\%/{InputFilename} \n
 *           The input filname of the calibration file. If it is "NotSet" it
 *           assumes a filename of "\%name\%.lnk" and tries to find the link
 *           target. If it can't find it, it will skip loading of the calibration
 *           file. This also happens if another filename is given, but the file
 *           can't be opened or the size of the images in the file and the
 *           Rawimages differ. Program execution won't be stopped only an error
 *           is written into the log file. Default is "NotSet"
 * @cassttng Processor/\%name\%/{ResetBadPixels} \n
 *           When set to true, it will reset a bad pixel to good if the pixel is
 *           within the checks for beeing good (with in the Noise range and enough
 *           good values). Default is false, which keeps it marked as bad.
 * @cassttng Processor/\%name\%/{UpdateCalibration} \n
 *           Flag to tell whether after the training has completed that the
 *           calibration should be updated with more images. Default is true.
 * @cassttng Processor/\%name\%/{UpdateCalibrationType} \n
 *           The type of update used. Default value is "cummulative". Possible
 *           values are:
 *           - "cummulative": A cummulative update
 *           - "moving": A exponential moving update
 * @cassttng Processor/\%name\%/{NbrOfImages} \n
 *           In case of using the moving update type, this tells how many images
 *           should be considered in the exponential moving average and standart
 *           deviation estimation.
 * @cassttng Processor/\%name\%/{UpdateBadPixPeriod} \n
 *           After how many received images should the bad pixel map be updated
 *           If negative, no updates will be made. Default is -1.
 * @cassttng Processor/\%name\%/{WritePeriod} \n
 *           When automagically updating the dark, this gives the time in
 *           seconds between two writings of the darkcalibration file. Note
 *           that a manual call to the training will force writing even though
 *           the time between two writes has not ellapsed.
 *           Default is 0, which causes no auto writing
 *
 * @author Lutz Foucar
 */
class pp330 : public AccumulatingProcessor
{
public:
  /** constructor. */
  pp330(const name_t&);

  /** overwrite default behaviour don't do anything */
  virtual void process(const CASSEvent&, result_t&);

  /** load the settings of this pp */
  virtual void loadSettings(size_t);

  /** write the calibrations before quitting */
  virtual void aboutToQuit();

  /** react on when the gui clients tell this to start the calibration
   *
   * @param command The command that was issued by the gui clients
   */
  virtual void processCommand(std::string command);

protected:
  /** write the calibration data to file */
  void loadCalibration();

  /** write the calibration data to file */
  void writeCalibration();

  /** set up the bad pixel map */
  void setBadPixMap();

  /** a cummulative average and variance calculation
   *
   * @param[in] image reference to the image array
   * @param[in/out] meanAr iterator to the beginning of the mean part of the output
   * @param[in/out] stdvAr iterator to the beginning of the stdv part of the output
   * @param[in/out] nValsAr iterator to the beginning of the counter part of the output
   * @param[in] sizeOfImage the number of pixels in the image
   */
  void cummulativeUpdate(result_t::const_iterator image,
                         result_t::iterator meanAr,
                         result_t::iterator stdvAr,
                         result_t::iterator nValsAr,
                         const size_t sizeOfImage);

  /** a moving exponential average and variance calculation
   *
   * taken from
   * http://mathforum.org/kb/message.jspa?messageID=1637905
   *
   * @param[in] image reference to the image array
   * @param[in/out] meanAr iterator to the beginning of the mean part of the output
   * @param[in/out] stdvAr iterator to the beginning of the stdv part of the output
   * @param[in/out] nValsAr iterator to the beginning of the counter part of the output
   * @param[in] sizeOfImage the number of pixels in the image
   */
  void movingUpdate(result_t::const_iterator image,
                    result_t::iterator meanAr,
                    result_t::iterator stdvAr,
                    result_t::iterator nValsAr,
                    const size_t sizeOfImage);

private:
  /** define the function */
  typedef std::tr1::function<void(result_t::const_iterator,
                                  result_t::iterator,
                                  result_t::iterator,
                                  result_t::iterator,
                                  const size_t)> updateFunc_t;

  /** the function that will update the statistics */
  updateFunc_t _updateStatistics;

private:
  /** define the position of the beginning of the different maps */
  enum {MEAN,STDV,NVALS,BADPIX,nbrOfOutputs};

  /** the offset to the first point of the mean array in the result */
  size_t _meanBeginOffset;

  /** the offset to one beyond the last point of the mean array in the result */
  size_t _meanEndOffset;

  /** the offset to the first point of the stdv array in the result */
  size_t _stdvBeginOffset;

  /** the offset to one beyond the last point of the stdv array in the result */
  size_t _stdvEndOffset;

  /** the offset to the first point of the bad pixels array in the result */
  size_t _bPixBeginOffset;

  /** the offset to one beyond the last point of the bad pixels array in the result */
  size_t _bPixEndOffset;

  /** the offset to the first point of the counts array in the result */
  size_t _nValBeginOffset;

  /** the offset to the first point of the counts array in the result */
  size_t _nValEndOffset;

  /** the lower noise boundary when determining bad pixels */
  float _NoiseLowerBound;

  /** the upper noise boundary when determining bad pixels */
  float _NoiseUpperBound;

  /** the signal to noise ratio that determines the noise boundaries in case of
   *  automatically determining the boundaries from the noise values statistics
   */
  float _autoNoiseSNR;

  /** the signal to noise ratio that determines which values will be taken into
   *  account when automatically determining the boundaries from the noise
   *  values statistics
   */
  float _autoNoiseSNRStat;

  /** the lower offset boundary when determining bad pixels */
  float _OffsetLowerBound;

  /** the upper offset boundary when determining bad pixels */
  float _OffsetUpperBound;

  /** the signal to noise ratio that determines the offset boundaries in case of
   *  automatically determining the boundaries from the noise values statistics
   */
  float _autoOffsetSNR;

  /** the signal to noise ratio that determines which values will be taken into
   *  account when automatically determining the boundaries from the offset
   *  values statistics
   */
  float _autoOffsetSNRStat;

  /** minimum number of pixels in the trainig set that are part of the distribution */
  float _minNbrPixels;

  /** the raw image */
  shared_pointer _image;

  /** flag telling whether training is needed */
  bool _train;

  /** the storage for the training images */
  std::vector< result_t::shared_pointer > _trainstorage;

  /** the minimum nbr of images necessary for training */
  size_t _minTrainImages;

  /** flag to tell whether the calibration should be written */
  bool _write;

  /** the value above which outliers are removed from the distribution */
  float _snr;

  /** the filename that is used to save the calibration */
  std::string _filename;

  /** the filename that is used to load the calibration */
  std::string _infilename;

  /** flag to tell when the bad pixel should be reset */
  bool _resetBadPixel;

  /** counter to count the total amount of images */
  size_t _counter;

  /** flag to tell whether the calibration should be updated after the training
   *  has completed.
   */
  bool _update;

  /** the period after which the data should be autosaved */
  int _updatePeriod;

  /** the alpha value for the moving average and stdv */
  float _alpha;

  /** the time_t value of the last time the calibration was written */
  uint32_t _lastwritten;

  /** the update time period in s */
  uint32_t _updateWritePeriod;
};





/** pixel detector gain calibrations
 *
 * @PPList "331": pixel detector gain calibrations
 *
 * @see Processor for a list of all commonly available cass.ini
 *      settings.
 *
 * The output of this Processor consits of 3 images stiched together along
 * the y (slow) coordinate. The ouput has the same x width as the input, but 3
 * times the y width as the input.
 *
 * Y from 0 * y_input .. 1 * y_input-1: The calculated Gain Map
 * Y from 1 * y_input .. 2 * y_input-1: The photons counts map
 * Y from 2 * y_input .. 3 * y_input-1: The Average value of the photons of
 *                                      interest in that pixel
 *
 * @cassttng Processor/\%name\%/{Image} \n
 *           the image of the pixel detector
 * @cassttng Processor/\%name\%/{Filename} \n
 *           the name of the file where the calibration will be written to.
 *           Default is out.cal
 * @cassttng Processor/\%name\%/{WriteCal} \n
 *           Flag to tell whether the calibration should be written. Default is
 *           true.
 * @cassttng Processor/\%name\%/{ADURangeLow|ADURangeHigh} \n
 *           The adu range that indicates that one photon has hit the pixel.
 *           Default is 0|0
 * @cassttng Processor/\%name\%/{MinimumNbrPhotons} \n
 *           The minimum number of photons that a pixel should have seen before
 *           the gain is calulated for this pixel. Default is 200
 * @cassttng Processor/\%name\%/{DefaultGainValue} \n
 *           The gain value that will be assinged to the pixels that haven't
 *           seen enough photons. Default is 1.
 * @cassttng Processor/\%name\%/{NbrOfFrames} \n
 *           The number of frames after which the gain map will be calculated.
 *           Default is -1, which sais that it will never be calulated during
 *           running and only when the program ends.
 * @cassttng Processor/\%name\%/{IsPnCCDNoCTE}\n
 *           If the detector is a pnCCD and one doesn't want to correct for the
 *           CTE this option will calcultate the gain for a column of a quadrant
 *           of the detector. Default is false
 *
 * @author Lutz Foucar
 */
class pp331 : public AccumulatingProcessor
{
public:
  /** constructor. */
  pp331(const name_t&);

  /** overwrite default behaviour don't do anything */
  virtual void process(const CASSEvent&, result_t&);

  /** load the settings of this pp */
  virtual void loadSettings(size_t);

  /** write the calibrations before quitting */
  virtual void aboutToQuit();

  /** receive commands from the gui */
  virtual void processCommand(std::string command);

protected:
  /** write the calibration data to file */
  void loadCalibration();

  /** write the calibration data to file */
  void writeCalibration();

  /** write the calibration data to file */
  void calculateGainMap(result_t& gainmap);

private:
  /** the raw image */
  shared_pointer _image;

  /** definition of the statistic */
  typedef std::pair<size_t,double> statistic_t;

  /** defintion of the statistics */
  typedef std::vector<statistic_t> statistics_t;

  /** the statistics for each pixel */
  statistics_t _statistics;

  /** the number of photons that a pixel should have seen before calculating the gain */
  size_t _minPhotonCount;

  /** define the range type */
  typedef std::pair<float,float> range_t;

  /** the range of adu that indicates whether a pixel contains a photon */
  range_t _aduRange;

  /** the gain value that will be assinged to the pixel that one could not
   *  calculate the gain for
   */
  float _constGain;

  /** flag to tell whether the calibration should be written */
  bool _write;

  /** the filename that is used to save the calibration */
  std::string _filename;

  /** the number of frames after which the gain map is calculted */
  int _nFrames;

  /** counter to count how many times this has been called */
  int _counter;

  /** offset to the gain part */
  size_t _gainOffset;

  /** offset to the counts part */
  size_t _countOffset;

  /** offset to the average part */
  size_t _aveOffset;

  /** the size of the input image */
  size_t _sizeOfImage;

  /** flag to tell whether its a pnCCD we are not interested in correction the cte */
  bool _isPnCCDNoCTE;
};










/** pixel detector hot pixel detection
 *
 * @PPList "332": pixel detector hot pixel detection
 *
 * @see Processor for a list of all commonly available cass.ini
 *      settings.
 *
 * @cassttng Processor/\%name\%/{Image} \n
 *           the image of the pixel detector
 * @cassttng Processor/\%name\%/{Filename} \n
 *           the name of the file where the calibration will be written to.
 *           Default is out.cal
 * @cassttng Processor/\%name\%/{WriteCal} \n
 *           Flag to tell whether the calibration should be written. Default is
 *           true.
 * @cassttng Processor/\%name\%/{ADURangeLow|ADURangeUp} \n
 *           The adu range that indicates that one photon has hit the pixel.
 *           Default is 0|0
 * @cassttng Processor/\%name\%/{MaximumConsecutiveFrames} \n
 *           The maximum number of frames that a pixel should have an adu value
 *           in the range before the pixel is mased as hot. Default is 5
 * @cassttng Processor/\%name\%/{MaxADUValue} \n
 *           If a pixel ever exceeds this value it will be masked as bad.
 *           Default is 1e6
 * @cassttng Processor/\%name\%/{NbrOfFrames} \n
 *           The number of frames after which the gain map will be calculated.
 *           Default is -1, which sais that it will never be calulated during
 *           running and only when the program ends.
 *
 * @author Lutz Foucar
 */
class pp332 : public AccumulatingProcessor
{
public:
  /** constructor. */
  pp332(const name_t&);

  /** overwrite default behaviour don't do anything */
  virtual void process(const CASSEvent&, result_t&);

  /** load the settings of this pp */
  virtual void loadSettings(size_t);

  /** write the calibrations before quitting */
  virtual void aboutToQuit();

protected:
  /** write the calibration data to file */
  void loadHotPixelMap();

  /** write the calibration data to file */
  void writeHotPixelMap();

private:
  /** define the output mask type */
  typedef char mask_t;

  /** the image to create the hotpixel map from */
  shared_pointer _image;

  /** the number of times a pixel is high before masking it as hot pixel */
  size_t _maxConsecutiveCount;

  /** define the range type */
  typedef std::pair<float,float> range_t;

  /** the range of adu that indicates whether a pixel is hot */
  range_t _aduRange;

  /** the maximum allowed adu value */
  float _maxADUVal;

  /** flag to tell whether the calibration should be written */
  bool _write;

  /** the filename that is used to save the calibration */
  std::string _filename;

  /** the number of frames after which the gain map is calculted */
  int _nFrames;

  /** counter to count how many times this has been called */
  int _counter;
};







/** pixel detector common mode background calculation
 *
 * @PPList "333": pixel detector common mode background calculation
 *
 * @see Processor for a list of all commonly available cass.ini
 *      settings.
 *
 * @cassttng Processor/\%name\%/{Image} \n
 *           the image of the pixel detector
 * @cassttng Processor/\%name\%/{Width} \n
 *           The width of the subpart of the detector for which the common
 *           mode level needs to be calculated.
 * @cassttng Processor/\%name\%/{CalculationType} \n
 *           The type of calculation used to calculate the common mode level.
 *           Default is "mean". Possible values are:
 *           - "mean": Iterative retrieval of the mean value of distribution of
 *                     pixels. In each iteration the outliers of the distribution
 *                     are remove from the distribution.
 *           - "median": Calculates the mean of the distribution of pixels.
 * @cassttng Processor/\%name\%/{SNR} \n
 *           In case of using the mean calculator the signal to noise ratio is
 *           indicating which values to remove from the distribution before
 *           recalculating the mean. Default is 4
 *
 * @author Lutz Foucar
 */
class pp333 : public Processor
{
public:
  /** constructor. */
  pp333(const name_t&);

  /** overwrite default behaviour don't do anything */
  virtual void process(const CASSEvent&, result_t&);

  /** load the settings of this pp */
  virtual void loadSettings(size_t);

protected:
  /** function to calulate the common mode level as mean value
   *
   * @return the commonmode level
   * @param begin iterator to the beginning of the distribution
   * @param end iterator to the end of the distribution
   */
  float meanCalc(result_t::const_iterator begin,
                 result_t::const_iterator end);

  /** function to calulate the common mode level via the median value
   *
   * @return the commonmode level
   * @param begin iterator to the beginning of the distribution
   * @param end iterator to the end of the distribution
   */
  float medianCalc(result_t::const_iterator begin,
                   result_t::const_iterator end);
private:
  /** the image to create the hotpixel map from */
  shared_pointer _image;

  /** the number of times a pixel is high before masking it as hot pixel */
  size_t _width;

  /** the signal to noise ratio in case one uses the mean calculator */
  float _snr;

  /** the function that calculates the commond mode level */
  std::tr1::function<float(result_t::const_iterator,
                           result_t::const_iterator)> _calcCommonmode;
};







/** pixel detector common mode background calculation
 *
 * @PPList "334": pixel detector common mode calc using histograms
 *
 * @see Processor for a list of all commonly available cass.ini
 *      settings.
 *
 * @cassttng Processor/\%name\%/{Image} \n
 *           the image of the pixel detector
 * @cassttng Processor/\%name\%/{Width} \n
 *           when using the histogram option this is the width of the 0 photons
 *           peak that the CsPAD typically has. Default is 30
 * @cassttng Processor/\%name\%/{MaxDistance} \n
 *           How much should the histogram value differ at the most from the
 *           unbonded pixel value, before the value of the unbonded pixel is
 *           taken. Default is 5
 * @cassttng Processor/\%name\%/{EnableChecks} \n
 *           Switch that allows to enable checks by outputting the histograms,
 *           and the found maxima. Default is false.
 *
 * @author Lutz Foucar
 */
class pp334 : public Processor
{
public:
  /** constructor. */
  pp334(const name_t&);

  /** overwrite default behaviour don't do anything */
  virtual void process(const CASSEvent&, result_t&);

  /** load the settings of this pp */
  virtual void loadSettings(size_t);

private:
  /** the image to create the hotpixel map from */
  shared_pointer _image;

  /** the number of times a pixel is high before masking it as hot pixel */
  float _width;

  /** the maximum distance between the value of the unbonded pxiels and the hist */
  float _maxDist;

  /** enable output that allow to check the parameters */
  bool _checks;
};

}//end namespace cass
#endif