Commit 53e3e7fb authored by Simon Conseil's avatar Simon Conseil

Merge branch 'combine' into 'master'

Allow to correct zero offset in mpdaf_merging_sigma_clipping

See merge request !163
parents 6a6166f1 279f97a2
Pipeline #3149 passed with stage
in 21 minutes and 21 seconds
...@@ -40,6 +40,7 @@ from astropy import units as u ...@@ -40,6 +40,7 @@ from astropy import units as u
from astropy.table import Table from astropy.table import Table
from astropy.utils.console import ProgressBar from astropy.utils.console import ProgressBar
from ctypes import c_char_p from ctypes import c_char_p
from datetime import datetime
from numpy import allclose, array_equal from numpy import allclose, array_equal
from .cube import Cube from .cube import Cube
...@@ -139,7 +140,7 @@ def _pycombine(self, nmax=2, nclip=5.0, var='propagate', nstop=2, nl=None, ...@@ -139,7 +140,7 @@ def _pycombine(self, nmax=2, nclip=5.0, var='propagate', nstop=2, nl=None,
if self.flux_scales is None and self.flux_offsets is None: if self.flux_scales is None and self.flux_offsets is None:
for l in range(nl): for l in range(nl):
if l % 100 == 0: if l % 100 == 0:
info('%d/%d', l, nl) info('%d/%d %s', l, nl, datetime.now())
arr.fill(np.nan) arr.fill(np.nan)
for i, f in enumerate(data): for i, f in enumerate(data):
x, y, x2, y2 = pos[i] x, y, x2, y2 = pos[i]
...@@ -158,17 +159,17 @@ def _pycombine(self, nmax=2, nclip=5.0, var='propagate', nstop=2, nl=None, ...@@ -158,17 +159,17 @@ def _pycombine(self, nmax=2, nclip=5.0, var='propagate', nstop=2, nl=None,
scales = np.ones(self.nfiles) scales = np.ones(self.nfiles)
else: else:
scales = np.asarray(self.flux_scales) scales = np.asarray(self.flux_scales)
self._logger.info('Using scales: %s', scales) self._logger.info('Using scales')
if self.flux_offsets is None: if self.flux_offsets is None:
offsets = np.zeros(self.nfiles) offsets = np.zeros(self.nfiles)
else: else:
offsets = np.asarray(self.flux_offsets) offsets = np.asarray(self.flux_offsets)
self._logger.info('Using offsets: %s', offsets) self._logger.info('Using offsets')
for l in range(nl): for l in range(nl):
if l % 100 == 0: if l % 100 == 0:
info('%d/%d', l, nl) info('%d/%d %s', l, nl, datetime.now())
arr.fill(np.nan) arr.fill(np.nan)
for i, f in enumerate(data): for i, f in enumerate(data):
x, y, x2, y2 = pos[i] x, y, x2, y2 = pos[i]
...@@ -335,6 +336,11 @@ class CubeList: ...@@ -335,6 +336,11 @@ class CubeList:
str(c.wcs.wcs.wcs.crpix), str(c.wcs.wcs.wcs.crval)) str(c.wcs.wcs.wcs.crpix), str(c.wcs.wcs.wcs.crval))
for c in self.cubes] for c in self.cubes]
t = Table(rows=rows, names=('filename', 'shape', 'crpix', 'crval')) t = Table(rows=rows, names=('filename', 'shape', 'crpix', 'crval'))
if self.flux_scales is not None:
t['scale'] = self.flux_scales
if self.flux_offsets is not None:
t['offset'] = self.flux_offsets
for line in t.pformat(): for line in t.pformat():
self._logger.info(line) self._logger.info(line)
...@@ -492,12 +498,19 @@ class CubeList: ...@@ -492,12 +498,19 @@ class CubeList:
files = files.encode('utf8') files = files.encode('utf8')
if self.flux_scales is None: if self.flux_scales is None:
scales = np.ones(self.nfiles) scale = np.ones(self.nfiles)
else: else:
scales = np.asarray(self.flux_scales) scale = np.asarray(self.flux_scales)
self._logger.info('Using scales')
if self.flux_offsets is None:
offset = np.zeros(self.nfiles)
else:
offset = np.asarray(self.flux_offsets)
self._logger.info('Using offsets')
ctools.mpdaf_merging_sigma_clipping( ctools.mpdaf_merging_sigma_clipping(
c_char_p(files), data, vardata, expmap, scales, select_pix, c_char_p(files), data, vardata, expmap, scale, offset, select_pix,
valid_pix, nmax, np.float64(nclip_low), np.float64(nclip_up), valid_pix, nmax, np.float64(nclip_low), np.float64(nclip_up),
nstop, np.int32(var_mean), np.int32(mad)) nstop, np.int32(var_mean), np.int32(mad))
......
...@@ -61,15 +61,24 @@ class TestCubeList(unittest.TestCase): ...@@ -61,15 +61,24 @@ class TestCubeList(unittest.TestCase):
shape = (5, 4, 3) shape = (5, 4, 3)
ncubes = 3 ncubes = 3
cubevals = [0, 1, 5] cubevals = np.array([0, 1, 5])
scalelist = [0.5, 1, 1.5]
offsetlist = [1.5, 2, 2.5]
@classmethod @classmethod
def setUpClass(cls): def setUpClass(cls):
cls.tmpdir = tempfile.mkdtemp() cls.tmpdir = tempfile.mkdtemp()
print('\n>>> Create cubes in', cls.tmpdir) print('\n>>> Create cubes in', cls.tmpdir)
cls.cubenames = [] cls.cubenames = []
cls.arr = np.arange(np.prod(cls.shape)).reshape(cls.shape)
cls.scaledcube = np.mean([
(cls.arr * val + cls.offsetlist[k]) * cls.scalelist[k]
for k, val in enumerate(cls.cubevals)], axis=0)
cls.combined_cube = np.mean([(cls.arr * i) for i in cls.cubevals],
axis=0)
for i in cls.cubevals: for i in cls.cubevals:
cube = generate_cube(data=i, shape=cls.shape) cube = generate_cube(data=cls.arr * i, shape=cls.shape)
cube.primary_header['CUBEIDX'] = i cube.primary_header['CUBEIDX'] = i
cube.primary_header['OBJECT'] = 'OBJECT %d' % i cube.primary_header['OBJECT'] = 'OBJECT %d' % i
cube.primary_header['EXPTIME'] = 100 cube.primary_header['EXPTIME'] = 100
...@@ -92,10 +101,10 @@ class TestCubeList(unittest.TestCase): ...@@ -92,10 +101,10 @@ class TestCubeList(unittest.TestCase):
def test_get_item(self): def test_get_item(self):
clist = CubeList(self.cubenames) clist = CubeList(self.cubenames)
assert_array_equal(clist[0, 2, 2], self.cubevals) assert_array_equal(clist[0, 0, 1], self.cubevals)
assert_array_equal(np.array([a.data for a in clist[0, :, :]])[:, 0, 0], assert_array_equal(np.array([a.data for a in clist[0, :, :]])[:, 0, 1],
self.cubevals) self.cubevals)
assert_array_equal(np.array([a.data for a in clist[0]])[:, 0, 0], assert_array_equal(np.array([a.data for a in clist[0]])[:, 0, 1],
self.cubevals) self.cubevals)
def test_checks(self): def test_checks(self):
...@@ -114,26 +123,22 @@ class TestCubeList(unittest.TestCase): ...@@ -114,26 +123,22 @@ class TestCubeList(unittest.TestCase):
@pytest.mark.skipif(not HAS_CFITSIO, reason="requires cfitsio") @pytest.mark.skipif(not HAS_CFITSIO, reason="requires cfitsio")
def test_median(self): def test_median(self):
clist = CubeList(self.cubenames) clist = CubeList(self.cubenames)
combined_cube = np.ones(self.shape)
cube, expmap, stat_pix = clist.median(header={'FOO': 'BAR'}) cube, expmap, stat_pix = clist.median(header={'FOO': 'BAR'})
self.assert_header(cube) self.assert_header(cube)
assert_array_equal(cube.data, combined_cube) assert_array_equal(cube.data, self.arr)
assert_array_equal(expmap.data, self.expmap) assert_array_equal(expmap.data, self.expmap)
@pytest.mark.skipif(not HAS_FITSIO, reason="requires fitsio") @pytest.mark.skipif(not HAS_FITSIO, reason="requires fitsio")
def test_pymedian(self): def test_pymedian(self):
clist = CubeList(self.cubenames) clist = CubeList(self.cubenames)
combined_cube = np.ones(self.shape)
cube, expmap, stat_pix = clist.pymedian(header={'FOO': 'BAR'}) cube, expmap, stat_pix = clist.pymedian(header={'FOO': 'BAR'})
self.assert_header(cube) self.assert_header(cube)
assert_array_equal(cube.data, combined_cube) assert_array_equal(cube.data, self.arr)
assert_array_equal(expmap.data, self.expmap) assert_array_equal(expmap.data, self.expmap)
@pytest.mark.skipif(not HAS_CFITSIO, reason="requires cfitsio") @pytest.mark.skipif(not HAS_CFITSIO, reason="requires cfitsio")
def test_combine(self): def test_combine(self):
clist = CubeList(self.cubenames) clist = CubeList(self.cubenames)
combined_cube = np.full(self.shape, 2, dtype=float)
cube, expmap, stat_pix = clist.combine(header={'FOO': 'BAR'}) cube, expmap, stat_pix = clist.combine(header={'FOO': 'BAR'})
cube2, expmap2, stat_pix2 = clist.combine(header={'FOO': 'BAR'}, cube2, expmap2, stat_pix2 = clist.combine(header={'FOO': 'BAR'},
mad=True) mad=True)
...@@ -141,17 +146,15 @@ class TestCubeList(unittest.TestCase): ...@@ -141,17 +146,15 @@ class TestCubeList(unittest.TestCase):
assert_array_equal(expmap.data, expmap2.data) assert_array_equal(expmap.data, expmap2.data)
self.assert_header(cube) self.assert_header(cube)
assert_array_equal(cube.data, combined_cube) assert_array_equal(cube.data, self.combined_cube)
assert_array_equal(expmap.data, self.expmap) assert_array_equal(expmap.data, self.expmap)
cube = clist.combine(nclip=(5., 5.), var='stat_mean')[0] cube = clist.combine(nclip=(5., 5.), var='stat_mean')[0]
assert_array_equal(cube.data, combined_cube) assert_array_equal(cube.data, self.combined_cube)
@pytest.mark.skipif(not HAS_FITSIO, reason="requires fitsio") @pytest.mark.skipif(not HAS_FITSIO, reason="requires fitsio")
def test_pycombine(self): def test_pycombine(self):
clist = CubeList(self.cubenames) clist = CubeList(self.cubenames)
combined_cube = np.full(self.shape, 2, dtype=float)
cube, expmap, stat_pix, rejmap = clist.pycombine(header={'FOO': 'BAR'}) cube, expmap, stat_pix, rejmap = clist.pycombine(header={'FOO': 'BAR'})
cube2, expmap2, stat_pix2, rejmap2 = clist.pycombine( cube2, expmap2, stat_pix2, rejmap2 = clist.pycombine(
header={'FOO': 'BAR'}, mad=True) header={'FOO': 'BAR'}, mad=True)
...@@ -159,43 +162,33 @@ class TestCubeList(unittest.TestCase): ...@@ -159,43 +162,33 @@ class TestCubeList(unittest.TestCase):
assert_array_equal(expmap.data, expmap2.data) assert_array_equal(expmap.data, expmap2.data)
self.assert_header(cube) self.assert_header(cube)
assert_array_equal(cube.data, combined_cube) assert_array_equal(cube.data, self.combined_cube)
assert_array_equal(expmap.data, self.expmap) assert_array_equal(expmap.data, self.expmap)
cube = clist.pycombine(nclip=(5., 5.), var='stat_mean')[0] cube = clist.pycombine(nclip=(5., 5.), var='stat_mean')[0]
assert_array_equal(cube.data, combined_cube) assert_array_equal(cube.data, self.combined_cube)
@pytest.mark.skipif(not HAS_CFITSIO, reason="requires cfitsio") @pytest.mark.skipif(not HAS_CFITSIO, reason="requires cfitsio")
def test_combine_scale(self): def test_combine_scale(self):
clist = CubeList(self.cubenames, scalelist=[2.] * self.ncubes) clist = CubeList(self.cubenames, scalelist=self.scalelist,
combined_cube = np.full(self.shape, 2 * 2, dtype=float) offsetlist=self.offsetlist)
cube, expmap, stat_pix = clist.combine(header={'FOO': 'BAR'}) cube, expmap, stat_pix = clist.combine(header={'FOO': 'BAR'})
assert_array_equal(cube.data, combined_cube) assert_array_equal(cube.data, self.scaledcube)
@pytest.mark.skipif(not HAS_FITSIO, reason="requires fitsio") @pytest.mark.skipif(not HAS_FITSIO, reason="requires fitsio")
def test_pycombine_scale(self): def test_pycombine_scale(self):
clist = CubeList(self.cubenames, scalelist=[2.] * self.ncubes) clist = CubeList(self.cubenames, scalelist=self.scalelist,
combined_cube = np.full(self.shape, 2 * 2, dtype=float) offsetlist=self.offsetlist)
cube2, expmap2, _, _ = clist.pycombine(header={'FOO': 'BAR'}) cube2, expmap2, _, _ = clist.pycombine(header={'FOO': 'BAR'})
assert_array_equal(cube2.data, combined_cube) assert_array_equal(cube2.data, self.scaledcube)
clist = CubeList(self.cubenames, scalelist=[2.] * self.ncubes,
offsetlist=[0.5] * self.ncubes)
combined_cube = np.full(self.shape, 5, dtype=float)
cube2, expmap2, _, _ = clist.pycombine(header={'FOO': 'BAR'})
assert_array_equal(cube2.data, combined_cube)
@pytest.mark.skipif(not HAS_FITSIO, reason="requires fitsio") @pytest.mark.skipif(not HAS_FITSIO, reason="requires fitsio")
def test_mosaic_combine(self): def test_mosaic_combine(self):
clist = CubeMosaic(self.cubenames, self.cubenames[0]) clist = CubeMosaic(self.cubenames, self.cubenames[0])
combined_cube = np.full(self.shape, 2, dtype=float)
cube, expmap, stat_pix, rejmap = clist.pycombine(header={'FOO': 'BAR'}) cube, expmap, stat_pix, rejmap = clist.pycombine(header={'FOO': 'BAR'})
self.assert_header(cube) self.assert_header(cube)
assert_array_equal(cube.data, combined_cube) assert_array_equal(cube.data, self.combined_cube)
assert_array_equal(expmap.data, self.expmap) assert_array_equal(expmap.data, self.expmap)
cube2, expmap2, _, _ = clist.pycombine(header={'FOO': 'BAR'}, mad=True) cube2, expmap2, _, _ = clist.pycombine(header={'FOO': 'BAR'}, mad=True)
......
...@@ -81,6 +81,7 @@ else: ...@@ -81,6 +81,7 @@ else:
array_1d_double, # double* var array_1d_double, # double* var
array_1d_int, # int* expmap array_1d_int, # int* expmap
array_1d_double, # double* scale array_1d_double, # double* scale
array_1d_double, # double* offset
array_1d_int, # int* selected_pix array_1d_int, # int* selected_pix
array_1d_int, # int* valid_pix array_1d_int, # int* valid_pix
ctypes.c_int, # int nmax ctypes.c_int, # int nmax
......
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment