Source code for halotools.sim_manager.user_supplied_halo_catalog

""" Module containing the UserSuppliedHaloCatalog class.
"""

import numpy as np
import os

from warnings import warn
import datetime

from astropy.table import Table, Column

from .halo_table_cache import HaloTableCache
from .halo_table_cache_log_entry import HaloTableCacheLogEntry, get_redshift_string
from .user_supplied_ptcl_catalog import UserSuppliedPtclCatalog

from ..utils.array_utils import custom_len

from ..custom_exceptions import HalotoolsError

__all__ = ('UserSuppliedHaloCatalog', )


[docs] class UserSuppliedHaloCatalog(object): """ Class used to transform a user-provided halo catalog into the standard form recognized by Halotools. See :ref:`user_supplied_halo_catalogs` for a tutorial on this class. """ def __init__(self, **kwargs): """ Parameters ------------ **metadata : float or string Keyword arguments storing catalog metadata. The quantities `Lbox` and `particle_mass` are required and must be in Mpc/h and Msun/h units, respectively. `redshift` is also required metadata. See Examples section for further notes. **halo_catalog_columns : sequence of arrays Sequence of length-*Nhalos* arrays passed in as keyword arguments. Each key will be the column name attached to the input array. All keys must begin with the substring ``halo_`` to help differentiate halo property from mock galaxy properties. At a minimum, there must be a ``halo_id`` keyword argument storing a unique integer for each halo, as well as columns ``halo_x``, ``halo_y`` and ``halo_z``. There must also be some additional mass-like variable, for which you can use any name that begins with ``halo_`` See Examples section for further notes. user_supplied_ptclcat : table, optional Instance of the `~halotools.sim_manager.UserSuppliedPtclCatalog` class. If this keyword is passed, the `UserSuppliedHaloCatalog` instance will have a ``ptcl_table`` attribute bound to it storing dark matter particles randomly selected from the snapshot. At a minimum, the table must have columns ``x``, ``y`` and ``z``. Default is None. Examples ---------- Here is an example using dummy data to show how to create a new `UserSuppliedHaloCatalog` instance from from your own halo catalog. First the setup: >>> redshift = 0.0 >>> Lbox = 250. >>> particle_mass = 1e9 >>> num_halos = 100 >>> x = np.random.uniform(0, Lbox, num_halos) >>> y = np.random.uniform(0, Lbox, num_halos) >>> z = np.random.uniform(0, Lbox, num_halos) >>> mass = np.random.uniform(1e12, 1e15, num_halos) >>> ids = np.arange(0, num_halos) Now we simply pass in both the metadata and the halo catalog columns as keyword arguments: >>> halo_catalog = UserSuppliedHaloCatalog(redshift = redshift, Lbox = Lbox, particle_mass = particle_mass, halo_x = x, halo_y = y, halo_z = z, halo_id = ids, halo_mvir = mass) Your ``halo_catalog`` object can be used throughout the Halotools package. The halo catalog itself is stored in the ``halo_table`` attribute, with columns accessed as follows: >>> array_of_masses = halo_catalog.halo_table['halo_mvir'] >>> array_of_x_positions = halo_catalog.halo_table['halo_x'] Each piece of metadata you passed in can be accessed as an ordinary attribute: >>> halo_catalog_box_size = halo_catalog.Lbox >>> particle_mass = halo_catalog.particle_mass If you wish to pass in additional metadata, just include additional keywords: >>> simname = 'my_personal_sim' >>> halo_catalog = UserSuppliedHaloCatalog(redshift = redshift, simname = simname, Lbox = Lbox, particle_mass = particle_mass, halo_x = x, halo_y = y, halo_z = z, halo_id = ids, halo_mvir = mass) Similarly, if you wish to include additional columns for your halo catalog, Halotools is able to tell the difference between metadata and columns of halo data: >>> spin = np.random.uniform(0, 0.2, num_halos) >>> halo_catalog = UserSuppliedHaloCatalog(redshift = redshift, halo_spin = spin, simname = simname, Lbox = Lbox, particle_mass = particle_mass, halo_x = x, halo_y = y, halo_z = z, halo_id = ids, halo_mvir = mass) If you want to store your halo catalog in the Halotools cache, use the `add_halocat_to_cache` method. You also have the option to supply a randomly selected downsample of dark matter particles via the ``user_supplied_ptclcat`` keyword. This keyword have an instance of the `~halotools.sim_manager.UserSuppliedPtclCatalog` class bound to it, which helps ensure consistency between the halo catalog and particles. Here's an example of how to use this argument using some fake data: >>> num_ptcls = int(1e4) >>> ptcl_x = np.random.uniform(0, Lbox, num_ptcls) >>> ptcl_y = np.random.uniform(0, Lbox, num_ptcls) >>> ptcl_z = np.random.uniform(0, Lbox, num_ptcls) >>> from halotools.sim_manager import UserSuppliedPtclCatalog >>> ptclcat = UserSuppliedPtclCatalog(x = ptcl_x, y = ptcl_y, z = ptcl_z, Lbox = Lbox, particle_mass = particle_mass, redshift = redshift) >>> halo_catalog = UserSuppliedHaloCatalog(user_supplied_ptclcat = ptclcat, redshift = redshift, halo_spin = spin, simname = simname, Lbox = Lbox, particle_mass = particle_mass, halo_x = x, halo_y = y, halo_z = z, halo_id = ids, halo_mvir = mass) In some scenarios, you may already have tabular data stored in an Astropy `astropy.table.Table` or a Numpy structured array. If you want to transfer *all* the columns of your ``table_of_halos`` to the `UserSuppliedHaloCatalog`, then you can do so by splatting a python dictionary view of the table: >>> from astropy.table import Table >>> table_of_halos = Table() >>> num_halos = int(1e3) >>> Lbox = 250. >>> table_of_halos['halo_mass'] = np.random.uniform(1e10, 1e15, num_halos) >>> table_of_halos['halo_x'] = np.random.uniform(0, Lbox, num_halos) >>> table_of_halos['halo_y'] = np.random.uniform(0, Lbox, num_halos) >>> table_of_halos['halo_z'] = np.random.uniform(0, Lbox, num_halos) >>> table_of_halos['halo_jcvd'] = np.random.random(num_halos) >>> table_of_halos['halo_id'] = np.arange(num_halos).astype('i8') >>> d = {key:table_of_halos[key] for key in table_of_halos.keys()} >>> halocat = UserSuppliedHaloCatalog(simname = simname, redshift = redshift, Lbox = Lbox, particle_mass = particle_mass, **d) """ halo_table_dict, metadata_dict = self._parse_constructor_kwargs(**kwargs) self.halo_table = Table(halo_table_dict) self._test_metadata_dict(**metadata_dict) # make Lbox a 3-vector _Lbox = metadata_dict.pop('Lbox') metadata_dict['Lbox'] = np.empty(3) metadata_dict['Lbox'][:] = _Lbox for key, value in metadata_dict.items(): setattr(self, key, value) self._passively_bind_ptcl_table(**kwargs) def _parse_constructor_kwargs(self, **kwargs): """ Private method interprets constructor keyword arguments and returns two dictionaries. One stores the halo catalog columns, the other stores the metadata. Parameters ------------ **kwargs : keyword arguments passed to constructor Returns ---------- halo_table_dict : dictionary Keys are the names of the halo catalog columns, values are length-*Nhalos* ndarrays. metadata_dict : dictionary Dictionary storing the catalog metadata. Keys will be attribute names bound to the `UserSuppliedHaloCatalog` instance. """ try: halo_id = np.array(kwargs['halo_id']) assert type(halo_id) is np.ndarray Nhalos = custom_len(halo_id) except KeyError: msg = ("\nThe UserSuppliedHaloCatalog requires a ``halo_id`` keyword argument.") raise HalotoolsError(msg) halo_table_dict = ( {key: np.array(kwargs[key]) for key in kwargs if ((type(kwargs[key]) is np.ndarray) | (type(kwargs[key]) is Column)) and (custom_len(kwargs[key]) == Nhalos) and (key[:5] == 'halo_')}) self._test_halo_table_dict(halo_table_dict) metadata_dict = ( {key: kwargs[key] for key in kwargs if (key not in halo_table_dict) and (key != 'ptcl_table')} ) return halo_table_dict, metadata_dict def _test_halo_table_dict(self, halo_table_dict): """ """ try: assert 'halo_x' in halo_table_dict assert 'halo_y' in halo_table_dict assert 'halo_z' in halo_table_dict assert len(halo_table_dict) >= 5 except AssertionError: msg = ("\nThe UserSuppliedHaloCatalog requires keyword arguments ``halo_x``, " "``halo_y`` and ``halo_z``,\nplus one additional column storing a mass-like variable.\n" "Each of these keyword arguments must storing an ndarray of the same length\n" "as the ndarray bound to the ``halo_id`` keyword argument.\n") raise HalotoolsError(msg) def _test_metadata_dict(self, **metadata_dict): """ """ try: assert 'Lbox' in metadata_dict assert custom_len(metadata_dict['Lbox']) in [1,3] except AssertionError: msg = ("\nThe UserSuppliedPtclCatalog requires keyword argument " "``Lbox``, storing either a scalar or 3-vector.\n") raise HalotoolsError(msg) try: assert 'particle_mass' in metadata_dict assert custom_len(metadata_dict['particle_mass']) == 1 assert 'redshift' in metadata_dict except AssertionError: msg = ("\nThe UserSuppliedHaloCatalog requires " "keyword arguments ``particle_mass`` and ``redshift``\n" "storing scalars that will be interpreted as metadata about the halo catalog.\n") raise HalotoolsError(msg) Lbox = np.empty(3) Lbox[:] = metadata_dict['Lbox'] try: x, y, z = ( self.halo_table['halo_x'], self.halo_table['halo_y'], self.halo_table['halo_z'] ) assert np.all(x >= 0) assert np.all(x <= Lbox[0]) assert np.all(y >= 0) assert np.all(y <= Lbox[1]) assert np.all(z >= 0) assert np.all(z <= Lbox[2]) except AssertionError: msg = ("The ``halo_x``, ``halo_y`` and ``halo_z`` columns must only store arrays\n" "that are bound by 0 and the input ``Lbox``. \n") raise HalotoolsError(msg) try: redshift = float(metadata_dict['redshift']) except: msg = ("\nThe ``redshift`` metadata must be a float.\n") raise HalotoolsError(msg) for key, value in metadata_dict.items(): if (type(value) == np.ndarray): if custom_len(value) == len(self.halo_table['halo_id']): msg = ("\nThe input ``" + key + "`` argument stores a length-Nhalos ndarray.\n" "However, this key is being interpreted as metadata because \n" "it does not begin with ``halo_``. If this is your intention, ignore this message.\n" "Otherwise, rename this key to begin with ``halo_``. \n") warn(msg, UserWarning) def _passively_bind_ptcl_table(self, **kwargs): """ """ try: user_supplied_ptclcat = kwargs['user_supplied_ptclcat'] try: assert isinstance(user_supplied_ptclcat, UserSuppliedPtclCatalog) except AssertionError: msg = ("\n``user_supplied_ptclcat`` must be " "an instance of UserSuppliedPtclCatalog\n") raise HalotoolsError(msg) ptcl_table = user_supplied_ptclcat.ptcl_table try: assert (user_supplied_ptclcat.Lbox == self.Lbox).all() except AssertionError: msg = ("\nInconsistent values of Lbox between halo and particle catalogs:\n" "For the halo catalog, Lbox = " + str(self.Lbox) + "\n" "For the ``user_supplied_ptclcat``, Lbox = " + str(user_supplied_ptclcat.Lbox) + "\n\n") raise HalotoolsError(msg) try: assert user_supplied_ptclcat.particle_mass == self.particle_mass except AssertionError: msg = ("\nInconsistent values of particle_mass between halo and particle catalogs:\n" "For the halo catalog, particle_mass = " + str(self.particle_mass) + "\n" "For the ``user_supplied_ptclcat``, particle_mass = " + str(user_supplied_ptclcat.particle_mass) + "\n\n") raise HalotoolsError(msg) try: z1 = get_redshift_string(user_supplied_ptclcat.redshift) z2 = get_redshift_string(self.redshift) assert z1 == z2 except AssertionError: msg = ("\nInconsistent values of redshift between halo and particle catalogs:\n" "For the halo catalog, redshift = " + str(self.redshift) + "\n" "For the ``user_supplied_ptclcat``, redshift = " + str(user_supplied_ptclcat.redshift) + "\n\n") raise HalotoolsError(msg) self.ptcl_table = ptcl_table except KeyError: pass
[docs] def add_halocat_to_cache(self, fname, simname, halo_finder, version_name, processing_notes, overwrite=False, **additional_metadata): """ Parameters ------------ fname : string Absolute path of the file where you will store the halo catalog. Your filename must conclude with an `.hdf5` extension. The Halotools cache system will remember whatever location you choose, so try to choose a reasonably permanent resting place on disk. You can always relocate your catalog after caching it by following the :ref:`relocating_simulation_data` documentation page. simname : string Nickname of the simulation used as a shorthand way to keep track of the halo catalogs in your cache. halo_finder : string Nickname of the halo-finder used to generate the hlist file from particle data. version_name : string Nickname of the version of the halo catalog. The ``version_name`` is used as a bookkeeping tool in the cache log. processing_notes : string String used to provide supplementary notes that will be attached to the hdf5 file storing your halo catalog. overwrite : bool, optional If the chosen ``fname`` already exists, then you must set ``overwrite`` to True in order to write the file to disk. Default is False. **additional_metadata : sequence of strings, optional Each keyword of ``additional_metadata`` defines the name of a piece of metadata stored in the hdf5 file. The value bound to each key can be any string. When you load your cached halo catalog into memory, each piece of metadata will be stored as an attribute of the `~halotools.sim_manager.CachedHaloCatalog` instance. """ try: import h5py except ImportError: msg = ("\nYou must have h5py installed if you want to \n" "store your catalog in the Halotools cache. \n") raise HalotoolsError(msg) ############################################################ # Perform some consistency checks in the fname if (os.path.isfile(fname)) & (overwrite is False): msg = ("\nYou attempted to store your halo catalog " "in the following location: \n\n" + str(fname) + "\n\nThis path points to an existing file. \n" "Either choose a different fname or set ``overwrite`` to True.\n") raise HalotoolsError(msg) try: dirname = os.path.dirname(fname) assert os.path.exists(dirname) except: msg = ("\nThe directory you are trying to store the file does not exist. \n") raise HalotoolsError(msg) if fname[-5:] != '.hdf5': msg = ("\nThe fname must end with an ``.hdf5`` extension.\n") raise HalotoolsError(msg) ############################################################ # Perform consistency checks on the remaining log entry attributes try: _ = str(simname) _ = str(halo_finder) _ = str(version_name) _ = str(processing_notes) except: msg = ("\nThe input ``simname``, ``halo_finder``, ``version_name`` " "and ``processing_notes``\nmust all be strings.") raise HalotoolsError(msg) for key, value in additional_metadata.items(): try: _ = str(value) except: msg = ("\nIf you use ``additional_metadata`` keyword arguments \n" "to provide supplementary metadata about your catalog, \n" "all such metadata will be bound to the hdf5 file in the " "format of a string.\nHowever, the value you bound to the " "``"+key+"`` keyword is not representable as a string.\n") raise HalotoolsError(msg) ############################################################ # Now write the file to disk and add the appropriate metadata self.halo_table.write(fname, path='data', overwrite=overwrite) f = h5py.File(fname, 'a') redshift_string = get_redshift_string(self.redshift) f.attrs.create('simname', np.string_(simname)) f.attrs.create('halo_finder', np.string_(halo_finder)) f.attrs.create('version_name', np.string_(version_name)) f.attrs.create('redshift', np.string_(redshift_string)) f.attrs.create('fname', np.string_(fname)) f.attrs.create('Lbox', self.Lbox) f.attrs.create('particle_mass', self.particle_mass) time_right_now = datetime.datetime.now().strftime( '%Y-%m-%d %H:%M:%S') f.attrs.create('time_catalog_was_originally_cached', np.string_(time_right_now)) f.attrs.create('processing_notes', np.string_(processing_notes)) for key, value in additional_metadata.items(): f.attrs.create(key, np.string_(value)) f.close() ############################################################ # Now that the file is on disk, add it to the cache cache = HaloTableCache() log_entry = HaloTableCacheLogEntry( simname=simname, halo_finder=halo_finder, version_name=version_name, redshift=self.redshift, fname=fname) cache.add_entry_to_cache_log(log_entry, update_ascii=True) self.log_entry = log_entry