diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 00000000..88691417 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,33 @@ +# This is a basic workflow to help you get started with Actions + +name: CI + +# Controls when the workflow will run +on: + # Triggers the workflow on push or pull request events but only for the master branch + push: + branches: [ master ] + pull_request: + branches: [ master ] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # Formatting check + formatting: + # The type of runner that the job will run on + runs-on: ubuntu-latest + + # Don't run for draft PRs + if: github.event.pull_request.draft == false + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v2 + + # Runs the formatting script + - name: Formatting + run: bash format.sh --check diff --git a/.gitignore b/.gitignore index 6c216129..3547d588 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,8 @@ __pycache__ *~ logs openmpi-5.0.3-hdf5-1.12.3-env +black_formatting_env +*egg-info documentation/SOAP.aux documentation/SOAP.log @@ -16,3 +18,5 @@ documentation/units.tex tests/FLAMINGO/test_parameters.yml tests/COLIBRE/test_parameters.yml +tests/test_SO_radius_*.png +test_data/* diff --git a/COPYING b/COPYING new file mode 100644 index 00000000..10926e87 --- /dev/null +++ b/COPYING @@ -0,0 +1,675 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. + diff --git a/COPYING.LESSER b/COPYING.LESSER new file mode 100644 index 00000000..341c30bd --- /dev/null +++ b/COPYING.LESSER @@ -0,0 +1,166 @@ + GNU LESSER GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + + This version of the GNU Lesser General Public License incorporates +the terms and conditions of version 3 of the GNU General Public +License, supplemented by the additional permissions listed below. + + 0. Additional Definitions. + + As used herein, "this License" refers to version 3 of the GNU Lesser +General Public License, and the "GNU GPL" refers to version 3 of the GNU +General Public License. + + "The Library" refers to a covered work governed by this License, +other than an Application or a Combined Work as defined below. + + An "Application" is any work that makes use of an interface provided +by the Library, but which is not otherwise based on the Library. +Defining a subclass of a class defined by the Library is deemed a mode +of using an interface provided by the Library. + + A "Combined Work" is a work produced by combining or linking an +Application with the Library. The particular version of the Library +with which the Combined Work was made is also called the "Linked +Version". + + The "Minimal Corresponding Source" for a Combined Work means the +Corresponding Source for the Combined Work, excluding any source code +for portions of the Combined Work that, considered in isolation, are +based on the Application, and not on the Linked Version. + + The "Corresponding Application Code" for a Combined Work means the +object code and/or source code for the Application, including any data +and utility programs needed for reproducing the Combined Work from the +Application, but excluding the System Libraries of the Combined Work. + + 1. Exception to Section 3 of the GNU GPL. + + You may convey a covered work under sections 3 and 4 of this License +without being bound by section 3 of the GNU GPL. + + 2. Conveying Modified Versions. + + If you modify a copy of the Library, and, in your modifications, a +facility refers to a function or data to be supplied by an Application +that uses the facility (other than as an argument passed when the +facility is invoked), then you may convey a copy of the modified +version: + + a) under this License, provided that you make a good faith effort to + ensure that, in the event an Application does not supply the + function or data, the facility still operates, and performs + whatever part of its purpose remains meaningful, or + + b) under the GNU GPL, with none of the additional permissions of + this License applicable to that copy. + + 3. Object Code Incorporating Material from Library Header Files. + + The object code form of an Application may incorporate material from +a header file that is part of the Library. You may convey such object +code under terms of your choice, provided that, if the incorporated +material is not limited to numerical parameters, data structure +layouts and accessors, or small macros, inline functions and templates +(ten or fewer lines in length), you do both of the following: + + a) Give prominent notice with each copy of the object code that the + Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the object code with a copy of the GNU GPL and this license + document. + + 4. Combined Works. + + You may convey a Combined Work under terms of your choice that, +taken together, effectively do not restrict modification of the +portions of the Library contained in the Combined Work and reverse +engineering for debugging such modifications, if you also do each of +the following: + + a) Give prominent notice with each copy of the Combined Work that + the Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the Combined Work with a copy of the GNU GPL and this license + document. + + c) For a Combined Work that displays copyright notices during + execution, include the copyright notice for the Library among + these notices, as well as a reference directing the user to the + copies of the GNU GPL and this license document. + + d) Do one of the following: + + 0) Convey the Minimal Corresponding Source under the terms of this + License, and the Corresponding Application Code in a form + suitable for, and under terms that permit, the user to + recombine or relink the Application with a modified version of + the Linked Version to produce a modified Combined Work, in the + manner specified by section 6 of the GNU GPL for conveying + Corresponding Source. + + 1) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (a) uses at run time + a copy of the Library already present on the user's computer + system, and (b) will operate properly with a modified version + of the Library that is interface-compatible with the Linked + Version. + + e) Provide Installation Information, but only if you would otherwise + be required to provide such information under section 6 of the + GNU GPL, and only to the extent that such information is + necessary to install and execute a modified version of the + Combined Work produced by recombining or relinking the + Application with a modified version of the Linked Version. (If + you use option 4d0, the Installation Information must accompany + the Minimal Corresponding Source and Corresponding Application + Code. If you use option 4d1, you must provide the Installation + Information in the manner specified by section 6 of the GNU GPL + for conveying Corresponding Source.) + + 5. Combined Libraries. + + You may place library facilities that are a work based on the +Library side by side in a single library together with other library +facilities that are not Applications and are not covered by this +License, and convey such a combined library under terms of your +choice, if you do both of the following: + + a) Accompany the combined library with a copy of the same work based + on the Library, uncombined with any other library facilities, + conveyed under the terms of this License. + + b) Give prominent notice with the combined library that part of it + is a work based on the Library, and explaining where to find the + accompanying uncombined form of the same work. + + 6. Revised Versions of the GNU Lesser General Public License. + + The Free Software Foundation may publish revised and/or new versions +of the GNU Lesser General Public License from time to time. Such new +versions will be similar in spirit to the present version, but may +differ in detail to address new problems or concerns. + + Each version is given a distinguishing version number. If the +Library as you received it specifies that a certain numbered version +of the GNU Lesser General Public License "or any later version" +applies to it, you have the option of following the terms and +conditions either of that published version or of any later version +published by the Free Software Foundation. If the Library as you +received it does not specify a version number of the GNU Lesser +General Public License, you may choose any version of the GNU Lesser +General Public License ever published by the Free Software Foundation. + + If the Library as you received it specifies that a proxy can decide +whether future versions of the GNU Lesser General Public License shall +apply, that proxy's public statement of acceptance of any version is +permanent authorization for you to choose that version for the +Library. + diff --git a/README.md b/README.md index bba1582a..048c839c 100644 --- a/README.md +++ b/README.md @@ -1,34 +1,69 @@ # SOAP: Spherical Overdensity and Aperture Processor +[![DOI](https://joss.theoj.org/papers/10.21105/joss.08252/status.svg)](https://doi.org/10.21105/joss.08252) + This repository contains programs which can be used to compute -properties of halos in spherical apertures in SWIFT snapshots and to -match halos between simulations using the particle IDs. +properties of halos in spherical apertures in [SWIFT](https://swift.strw.leidenuniv.nl/) snapshots. +The resulting output halo catalogues can be read using the +[swiftsimio](https://swiftsimio.readthedocs.io/en/latest/) +python package. + +## Installation The code is written in python and uses mpi4py for parallelism. +IO is carried out in parallel, and so [parallel h5py](https://docs.h5py.org/en/stable/mpi.html) is required. SOAP and it's dependencies can also be +installed directly using the command +`pip install git+https://github.com/SWIFTSIM/SOAP.git@soap_runtime` +but this may install a serial version of h5py. Therefore the following +steps are recommended for install +``` +pip install mpi4py +export HDF5_MPI="ON"; export CC=mpicc; pip install --no-binary=h5py h5py +pip install git+https://github.com/SWIFTSIM/SOAP.git@soap_runtime +``` -## Running on cosma +### Installation on COSMA -The files in the `scripts` directory are made for running on cosma. -All scripts should be run from the base SOAP directory. Before running -SOAP you should first create a python environment with +If you are using the [COSMA system](https://cosma.readthedocs.io/en/latest/), +you can install an SOAP virtual environment by running `./scripts/cosma_python_env.sh` -## Computing halo membership for particles in the snapshot +## Running SOAP + +The command `./tests/run_small_volume.sh` will download a small example +simulation, run the group membership and halo properties scripts on it. +This uses the parameter file at `./tests/run_small_volume.yml`, and the +resulting catalogue is placed in the `output` directory. It also generates the +pdf documentation to describe the output file (which is written to +`documentation/SOAP.pdf`). -The first program, `group_membership.py`, will compute bound -halo indexes for all particles in a snapshot. The output -consists of the same number of files as the snapshot with particle halo +### Computing halo membership for particles in the snapshot + +The first step is to extract the subhalo index for all particles in a +snapshot using an input halo-finder catalogue. The output of this step +consists of the same number of files as the snapshot, with particle halo indexes written out in the same order as the snapshot. -## Computing halo properties +To run the group membership program you must pass the name of the simulation, +the snapshot number, and a parameter file. For example: + +``` +snapnum=0077 +sim=L1000N0900/DMO_FIDUCIAL +mpirun python python SOAP/group_membership.py \ + --sim-name=${sim} --snap-nr=${snapnum} parameter_files/FLAMINGO.yml +``` + +### Computing halo properties -The second program, `compute_halo_properties.py`, reads the simulation -snapshot and the output from `group_membership.py` and uses it to +The second program, `SOAP/compute_halo_properties.py`, reads the simulation +snapshot and the output from `SOAP/group_membership.py` and uses it to calculate halo properties. It works as follows: The simulation volume is split into chunks. Each compute node reads in the particles in one chunk at a time and calculates the properties -of all halos in that chunk. +of all halos in that chunk. Therefore when running SOAP the number of +chunks should always be greater than or equal to the number of nodes. Within a compute node there is one MPI process per core. The particle data and halo catalogue for the chunk are stored in shared memory. @@ -37,129 +72,111 @@ around the halo, and calculates the required properties. When all halos in the chunk have been done the compute node will move on to the next chunk. -## Parameter files - -To run either of the programs a parameters file must be passed. This -contains information including the input and output directories, -the halo finder to use, which halo definitions to use, and -which properties to calculate for each halo definition. Example -parameter files can be found in the `parameters_files` directory. - -## Compression - -Two types of compression are useful for reducting the size of SOAP output. -The first is lossless compression via GZIP, the second is lossy compression. -For the group membership files we only apply lossless compression. However, -each property in the final SOAP catalogue has a lossy compression filter -associated with it, which are set in `property_table.py`. The script -`compression/compress_fast_metadata.py` will apply both lossy and -lossless compression to SOAP catalogues. - -## Usage on COSMA - -### Required modules - -The same MPI module which was used to compile mpi4py must be loaded: -``` -module load python/3.10.1 gnu_comp/11.1.0 openmpi/4.1.1 -``` +All particle data is stored in unyt arrays internally. On opening the snapshot +a unyt UnitSystem is defined which corresponds to the simulation units. When +particles are read in unyt arrays are created with units based on the +attributes in the snapshot. These units are propagated through the halo +property calculations and used to write the unit attributes in the output. +Comoving quantities are handled by defining a dimensionless unit corresponding +to the expansion factor a. -### Calculating particle group membership +To calculate halo properties you must pass the same information as for +group membership, and also specify the number of chunks. +If the run is dark matter only the flag `--dmo` should +be passed. For example: -To run the group membership program needs the name of the simulation, -the snapshot number, and a parameter file. For example: ``` snapnum=0077 sim=L1000N0900/DMO_FIDUCIAL -mpirun python3 -u -m mpi4py ./group_membership.py \ - --sim-name=${sim} --snap-nr${snapnum} parameter_files/FLAMINGO.yml +mpirun python -u SOAP/compute_halo_properties.py \ + --sim-name=${sim} --snap-nr=${snapnum} --chunks=1 --dmo \ + parameter_files/FLAMINGO.yml ``` -See `scripts/FLAMINGO/L1000N1800/group_membership_L1000N1800.sh` for an example -batch script. +Here, `--chunks` determines how many chunks the simulation box is +split into. Ideally it should be set such that one chunk fills a compute node. -The code can optionally also write group membership to a single file -virtual snapshot specified with the `--update-virtual-file` flag. This -can be used to create a single file snapshot with group membership -included that can be read with swiftsimio or gadgetviewer. +The optional `--max-ranks-reading` flag determines how many MPI ranks per node +read the snapshot. This can be used to avoid overloading the file system. The +default value is 32. -The `--output-prefix` flag can be used to specify a prefix used to name the -datasets written to the virtual file. This may be useful if writing group -membership from several different VR runs to a single file. +### Parameter files -### Calculating halo properties +To run either of the programs a parameters file must be passed. This +contains information including the input and output directories, +the halo finder to use, which halo definitions to use, and +which properties to calculate for each halo definition. A description +of all possible fields, and a number of example parameter files +can be found in the `parameters_files` directory. -To calculate halo properties you must pass the same information as for -group membership. If the run is dark matter only the flag `--dmo` should -be passed. For example: +### Compression -``` -snapnum=0077 -sim=L1000N0900/DMO_FIDUCIAL -mpirun python3 -u -m mpi4py ./compute_halo_properties.py \ - --sim-name=${sim} --snap-nr=${snapnum} --chunks=1 ${dmo_flag} \ - parameter_files/FLAMINGO.yml -``` +Two types of compression are useful for reducting the size of SOAP output. +The first is lossless compression via GZIP, the second is lossy compression. +For the group membership files we only apply lossless compression. However, +each property in the final SOAP catalogue has a lossy compression filter +associated with it, which are set in `SOAP/property_table.py`. The script +`compression/compress_soap_catalogue.py` will apply both lossy and +lossless compression to SOAP catalogues. -Here, `--chunks` determines how many chunks the simulation box is -split into. Ideally it should be set such that one chunk fills a compute node. +### Documentation -The `--max-ranks-reading` flag determines how many MPI ranks per node read the -snapshot. This can be used to avoid overloading the file system. The default -value is 32. +A pdf describing the SOAP output can be generated. First run `SOAP/property_table.py` passing the parameter file used to run SOAP (to get the properties and halo types to include) and a snapshot (to get the units), e.g. `python SOAP/property_table.py parameter_files/COLIBRE_THERMAL.yml /cosma8/data/dp004/colibre/Runs/L0100N0752/Thermal/snapshots/colibre_0127/colibre_0127.hdf5`. This will generate a table containing all the properties which are enabled in the parameter file. To create the pdf run `cd documentation; pdflatex SOAP.tex; pdflatex SOAP.tex`. If you wish to see all possible properties then first run `python SOAP/property_table.py`, and then generate the pdf. -### Batch scripts for running on FLAMINGO simulations on Cosma-8 +### Slurm scripts for running on COSMA -There are slurm scripts to run on FLAMINGO in `./scripts/FLAMINGO/`. These -are intended to be run as array jobs where the job array indexes determine -which snapshots to process. +The files in the `scripts` directory are made for running on cosma. +All scripts should be run from the base SOAP directory. -In order to reduce duplication only one script is provided per simulation +The scripts are intended to be run as array jobs where the job array indexes determine +which snapshots to process. In order to reduce duplication only one script is provided per simulation box size and resolution. The simulation to process is specified by setting the job name with the slurm sbatch -J flag. -## Adding quantities +## Modifying the code -The property calculations are defined in these files: +You can install an editable version of SOAP by cloning this repository and running: +``` +pip install mpi4py +export HDF5_MPI="ON"; export CC=mpicc; pip install --no-binary=h5py h5py +pip install -e . +``` + +The property calculations are defined in the following files in the `SOAP/particle_selection` directory: - * Properties of particles in halos `subhalo_properties.py` + * Properties of particles in subhalos `subhalo_properties.py` * Properties of particles in spherical apertures `aperture_properties.py` * Properties of particles in projected apertures `projected_aperture_properties.py` * Properties of particles in spheres of a specified overdensity `SO_properties.py` -Adding new quantities to already defined SOAP apertures is a relatively easy business. There are five steps. +Adding new quantities to already defined SOAP apertures is relatively easy. There are five steps. - * Start by adding an entry to the property table (https://github.com/SWIFTSIM/SOAP/blob/master/property_table.py). Here we store all the properties of the quantities (name, type, unit etc.) All entries in this table are checked with unit tests and added to the documentation. Adding your quantity here will make sure the code and the documentation are in line with each other. - * Next you have to add the quantity to the type of aperture you want it to be calculated for (aperture_properties.py, SO_properties.py, subhalo_properties.py or projected_aperture_properties.py). In all these files there is a class named `property_list` which defines the subset of all properties that are calculated for this specific aperture. + * Start by adding an entry to `SOAP/property_table.py`. Here we store all the properties of the quantities (name, type, unit etc.) All entries in this table are checked with unit tests and added to the documentation. Adding your quantity here will make sure the code and the documentation are in line with each other. + * Next you have to add the quantity to the type of aperture you want it to be calculated for (`aperture_properties.py`, `SO_properties.py`, `subhalo_properties.py`, or `projected_aperture_properties.py`). In all these files there is a class named `property_list` which defines the subset of all properties that are calculated for this specific aperture. * To calculate your quantity you have to define a `@lazy_property` with the same name in the `XXParticleData` class in the same file. There should be a lot of examples of different quantities that are already calculated. An important thing to note is that fields that are used for multiple calculations should have their own `@lazy_property` to avoid loading things multiple times, so check if the things that you need are already there. - * Add the property to the parameter file, though if a property is missing from the parameter file then SOAP will calculate it by default. - * At this point everything should now work. To test the newly added quantities you can run a unit test using `python3 -W error -m pytest NAME_OF_FILE`. This checks whether the code crashes, and whether there are problems with units and overflows. This should make sure that SOAP never crashes while calculating the new properties. + * Add the property to the parameter file. + * At this point everything should now work. To test the newly added quantities you can run a unit test using `pytest -W error -m pytest tests/test_{NAME_OF_FILE}`. This checks whether the code crashes, and whether there are problems with units and overflows. This should make sure that SOAP never crashes while calculating the new properties. If SOAP does crash while evaluating your new property it will try to output the ID of the halo it was processing when it crashed. Then you can re-run that halo on a single MPI rank in the python debugger as described in the debugging section below. -## Units - -All particle data are stored in unyt arrays internally. On opening the snapshot -a unyt UnitSystem is defined which corresponds to the simulation units. When -particles are read in unyt arrays are created with units based on the -attributes in the snapshot. These units are propagated through the halo -property calculations and used to write the unit attributes in the output. - -Comoving quantities are handled by defining a dimensionless unit corresponding -to the expansion factor a. +### Tests -## Tests +The directory `tests` contains a number of unit tests for SOAP, some of which +require MPI. They require the optional dependency `pytest-mpi`, and can be run with -The command `./tests/run_tests.sh` will run the unit tests for SOAP. Some tests -rely on data stored on cosma, and therefore cannot be run from other systems. +``` +cd tests +pytest -m "not mpi" -W error; mpirun -np 4 pytest -m mpi --with-mpi -W error +``` -The scripts in `tests/FLAMINGO` for showing how to -run SOAP on a few halos from the FLAMINGO simulations. +The scripts in `tests/FLAMINGO` and `tests/COLIBRE` run SOAP on a few +halos from the FLAMINGO/COLIBRE simulations. These tests rely on data +stored on COSMA, and therefore cannot be run from other systems. -## Debugging +### Debugging For debugging it might be helpful to run on one MPI rank in the python debugger and reduce the run time by limiting the number of halo to process with the @@ -179,7 +196,17 @@ flag. This specifies the index of the required halos in the halo catalogue. E.g. python3 -Werror -m pdb ./compute_halo_properties.py --halo-indices 1 2 3 ... ``` -## Profiling +### Timing + +The flag `--record-halo-timings` can be passed to record the total amount of time +spent calculating properties for subhalos of different masses, and can be useful for +identifying what objects/apertures are dominating the SOAP runtime. The flag +`--record-property-timings` can be passed to record the amount of time spent +calculating each property for each subhalo. Note that this will double the size +of the final output catalogue. The timings can be analysed with the script +`misc/plot_times.py`. + +### Profiling The code can be profiled by running with the `--profile` flag, which uses the python cProfile module. Use `--profile=1` to profile MPI rank zero only or @@ -194,62 +221,3 @@ pip install snakeviz --user snakeviz -b "firefox -no-remote %s" ./profile.0.dat ``` -## Matching halos between outputs - -This repository also contains a script to find halos which contain the same -particle IDs between two outputs. It can be used to find the same halos between -different snapshots or between hydro and dark matter only simulations. - -For each halo in the first output we find the N most bound particle IDs and -determine which halo in the second output contains the largest number of these -IDs. This matching process is then repeated in the opposite direction and we -check for cases were we have consistent matches in both directions. - -### Matching to field halos only - -The `--to-field-halos-only` flag can be used to match central halos -between outputs. If it is set we follow the -first `nr_particles` most bound particles from each halo as usual, but when -locating them in the other output any particles in satellite subhalos -are treated as belonging to the host halo. - -In this mode field halos in one catalogue will only ever be matched to field -halos in the other catalogue. - -Output is still generated for non-field halos. These halos will be matched to -the field halo which contains the largest number of their `nr_particles` most -bound particles. These matches will never be consistent in both directions -because matches to non-field halos are not possible. - -### Output - -The output is a HDF5 file with the following datasets: - - * `BoundParticleNr1` - number of bound particles in each halo in the first catalogue - * `MatchIndex1to2` - for each halo in the first catalogue, index of the matching halo in the second - * `MatchCount1to2` - how many of the most bound particles from the halo in the first catalogue are in the matched halo in the second - * `Consistent1to2` - whether the match from first to second catalogue is consistent with second to first (1) or not (0) - -There are corresponding datasets with `1` and `2` reversed with information about matching in the opposite direction. - -## Documentation - -### PDF document - -A pdf describing the SOAP output can be generated. First run `property_table.py` passing the parameter file used to run SOAP, e.g. `python property_table.py parameter_files/FLAMINGO.yml`. This will generate a table containing all the properties which are enabled in the parameter file. To create the pdf run `pdflatex documentation/SOAP.tex`. - -### API reference - -Most of the files containing halo property calculations have been extensively documented -using docstrings. To generate documentation, you can for example use -``` -python3 -m pydoc aperture_properties -``` -This will display the documentation for the file `aperture_properties.py`. -``` -python3 -m pydoc -b -``` -will display an interactive web page in your browser with a lot of documentation, including all documented -files and functionality of SOAP. - -Please have a look at this documentation before making any significant changes! diff --git a/SOAP/__init__.py b/SOAP/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/SOAP/catalogue_readers/__init__.py b/SOAP/catalogue_readers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/read_hbtplus.py b/SOAP/catalogue_readers/read_hbtplus.py similarity index 51% rename from read_hbtplus.py rename to SOAP/catalogue_readers/read_hbtplus.py index fb7fc005..d91c5849 100644 --- a/read_hbtplus.py +++ b/SOAP/catalogue_readers/read_hbtplus.py @@ -14,9 +14,13 @@ def hbt_filename(hbt_basename, file_nr): return f"{hbt_basename}.{file_nr}.hdf5" -def read_hbtplus_groupnr(basename): +def read_hbtplus_groupnr(basename, read_potential_energies=False, registry=None): """ Read HBTplus output and return group number for each particle ID + + Potential energies will not be returned by default. To return the potential + energies a unit registry must be passed. + """ from mpi4py import MPI @@ -25,72 +29,129 @@ def read_hbtplus_groupnr(basename): comm_rank = comm.Get_rank() comm_size = comm.Get_size() - # Find number of HBT output files + # Find number of HBT output files, and if we're dealing with sorted output if comm_rank == 0: - with h5py.File(hbt_filename(basename, 0), "r") as infile: - nr_files = int(infile["NumberOfFiles"][...]) + if os.path.exists(hbt_filename(basename, 0)): + with h5py.File(hbt_filename(basename, 0), "r") as infile: + nr_files = int(infile["NumberOfFiles"][...]) + sorted_file = False + elif os.path.exists(basename): + with h5py.File(basename, "r") as infile: + assert "Particles" in infile + nr_files = 1 + sorted_file = True + else: + print(f"No HBT files found for basename {basename}") + comm.Abort() else: nr_files = None + sorted_file = None nr_files = comm.bcast(nr_files) + sorted_file = comm.bcast(sorted_file) + + # There are two different read routines. One is for the original HBTplus + # output. The second is for the "nice" output, where the subhalos are + # sorted by TrackId. + if not sorted_file: + # Assign files to MPI ranks + files_per_rank = np.zeros(comm_size, dtype=int) + files_per_rank[:] = nr_files // comm_size + remainder = nr_files % comm_size + if remainder == 1: + files_per_rank[0] += 1 + elif remainder > 1: + for i in range(remainder): + files_per_rank[int((comm_size - 1) * i / (remainder - 1))] += 1 + assert np.sum(files_per_rank) == nr_files, f"{nr_files=}, {comm_size=}" + first_file_on_rank = np.cumsum(files_per_rank) - files_per_rank + + halos = [] + ids_bound = [] + if read_potential_energies: + potential_energies = [] + + for file_nr in range( + first_file_on_rank[comm_rank], + first_file_on_rank[comm_rank] + files_per_rank[comm_rank], + ): + with h5py.File(hbt_filename(basename, file_nr), "r") as infile: + halos.append(infile["Subhalos"][...]) + ids_bound.append(infile["SubhaloParticles"][...]) + if read_potential_energies: + potential_energies.append(infile["PotentialEnergies"][...]) + + # Concatenate arrays of halos from different files + if len(halos) > 0: + halos = np.concatenate(halos) + else: + # This rank was assigned no files + halos = None + halos = virgo.mpi.util.replace_none_with_zero_size(halos, comm=comm) - # Assign files to MPI ranks - files_per_rank = np.zeros(comm_size, dtype=int) - files_per_rank[:] = nr_files // comm_size - files_per_rank[: nr_files % comm_size] += 1 - assert np.sum(files_per_rank) == nr_files - first_file_on_rank = np.cumsum(files_per_rank) - files_per_rank - - # Read in the halos from the HBT output: - # 'halos' will be an array of structs with the halo catalogue - # 'ids_bound' will be an array of particle IDs in halos, sorted by halo - halos = [] - ids_bound = [] - for file_nr in range( - first_file_on_rank[comm_rank], - first_file_on_rank[comm_rank] + files_per_rank[comm_rank], - ): - with h5py.File(hbt_filename(basename, file_nr), "r") as infile: - halos.append(infile["Subhalos"][...]) - ids_bound.append(infile["SubhaloParticles"][...]) - - # Get the dtype for particle IDs - if len(ids_bound) > 0: - id_dtype = h5py.check_vlen_dtype(ids_bound[0].dtype) - else: - id_dtype = None - - # Concatenate arrays of halos from different files - if len(halos) > 0: - halos = np.concatenate(halos) - else: - # This rank was assigned no files - halos = None - halos = virgo.mpi.util.replace_none_with_zero_size(halos, comm=comm) - - # Combine arrays of particles in halos - if len(ids_bound) > 0: - ids_bound = np.concatenate( - ids_bound - ) # Combine arrays of halos from different files if len(ids_bound) > 0: - ids_bound = np.concatenate( - ids_bound - ) # Combine arrays of particles from different halos + # Get the dtype for particle IDs + id_dtype = h5py.check_vlen_dtype(ids_bound[0].dtype) + # Combine arrays of halos from different files + ids_bound = np.concatenate(ids_bound) + if len(ids_bound) > 0: + # Combine arrays of particles from different halos + ids_bound = np.concatenate(ids_bound) + else: + # The files assigned to this rank contain zero halos + ids_bound = np.zeros(0, dtype=id_dtype) else: - # The files assigned to this rank contain zero halos - ids_bound = np.zeros(0, dtype=id_dtype) + # This rank was assigned no files + ids_bound = None + ids_bound = virgo.mpi.util.replace_none_with_zero_size(ids_bound, comm=comm) + + # Number of particles in each subhalo + halo_size = halos["Nbound"] + del halos + + # Apply same combination process to potential energies + if read_potential_energies: + if len(potential_energies) > 0: + potential_dtype = h5py.check_vlen_dtype(potential_energies[0].dtype) + potential_energies = np.concatenate(potential_energies) + if len(potential_energies) > 0: + potential_energies = np.concatenate(potential_energies) + else: + potential_energies = np.zeros(0, dtype=potential_dtype) + else: + # This rank was assigned no files + potential_energies = None + potential_energies = virgo.mpi.util.replace_none_with_zero_size( + potential_energies, comm=comm + ) else: - # This rank was assigned no files - ids_bound = None - ids_bound = virgo.mpi.util.replace_none_with_zero_size(ids_bound, comm=comm) + # Read the fields we require from the sorted catalogues + hbt_file = phdf5.MultiFile([basename], comm=comm) + ids_bound = hbt_file.read("Particles/ParticleIDs") + halo_size = hbt_file.read("Subhalos/Nbound") + if read_potential_energies: + potential_energies = hbt_file.read("Particles/PotentialEnergies") + + if read_potential_energies: + # Get HBTplus unit information + if comm_rank == 0: + filename = basename if sorted_file else hbt_filename(basename, 0) + with h5py.File(filename, "r") as infile: + if "Units" in infile: + VelInKmS = float(infile["Units/VelInKmS"][...]) + else: + VelInKmS = None + VelInKmS = comm.bcast(VelInKmS) + + # Add units to potential energies + potential_energies = (potential_energies * (VelInKmS**2)) * unyt.Unit( + "km/s", registry=registry + ) ** 2 # Assign halo indexes to the particles - nr_local_halos = len(halos) + nr_local_halos = len(halo_size) total_nr_halos = comm.allreduce(nr_local_halos) - halo_offset = comm.scan(len(halos), op=MPI.SUM) - len(halos) + halo_offset = comm.scan(len(halo_size), op=MPI.SUM) - len(halo_size) halo_index = np.arange(nr_local_halos, dtype=int) + halo_offset - halo_size = halos["Nbound"] - del halos grnr_bound = np.repeat(halo_index, halo_size) # Assign ranking by binding energy to the particles @@ -114,10 +175,15 @@ def read_hbtplus_groupnr(basename): ) assert len(unique_counts) == 0 or np.max(unique_counts) == 1 + if read_potential_energies: + return total_nr_halos, ids_bound, grnr_bound, rank_bound, potential_energies + return total_nr_halos, ids_bound, grnr_bound, rank_bound -def read_hbtplus_catalogue(comm, basename, a_unit, registry, boxsize, keep_orphans=False): +def read_hbtplus_catalogue( + comm, basename, a_unit, registry, boxsize, keep_orphans=False +): """ Read in the HBTplus halo catalogue, distributed over communicator comm. @@ -135,7 +201,7 @@ def read_hbtplus_catalogue(comm, basename, a_unit, registry, boxsize, keep_orpha search_radius - initial search radius which includes all member particles is_central - integer 1 for centrals, 0 for satellites nr_bound_part - number of bound particles in each halo - + Any other arrays will be passed through to the output ONLY IF they are documented in property_table.py. @@ -163,9 +229,16 @@ def read_hbtplus_catalogue(comm, basename, a_unit, registry, boxsize, keep_orpha # Get HBTplus unit information if comm_rank == 0: + # Check if this is a sorted HBT file + if os.path.exists(hbt_filename(basename, 0)): + filename = hbt_filename(basename, 0) + sorted_file = False + else: + filename = basename + sorted_file = True + # Try to get units from the HDF5 output have_units = False - filename = hbt_filename(basename, 0) with h5py.File(filename, "r") as infile: if "Units" in infile: LengthInMpch = float(infile["Units/LengthInMpch"][...]) @@ -190,14 +263,30 @@ def read_hbtplus_catalogue(comm, basename, a_unit, registry, boxsize, keep_orpha LengthInMpch = None MassInMsunh = None VelInKmS = None + sorted_file = None (LengthInMpch, MassInMsunh, VelInKmS) = comm.bcast( (LengthInMpch, MassInMsunh, VelInKmS) ) + sorted_file = comm.bcast(sorted_file) # Read the subhalos for this snapshot - filename = f"{basename}.%(file_nr)d.hdf5" - mf = phdf5.MultiFile(filename, file_nr_dataset="NumberOfFiles", comm=comm) - subhalo = mf.read("Subhalos") + if not sorted_file: + filename = f"{basename}.%(file_nr)d.hdf5" + mf = phdf5.MultiFile(filename, file_nr_dataset="NumberOfFiles", comm=comm) + subhalo = mf.read("Subhalos") + subhalo_props = list(subhalo.dtype.names) + else: + # Just recreate an object the same as the unsorted file + if comm_rank == 0: + with h5py.File(filename, "r") as infile: + subhalo_props = list(infile["Subhalos"].keys()) + else: + subhalo_props = None + subhalo_props = comm.bcast(subhalo_props) + mf = phdf5.MultiFile([basename], comm=comm) + subhalo = {} + for prop in subhalo_props: + subhalo[prop] = mf.read(f"Subhalos/{prop}") # Load the number of bound particles nr_bound_part = unyt.unyt_array( @@ -250,12 +339,6 @@ def read_hbtplus_catalogue(comm, basename, a_unit, registry, boxsize, keep_orpha depth = unyt.unyt_array( subhalo["Depth"][keep], units=unyt.dimensionless, dtype=int, registry=registry ) - snapshot_birth = unyt.unyt_array( - subhalo["SnapshotIndexOfBirth"][keep], - units=unyt.dimensionless, - dtype=int, - registry=registry, - ) parent_id = unyt.unyt_array( subhalo["NestedParentTrackId"][keep], units=unyt.dimensionless, @@ -271,17 +354,9 @@ def read_hbtplus_catalogue(comm, basename, a_unit, registry, boxsize, keep_orpha # Peak mass max_mass = (subhalo["LastMaxMass"][keep] * MassInMsunh / h) * swift_msun - snapshot_max_mass = subhalo["SnapshotIndexOfLastMaxMass"][keep] - snapshot_max_mass = unyt.unyt_array( - snapshot_max_mass, units=unyt.dimensionless, dtype=int, registry=registry - ) # Peak vmax max_vmax = (subhalo["LastMaxVmaxPhysical"][keep] * VelInKmS) * kms - snapshot_max_vmax = subhalo["SnapshotIndexOfLastMaxVmax"][keep] - snapshot_max_vmax = unyt.unyt_array( - snapshot_max_vmax, units=unyt.dimensionless, dtype=int, registry=registry - ) # Number of bound particles nr_bound_part = nr_bound_part[keep] @@ -295,12 +370,34 @@ def read_hbtplus_catalogue(comm, basename, a_unit, registry, boxsize, keep_orpha "HostHaloId": host_halo_id, "Depth": depth, "TrackId": track_id, - "SnapshotIndexOfBirth": snapshot_birth, "NestedParentTrackId": parent_id, "DescendantTrackId": descendant_id, "LastMaxMass": max_mass, - "SnapshotIndexOfLastMaxMass": snapshot_max_mass, "LastMaxVmaxPhysical": max_vmax, - "SnapshotIndexOfLastMaxVmax": snapshot_max_vmax, } + + if "SnapshotIndexOfBirth" in subhalo_props: + snapshot_birth = subhalo["SnapshotIndexOfBirth"][keep] + snapshot_max_mass = subhalo["SnapshotIndexOfLastMaxMass"][keep] + snapshot_max_vmax = subhalo["SnapshotIndexOfLastMaxVmax"][keep] + snapshot_isolation = subhalo["SnapshotIndexOfLastIsolation"][keep] + else: + snapshot_birth = subhalo["SnapshotOfBirth"][keep] + snapshot_max_mass = subhalo["SnapshotOfLastMaxMass"][keep] + snapshot_max_vmax = subhalo["SnapshotOfLastMaxVmax"][keep] + snapshot_isolation = subhalo["SnapshotOfLastIsolation"][keep] + + local_halo["SnapshotOfBirth"] = unyt.unyt_array( + snapshot_birth, units=unyt.dimensionless, dtype=int, registry=registry + ) + local_halo["SnapshotOfLastMaxMass"] = unyt.unyt_array( + snapshot_max_mass, units=unyt.dimensionless, dtype=int, registry=registry + ) + local_halo["SnapshotOfLastMaxVmax"] = unyt.unyt_array( + snapshot_max_vmax, units=unyt.dimensionless, dtype=int, registry=registry + ) + local_halo["SnapshotOfLastIsolation"] = unyt.unyt_array( + snapshot_isolation, units=unyt.dimensionless, dtype=int, registry=registry + ) + return local_halo diff --git a/read_rockstar.py b/SOAP/catalogue_readers/read_rockstar.py similarity index 73% rename from read_rockstar.py rename to SOAP/catalogue_readers/read_rockstar.py index 82e7c09e..08f3b4e5 100644 --- a/read_rockstar.py +++ b/SOAP/catalogue_readers/read_rockstar.py @@ -3,7 +3,6 @@ import os import numpy as np -import pandas as pd import unyt import virgo.mpi.parallel_hdf5 as phdf5 @@ -55,6 +54,21 @@ def locate_files(basename): return snap_format_string, group_format_string, n_bin_file, n_group_file +def read_group_file(filename): + usecols = (0, 1, 5, 7, 8, 9, 10, 45) + dtype = [ + ("ID", "i4"), + ("DescID", "i4"), + ("Rvir", "f4"), + ("Np", "i8"), + ("X", "f4"), + ("Y", "f4"), + ("Z", "f4"), + ("PID", "i4"), + ] + return np.genfromtxt(filename, usecols=usecols, dtype=dtype) + + def read_rockstar_groupnr(basename): """ Read particle IDs and group numbers from rockstar binary output. @@ -111,9 +125,9 @@ def read_rockstar_groupnr(basename): for halo_id, halo_offset in zip( halo_file["Halo"]["id"], halo_file["Halo"]["num_p"] ): - local_grnr[ - offset + file_offset : offset + file_offset + halo_offset - ] = halo_id + local_grnr[offset + file_offset : offset + file_offset + halo_offset] = ( + halo_id + ) file_offset += halo_offset offset += n_part_file @@ -141,7 +155,7 @@ def read_rockstar_catalogue(comm, basename, a_unit, registry, boxsize): search_radius - initial search radius which includes all member particles is_central - integer 1 for centrals, 0 for satellites nr_bound_part - number of bound particles in each halo - + Any other arrays will be passed through to the output ONLY IF they are documented in property_table.py. @@ -181,28 +195,24 @@ def read_rockstar_catalogue(comm, basename, a_unit, registry, boxsize): first_file[comm_rank], first_file[comm_rank] + files_on_rank[comm_rank] ): filename = group_format_string % {"file_nr": file_nr} + data = read_group_file(filename) - with open(filename, "r") as file: - cols = file.readline()[1:] - cols = cols.split() - data = pd.read_csv(filename, names=cols, comment="#", delim_whitespace=True) - - local_halo["index"].append(np.array(data["ID"])) + local_halo["index"].append(data["ID"]) # Note this is not the most bound particle - x = np.array(data["X"]).reshape(-1, 1) - y = np.array(data["Y"]).reshape(-1, 1) - z = np.array(data["Z"]).reshape(-1, 1) + x = data["X"] + y = data["Y"] + z = data["Z"] local_halo["cofp"].append(np.concatenate([x, y, z], axis=1)) - parent_id = np.array(data["PID"]) + parent_id = data["PID"] local_halo["is_central"].append(parent_id == -1) local_halo["PID"].append(parent_id) - local_halo["DescID"].append(np.array(data["DescID"])) + local_halo["DescID"].append(data["DescID"]) - local_halo["nr_bound_part"].append(np.array(data["Np"])) + local_halo["nr_bound_part"].append(data["Np"]) - local_halo["search_radius"].append(np.array(data["Rvir"])) + local_halo["search_radius"].append(data["Rvir"]) # Get SWIFT's definition of physical and comoving Mpc units swift_pmpc = unyt.Unit("swift_mpc", registry=registry) @@ -241,66 +251,3 @@ def read_rockstar_catalogue(comm, basename, a_unit, registry, boxsize): ) return local_halo - - -def test_read_rockstar_groupnr(basename): - """ - Read in rockstar group numbers and compute the number of particles - in each group. This is then compared with the input catalogue as a - sanity check on the group membershp files. - """ - - from mpi4py import MPI - - comm = MPI.COMM_WORLD - comm_rank = comm.Get_rank() - comm_size = comm.Get_size() - - _, ids, grnr = read_rockstar_groupnr(basename) - del ids # Don't need the particle IDs - - # Find maximum group number - max_grnr = comm.allreduce(np.amax(grnr), op=MPI.MAX) - nr_groups_from_grnr = max_grnr + 1 - if comm_rank == 0: - print(f"Number of groups from membership files = {nr_groups_from_grnr}") - - # Discard particles in no group - keep = grnr >= 0 - grnr = grnr[keep] - - # Compute group sizes - import virgo.mpi.parallel_sort as psort - - nbound_from_grnr = psort.parallel_bincount(grnr, comm=comm) - - # Rockstar outputs are csv, so can't use phdf5 to read - if comm_rank == 0: - snap_format_string, group_format_string, _, n_group_file = locate_files( - basename - ) - for file_nr in range(n_group_file): - - filename = group_format_string % {"file_nr": file_nr} - - with open(filename, "r") as file: - cols = file.readline()[1:] - cols = cols.split() - data = pd.read_csv(filename, names=cols, comment="#", delim_whitespace=True) - - # Extract halo ids and number of particles from group files - halo_ids = np.array(data["ID"], dtype=int) - num_p = np.array(data["Np"], dtype=int) - - # Compare - if not np.all(nbound_from_grnr[halo_ids] == num_p): - different = nbound_from_grnr[halo_ids] != num_p - print("The following halo ids differ:", halo_ids[different]) - - -if __name__ == "__main__": - - import sys - - basename = sys.argv[1] - test_read_rockstar_groupnr(basename) diff --git a/read_subfind.py b/SOAP/catalogue_readers/read_subfind.py similarity index 76% rename from read_subfind.py rename to SOAP/catalogue_readers/read_subfind.py index 2fab1abc..b6074dd2 100644 --- a/read_subfind.py +++ b/SOAP/catalogue_readers/read_subfind.py @@ -139,7 +139,7 @@ def read_gadget4_catalogue(comm, basename, a_unit, registry, boxsize): search_radius - initial search radius which includes all member particles is_central - integer 1 for centrals, 0 for satellites nr_bound_part - number of bound particles in each halo - + Any other arrays will be passed through to the output ONLY IF they are documented in property_table.py. @@ -225,7 +225,7 @@ def read_gadget4_catalogue(comm, basename, a_unit, registry, boxsize): registry=registry, ) - # Store initial search radius (TODO: check this is still in physical units, unlike the position) + # Store initial search radius search_radius = ( data["Subhalo/SubhaloHalfmassRad"] * length_conversion * swift_pmpc ) # different units from cofm, not a typo! @@ -243,76 +243,3 @@ def read_gadget4_catalogue(comm, basename, a_unit, registry, boxsize): local_halo[name] = unyt.unyt_array(local_halo[name], registry=registry) return local_halo - - -def test_read_gadget4_groupnr(basename): - """ - Read in Gadget-4 group numbers and compute the number of particles - in each group. This is then compared with the input catalogue as a - sanity check on the group membershp files. - """ - - from mpi4py import MPI - - comm = MPI.COMM_WORLD - comm_rank = comm.Get_rank() - comm_size = comm.Get_size() - - n_halo, ids, grnr = read_gadget4_groupnr(basename) - del ids # Don't need the particle IDs - - # Find maximum group number - max_grnr = comm.allreduce(np.amax(grnr), op=MPI.MAX) - nr_groups_from_grnr = max_grnr + 1 - if comm_rank == 0: - print(f"Number of groups from membership files = {nr_groups_from_grnr}") - - # Discard particles in no group - keep = grnr >= 0 - grnr = grnr[keep] - - # Compute group sizes - import virgo.mpi.parallel_sort as psort - - nbound_from_grnr = psort.parallel_bincount(grnr, comm=comm) - - # Locate the snapshot and fof_subhalo_tab files - if comm_rank == 0: - snap_format_string, group_format_string = locate_files(basename) - else: - snap_format_string = None - group_format_string = None - snap_format_string, group_format_string = comm.bcast( - (snap_format_string, group_format_string) - ) - - # Read group sizes from the group catalogue - subtab = phdf5.MultiFile(group_format_string, file_nr_attr=("Header", "NumFiles")) - nbound_from_subtab = subtab.read("Subhalo/SubhaloLen") - - # Find number of groups in the subfind output - nr_groups_from_subtab = comm.allreduce(len(nbound_from_subtab)) - if comm_rank == 0: - print(f"Number of groups from fof_subhalo_tab = {nr_groups_from_subtab}") - if nr_groups_from_subtab != nr_groups_from_grnr: - print("Number of groups does not agree!") - comm.Abort(1) - - # Ensure nbound arrays are partitioned the same way - nr_per_rank = comm.allgather(len(nbound_from_subtab)) - nbound_from_grnr = psort.repartition( - nbound_from_grnr, ndesired=nr_per_rank, comm=comm - ) - - # Compare - nr_different = comm.allreduce(np.sum(nbound_from_grnr != nbound_from_subtab)) - if comm_rank == 0: - print(f"Number of group sizes which differ = {nr_different} (should be 0!)") - - -if __name__ == "__main__": - - import sys - - basename = sys.argv[1] - test_read_gadget4_groupnr(basename) diff --git a/SOAP/catalogue_readers/read_subfind_eagle.py b/SOAP/catalogue_readers/read_subfind_eagle.py new file mode 100644 index 00000000..31ff7fb0 --- /dev/null +++ b/SOAP/catalogue_readers/read_subfind_eagle.py @@ -0,0 +1,135 @@ +#!/bin/env python + +import os + +import numpy as np +import h5py +import unyt + +import virgo.mpi.util +import virgo.mpi.parallel_hdf5 as phdf5 + + +def read_subfind_catalogue(comm, basename, a_unit, registry, boxsize): + """ + Read in the EAGLE subhalo catalogue, distributed over communicator comm. + + comm - communicator to distribute catalogue over + basename - SubFind catalogue filename without the .N.hdf5 suffix + a_unit - unyt a factor + registry - unyt unit registry + boxsize - box size as a unyt quantity + + Returns a dict of unyt arrays with the halo properies. + Arrays which must always be returned: + + index - index of each halo in the input catalogue + cofp - (N,3) array with centre to use for SO calculations + search_radius - initial search radius which includes all member particles + is_central - integer 1 for centrals, 0 for satellites + nr_bound_part - number of bound particles in each halo + + """ + + comm_size = comm.Get_size() + comm_rank = comm.Get_rank() + + sub_format_string = basename + ".{file_nr}.hdf5" + sub_file = phdf5.MultiFile( + sub_format_string, file_nr_attr=("Header", "NumFilesPerSnapshot") + ) + + # Get SWIFT's definition of physical and comoving Mpc units + swift_pmpc = unyt.Unit("swift_mpc", registry=registry) + swift_cmpc = unyt.Unit(a_unit * swift_pmpc, registry=registry) + + if comm_rank == 0: + with h5py.File(sub_format_string.format(file_nr=0), "r") as file: + h = file["Header"].attrs["HubbleParam"] + # Check units are indeed what we are assuming below + units_header = file["Units"].attrs + mpc_in_cm = (1 * unyt.Mpc).to("cm").value + assert np.isclose(units_header["UnitLength_in_cm"], mpc_in_cm) + M_in_g = (10**10 * unyt.Msun).to("g").value + assert np.isclose(units_header["UnitMass_in_g"], M_in_g, rtol=1e-2) + assert file["Subhalo/CentreOfPotential"].attrs["h-scale-exponent"] == -1 + assert file["Subhalo/CentreOfPotential"].attrs["aexp-scale-exponent"] == 1 + assert file["Subhalo/VmaxRadius"].attrs["h-scale-exponent"] == -1 + assert file["Subhalo/VmaxRadius"].attrs["aexp-scale-exponent"] == 1 + else: + h = None + h = comm.bcast(h) + + sub_format_string = basename + ".{file_nr}.hdf5" + sub_file = phdf5.MultiFile( + sub_format_string, file_nr_attr=("Header", "NumFilesPerSnapshot") + ) + + # Read halo properties we need + datasets = ( + "Subhalo/GroupNumber", + "Subhalo/SubGroupNumber", + "Subhalo/CentreOfPotential", + "Subhalo/SubLength", + "Subhalo/VmaxRadius", + ) + data = sub_file.read(datasets) + + # Assign indexes to the subhalos + nr_local_halos = data["Subhalo/CentreOfPotential"].shape[0] + local_offset = comm.scan(nr_local_halos) - nr_local_halos + index = np.arange(nr_local_halos, dtype=int) + local_offset + index = unyt.unyt_array( + index, dtype=int, units=unyt.dimensionless, registry=registry + ) + + # Store GroupNumber and SubGroupNumber + group_nr = unyt.unyt_array( + data["Subhalo/GroupNumber"], + dtype=int, + units=unyt.dimensionless, + registry=registry, + ) + sub_group_nr = unyt.unyt_array( + data["Subhalo/SubGroupNumber"], + dtype=int, + units=unyt.dimensionless, + registry=registry, + ) + + # Get position in comoving Mpc + cofp = (data["Subhalo/CentreOfPotential"] / h) * swift_cmpc + + # Store central halo flag + is_central = unyt.unyt_array( + data["Subhalo/SubGroupNumber"] == 0, + dtype=int, + units=unyt.dimensionless, + registry=registry, + ) + + # Store number of bound particles in each halo + nr_bound_part = unyt.unyt_array( + data["Subhalo/SubLength"], + dtype=int, + units=unyt.dimensionless, + registry=registry, + ) + + # Store initial search radius + search_radius = (5 * data["Subhalo/VmaxRadius"] / h) * swift_cmpc + + local_halo = { + "cofp": cofp, + "index": index, + "search_radius": search_radius, + "is_central": is_central, + "nr_bound_part": nr_bound_part, + "group_nr": group_nr, + "sub_group_nr": sub_group_nr, + } + + for name in local_halo: + local_halo[name] = unyt.unyt_array(local_halo[name], registry=registry) + + return local_halo diff --git a/read_vr.py b/SOAP/catalogue_readers/read_vr.py similarity index 95% rename from read_vr.py rename to SOAP/catalogue_readers/read_vr.py index 95171fcf..f14cdff6 100644 --- a/read_vr.py +++ b/SOAP/catalogue_readers/read_vr.py @@ -2,16 +2,13 @@ import os +from mpi4py import MPI import numpy as np import h5py import unyt -import pytest - import virgo.mpi.util import virgo.mpi.parallel_hdf5 as phdf5 -from mpi4py import MPI - comm = MPI.COMM_WORLD comm_rank = comm.Get_rank() comm_size = comm.Get_size() @@ -34,7 +31,7 @@ def read_vr_datasets(vr_basename, file_type, datasets, return_file_nr=None): def compute_lengths(offsets, total_nr_ids): """ Compute group lengths given the offsets and the total number - of particle IDs. + of particle IDs. """ # Only include ranks with >0 groups @@ -187,7 +184,7 @@ def vr_group_membership_from_ids( def read_vr_groupnr(basename): """ - Read VR output and return group number for each particle ID + Read VR output and return group number for each particle ID """ ( length_bound, @@ -207,7 +204,7 @@ def read_vr_groupnr(basename): local_nr_halos = len(offset_bound) total_nr_halos = comm.allreduce(local_nr_halos) - return total_nr_halos, ids_bound, grnr_bound, rank_bound, ids_unbound, grnr_unbound + return total_nr_halos, ids_bound, grnr_bound, rank_bound def read_vr_catalogue(comm, basename, a_unit, registry, boxsize): @@ -229,7 +226,7 @@ def read_vr_catalogue(comm, basename, a_unit, registry, boxsize): is_central - integer 1 for centrals, 0 for satellites nr_bound_part - number of bound particles in each halo nr_unbound_part - number of unbound particles in each halo - + Any other arrays will be passed through to the output ONLY IF they are documented in property_table.py. """ @@ -376,7 +373,7 @@ def vr_filename(file_type, file_nr=None): for dim in range(3): need_wrap = dist[:, dim] > 0.5 * boxsize dist[need_wrap, dim] = boxsize - dist[need_wrap, dim] - dist = np.sqrt(np.sum(dist ** 2, axis=1)) + dist = np.sqrt(np.sum(dist**2, axis=1)) # Store the initial search radius local_halo["search_radius"] = local_halo["R_size"] * 1.01 + dist @@ -393,12 +390,12 @@ def read_vr_group_sizes(basename, suffix, comm): Compute number of bound and unbound particles in each group. This is much more complicated than it should be because VR doesn't write out bound and unbound sizes. Instead we have to compute them in an awkward way. - + For groups which are not last in the file, group i has size Offset[i+1]-Offset[i]. For the last group in each file we need to know the number of particles in the corresponding catalog_particles file to compute its size. - + basename: VR output filename without trailing .properties[.0] or similar suffix: format string to add file number suffix, if necesary """ @@ -542,25 +539,3 @@ def vr_filename(file_type): ) return nr_parts_bound, nr_parts_unbound - - -@pytest.mark.mpi -def test_read_vr(snap_nr=57): - - basename = f"/cosma8/data/dp004/flamingo/Runs/L1000N1800/HYDRO_FIDUCIAL/VR/catalogue_{snap_nr:04d}/vr_catalogue_{snap_nr:04d}" - # basename = f"/cosma8/data/dp004/flamingo/Runs/L1000N0900/HYDRO_FIDUCIAL/VR/catalogue_{snap_nr:04d}/vr_catalogue_{snap_nr:04d}" - suffix = ".%(file_nr)d" - - nr_parts_bound, nr_parts_unbound = read_vr_group_sizes(basename, suffix, comm) - - comm.barrier() - nr_halos_total = comm.allreduce(len(nr_parts_bound)) - if comm_rank == 0: - print(f"Read {nr_halos_total} halos") - - -if __name__ == "__main__": - import sys - - snap_nr = int(sys.argv[1]) - test_read_vr(snap_nr=snap_nr) diff --git a/compute_halo_properties.py b/SOAP/compute_halo_properties.py similarity index 68% rename from compute_halo_properties.py rename to SOAP/compute_halo_properties.py index dbb7729a..d9a9be12 100644 --- a/compute_halo_properties.py +++ b/SOAP/compute_halo_properties.py @@ -17,25 +17,28 @@ import numpy as np import unyt - -import halo_centres -import swift_cells -import chunk_tasks -import task_queue -import lustre -import soap_args -import SO_properties -import subhalo_properties -import aperture_properties -import result_set -from combine_chunks import combine_chunks, sub_snapnum -import projected_aperture_properties -from recently_heated_gas_filter import RecentlyHeatedGasFilter -from stellar_age_calculator import StellarAgeCalculator -from cold_dense_gas_filter import ColdDenseGasFilter -from category_filter import CategoryFilter -from parameter_file import ParameterFile -from mpi_timer import MPITimer +from SOAP.core import ( + chunk_tasks, + halo_centres, + lustre, + result_set, + soap_args, + swift_cells, + task_queue, +) +from SOAP.core.parameter_file import ParameterFile +from SOAP.core.mpi_timer import MPITimer +from SOAP.core.combine_chunks import combine_chunks, sub_snapnum +from SOAP.core.category_filter import CategoryFilter +from SOAP.particle_selection import ( + aperture_properties, + SO_properties, + projected_aperture_properties, + subhalo_properties, +) +from SOAP.property_calculation.stellar_age_calculator import StellarAgeCalculator +from SOAP.particle_filter.cold_dense_gas_filter import ColdDenseGasFilter +from SOAP.particle_filter.recently_heated_gas_filter import RecentlyHeatedGasFilter # Set numpy to raise divide by zero, overflow and invalid operation errors as exceptions @@ -65,8 +68,12 @@ def get_rank_and_size(comm): def compute_halo_properties(): + # Start the clock + t0 = time.time() + # Read command line parameters args = soap_args.get_soap_args(comm_world) + comm_world.barrier() # Enable profiling, if requested if args.profile == 2 or (args.profile == 1 and comm_world_rank == 0): @@ -75,10 +82,6 @@ def compute_halo_properties(): pr = cProfile.Profile() pr.enable() - # Start the clock - comm_world.barrier() - t0 = time.time() - # Split MPI ranks according to which node they are on. # Only the first rank on each node belongs to comm_inter_node. # Others have comm_inter_node=MPI_COMM_NULL and inter_node_rank=-1. @@ -156,6 +159,7 @@ def compute_halo_properties(): cellgrid.snapshot_datasets.setup_defined_constants( parameter_file.get_defined_constants() ) + parameter_file.record_property_timings = args.record_property_timings # Try to load parameters for RecentlyHeatedGasFilter. If a property that uses the # filter is calculated when the parameters could not be found, the code will @@ -185,16 +189,12 @@ def compute_halo_properties(): # Try to load parameters for ColdDenseGasFilter. If a property that uses the # filter is calculated when the parameters could not be found, the code will # crash. - try: - cold_dense_params = args.calculations["cold_dense_gas_filter"] - cold_dense_gas_filter = ColdDenseGasFilter( - float(cold_dense_params["maximum_temperature_K"]) * unyt.K, - float(cold_dense_params["minimum_hydrogen_number_density_cm3"]) - / unyt.cm ** 3, - True, - ) - except KeyError: - cold_dense_gas_filter = ColdDenseGasFilter(0 * unyt.K, 0 / unyt.cm ** 3, False) + cold_dense_params = parameter_file.get_cold_dense_params() + cold_dense_gas_filter = ColdDenseGasFilter( + cold_dense_params["maximum_temperature_K"] * unyt.K, + cold_dense_params["minimum_hydrogen_number_density_cm3"] / unyt.cm**3, + cold_dense_params["initialised"], + ) default_filters = { "general": { @@ -222,9 +222,8 @@ def compute_halo_properties(): "combine_properties": "sum", }, } - category_filter = CategoryFilter( - parameter_file.get_filters(default_filters), dmo=args.dmo - ) + filters = parameter_file.get_filters(default_filters) + category_filter = CategoryFilter(filters, dmo=args.dmo) # Get the full list of property calculations we can do # Note that the order matters: we need to do the BoundSubhalo first, @@ -232,36 +231,21 @@ def compute_halo_properties(): # Similarly, things like SO 5xR500_crit can only be done after # SO 500_crit for obvious reasons halo_prop_list = [] - # Make sure BoundSubhalo is always first, since it's used for filters - subhalo_variations = parameter_file.get_halo_type_variations( - "SubhaloProperties", {"Bound": {"bound_only": True}} + + # We require BoundSubhalo since it's used for filters + if comm_world_rank == 0: + if "SubhaloProperties" not in parameter_file.parameters: + print("SubhaloProperties must be in the parameter file") + comm_world.Abort(1) + halo_prop_list.append( + subhalo_properties.SubhaloProperties( + cellgrid, + parameter_file, + recently_heated_gas_filter, + stellar_age_calculator, + category_filter, + ) ) - for variation in subhalo_variations: - if subhalo_variations[variation]["bound_only"]: - halo_prop_list.append( - subhalo_properties.SubhaloProperties( - cellgrid, - parameter_file, - recently_heated_gas_filter, - stellar_age_calculator, - category_filter, - bound_only=subhalo_variations[variation]["bound_only"], - ) - ) - assert len(halo_prop_list) > 0, "BoundSubhalo must be calculated" - # Adding FOFSubhaloProperties if present - for variation in subhalo_variations: - if not subhalo_variations[variation]["bound_only"]: - halo_prop_list.append( - subhalo_properties.SubhaloProperties( - cellgrid, - parameter_file, - recently_heated_gas_filter, - stellar_age_calculator, - category_filter, - bound_only=subhalo_variations[variation]["bound_only"], - ) - ) SO_variations = parameter_file.get_halo_type_variations( "SOProperties", @@ -352,18 +336,52 @@ def compute_halo_properties(): "exclusive_3000_kpc": {"radius_in_kpc": 3000.0, "inclusive": False}, }, ) + + # Sort the aperture variations based on their radii, and create a list + # of all apertures. This is required since we can skip some of the larger + # apertures if all the particles were already included in the previous aperture + aperture_variations = dict( + sorted(aperture_variations.items(), key=lambda x: x[1].get("radius_in_kpc", 0)) + ) + inclusive_radii_kpc = [] + exclusive_radii_kpc = [] for variation in aperture_variations: + # We don't consider this for apertures defined based on properties + if "radius_in_kpc" not in aperture_variations[variation]: + continue if aperture_variations[variation]["inclusive"]: + inclusive_radii_kpc.append(aperture_variations[variation]["radius_in_kpc"]) + else: + exclusive_radii_kpc.append(aperture_variations[variation]["radius_in_kpc"]) + assert inclusive_radii_kpc == sorted(inclusive_radii_kpc) + assert exclusive_radii_kpc == sorted(exclusive_radii_kpc) + + # Add the apertures defined with fixed physical radii + for variation in aperture_variations: + if "radius_in_kpc" not in aperture_variations[variation]: + continue + assert "property" not in aperture_variations[variation] + assert "radius_multiple" not in aperture_variations[variation] + if aperture_variations[variation]["inclusive"]: + # If skip_gt_enclose_radius is False (which is the default) then + # we always want to calculate its properties, regardless of the + # size of the next smallest aperture. + radii_kpc = [aperture_variations[variation]["radius_in_kpc"]] + if aperture_variations[variation].get("skip_gt_enclose_radius", False): + radii_kpc = inclusive_radii_kpc + halo_prop_list.append( aperture_properties.InclusiveSphereProperties( cellgrid, parameter_file, aperture_variations[variation]["radius_in_kpc"], + None, recently_heated_gas_filter, stellar_age_calculator, cold_dense_gas_filter, category_filter, aperture_variations[variation].get("filter", "basic"), + radii_kpc, ) ) else: @@ -372,13 +390,56 @@ def compute_halo_properties(): cellgrid, parameter_file, aperture_variations[variation]["radius_in_kpc"], + None, recently_heated_gas_filter, stellar_age_calculator, cold_dense_gas_filter, category_filter, aperture_variations[variation].get("filter", "basic"), + exclusive_radii_kpc, ) ) + + # Add the apertures based on SOAP properties + for variation in aperture_variations: + if "radius_in_kpc" in aperture_variations[variation]: + continue + assert "property" in aperture_variations[variation] + radius_multiple = aperture_variations[variation].get("radius_multiple", 1) + # Only allow integer radius mutiples, otherwise swiftsimio will + # struggle to handle the group names + assert int(radius_multiple) == radius_multiple + if aperture_variations[variation]["inclusive"]: + halo_prop_list.append( + aperture_properties.InclusiveSphereProperties( + cellgrid, + parameter_file, + None, + (aperture_variations[variation]["property"], radius_multiple), + recently_heated_gas_filter, + stellar_age_calculator, + cold_dense_gas_filter, + category_filter, + aperture_variations[variation].get("filter", "basic"), + [], + ) + ) + else: + halo_prop_list.append( + aperture_properties.ExclusiveSphereProperties( + cellgrid, + parameter_file, + None, + (aperture_variations[variation]["property"], radius_multiple), + recently_heated_gas_filter, + stellar_age_calculator, + cold_dense_gas_filter, + category_filter, + aperture_variations[variation].get("filter", "basic"), + [], + ) + ) + projected_aperture_variations = parameter_file.get_halo_type_variations( "ProjectedApertureProperties", { @@ -388,16 +449,67 @@ def compute_halo_properties(): "100_kpc": {"radius_in_kpc": 100.0}, }, ) + # Sort the aperture variations based on their radii, and create a list + # of all apertures. This is required since we can skip some of the larger + # apertures if all the particles were already included in the previous aperture + projected_aperture_variations = dict( + sorted( + projected_aperture_variations.items(), + key=lambda x: x[1].get("radius_in_kpc", 0), + ) + ) + projected_radii_kpc = [] for variation in projected_aperture_variations: + # We don't consider this for apertures defined based on properties + if "radius_in_kpc" not in projected_aperture_variations[variation]: + continue + projected_radii_kpc.append( + projected_aperture_variations[variation]["radius_in_kpc"] + ) + assert projected_radii_kpc == sorted(projected_radii_kpc) + # Add the apertures defined with fixed physical radii + for variation in projected_aperture_variations: + if "radius_in_kpc" not in projected_aperture_variations[variation]: + continue + assert "property" not in projected_aperture_variations[variation] + assert "radius_multiple" not in projected_aperture_variations[variation] halo_prop_list.append( projected_aperture_properties.ProjectedApertureProperties( cellgrid, parameter_file, projected_aperture_variations[variation]["radius_in_kpc"], + None, category_filter, projected_aperture_variations[variation].get("filter", "basic"), + projected_radii_kpc, ) ) + # Add the apertures based on SOAP properties + for variation in projected_aperture_variations: + if "radius_in_kpc" in projected_aperture_variations[variation]: + continue + assert "property" in projected_aperture_variations[variation] + radius_multiple = projected_aperture_variations[variation].get( + "radius_multiple", 1 + ) + assert int(radius_multiple) == radius_multiple + halo_prop_list.append( + projected_aperture_properties.ProjectedApertureProperties( + cellgrid, + parameter_file, + None, + (projected_aperture_variations[variation]["property"], radius_multiple), + category_filter, + projected_aperture_variations[variation].get("filter", "basic"), + projected_radii_kpc, + ) + ) + + # The category_filter needs access to the filters for each property + # whenever we are writing the final output file. It needs to get this + # information from the parmeter file. It would be better to get rid of + # the category_filter object, and combine it with the parameter_file + category_filter.set_property_filters(parameter_file.property_filters) if comm_world_rank == 0 and args.output_parameters: parameter_file.write_parameters(args.output_parameters) @@ -416,13 +528,25 @@ def compute_halo_properties(): print("for central and satellite halos") if args.snipshot: print("Running in snipshot mode") + if args.record_halo_timings: + print("Storing processing time for each halo") + if args.record_property_timings: + print("Storing processing time for each property") parameter_file.print_unregistered_properties() - parameter_file.print_invalid_properties() + parameter_file.print_invalid_properties(halo_prop_list) + if not parameter_file.renclose_enabled(): + print( + "BoundSubhalo/EncloseRadius is not enabled. This means apertures with r > r_enclose will be calculated explicitly, rather than copying over values from smaller apertures" + ) category_filter.print_filters() # Ensure output dir exists if comm_world_rank == 0: - lustre.ensure_output_dir(args.output_file) + try: + os.makedirs(os.path.dirname(args.output_file), exist_ok=True) + except OSError as e: + print(f"Error creating output directory: {e}") + comm_world.Abort(1) comm_world.barrier() # Read in the halo catalogue: @@ -430,17 +554,11 @@ def compute_halo_properties(): halo_basename = sub_snapnum(args.halo_basename, args.snapshot_nr) so_cat = halo_centres.SOCatalogue( comm_world, - halo_basename, - args.halo_format, cellgrid.a_unit, cellgrid.snap_unit_registry, cellgrid.boxsize, - args.max_halos, - args.centrals_only, - args.halo_indices, halo_prop_list, - args.chunks, - args.min_read_radius_cmpc, + args, ) so_cat.start_request_thread() @@ -454,15 +572,6 @@ def compute_halo_properties(): else: tasks = None - # Report initial set-up time - comm_world.barrier() - t1 = time.time() - if comm_world_rank == 0: - print( - "Reading %d input halos and setting up %d chunk(s) took %.1fs" - % (so_cat.nr_halos, len(tasks), t1 - t0) - ) - # Make a format string to generate the name of the file each chunk task will write to scratch_file_format = ( args.scratch_dir @@ -476,11 +585,21 @@ def compute_halo_properties(): scratch_file_name = scratch_file_format % {"file_nr": file_nr} scratch_file_dir = os.path.dirname(scratch_file_name) try: - os.makedirs(scratch_file_dir) - except OSError: - pass + os.makedirs(scratch_file_dir, exist_ok=True) + except OSError as e: + print(f"Error creating scratch directory: {e}") + comm_world.Abort(1) comm_world.barrier() + # Report initial set-up time + setup_time_local = time.time() - t0 + t1 = time.time() + if comm_world_rank == 0: + print( + "Reading %d input halos and setting up %d chunk(s) took %.1fs" + % (so_cat.nr_halos, len(tasks), t1 - t0) + ) + # Execute the chunk tasks. This writes one file per chunk with the halo properties. # For each chunk it returns a list with (name, size, units, description) for each # quantity that was calculated. @@ -515,6 +634,8 @@ def compute_halo_properties(): ref_metadata = result_set.check_metadata(metadata, comm_inter_node, comm_world) # Combine chunks into a single output file + comm_world.barrier() + t0_combine = time.time() combine_chunks( args, cellgrid, @@ -529,7 +650,6 @@ def compute_halo_properties(): ) # Delete scratch files - comm_world.barrier() if comm_world_rank == 0: for file_nr in range(nr_chunks): os.remove(scratch_file_format % {"file_nr": file_nr}) @@ -537,17 +657,9 @@ def compute_halo_properties(): comm_world.barrier() # Stop the clock - comm_world.barrier() + combine_time_local = time.time() - t0_combine t1 = time.time() - # Find total time spent running tasks - if len(timings) > 0: - task_time_local = sum(timings) - else: - task_time_local = 0.0 - task_time_total = comm_world.allreduce(task_time_local) - task_time_fraction = task_time_total / (comm_world_size * (t1 - t0)) - # Save profiling results for each MPI rank if args.profile == 2 or (args.profile == 1 and comm_world_rank == 0): pr.disable() @@ -561,11 +673,25 @@ def compute_halo_properties(): with open("./profile.%d.txt" % comm_world_rank, "w") as profile_file: profile_file.write(s.getvalue()) + # Find total time spent running tasks + if len(timings) > 0: + task_time_local = sum(timings) + else: + task_time_local = 0.0 + setup_time_total = comm_world.allreduce(setup_time_local) + setup_time_fraction = setup_time_total / (comm_world_size * (t1 - t0)) + task_time_total = comm_world.allreduce(task_time_local) + task_time_fraction = task_time_total / (comm_world_size * (t1 - t0)) + combine_time_total = comm_world.allreduce(combine_time_local) + combine_time_fraction = combine_time_total / (comm_world_size * (t1 - t0)) + if comm_world_rank == 0: + print("Fraction of time spent setting up = %.2f" % setup_time_fraction) print( "Fraction of time spent calculating halo properties = %.2f" % task_time_fraction ) + print("Fraction of time spent combining chunks = %.2f" % combine_time_fraction) print("Total elapsed time: %.1f seconds" % (t1 - t0)) print("Done.") diff --git a/SOAP/core/__init__.py b/SOAP/core/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/category_filter.py b/SOAP/core/category_filter.py similarity index 82% rename from category_filter.py rename to SOAP/core/category_filter.py index 96a4aca7..edfec534 100644 --- a/category_filter.py +++ b/SOAP/core/category_filter.py @@ -22,16 +22,17 @@ BoundSubhalo properties. """ -from property_table import PropertyTable from typing import Dict +from SOAP.property_table import PropertyTable + class CategoryFilter: """ Filter used to determine whether properties need to be calculated for a certain halo or not. - This decision is always based on the number of particles in the subhalo, + This decision is always based on the number of particles in the subhalo, and requires the calculation of BoundSubhalo for each halo. """ @@ -63,6 +64,7 @@ def __init__(self, filters: Dict, dmo: bool = False): """ self.filters = filters self.dmo = dmo + self.property_filters = {} def get_do_calculation( self, halo_result: Dict, precomputed_properties: Dict = {} @@ -79,7 +81,7 @@ def get_do_calculation( Returns a dictionary containing True/False for each filter category. """ - do_calculation = {"basic": True, "DMO": self.dmo} + do_calculation = {"basic": True} if self.dmo: precomputed_properties["BoundSubhalo/NumberOfGasParticles"] = 0 precomputed_properties["BoundSubhalo/NumberOfStarParticles"] = 0 @@ -106,6 +108,8 @@ def get_do_calculation( def get_compression_metadata(self, property_output_name: str) -> Dict: """ Get the dictionary with compression metadata for a particular property. + This function should only be called for properties that exist in the + property table. Parameters: - property_output_name: str @@ -120,16 +124,21 @@ def get_compression_metadata(self, property_output_name: str) -> Dict: actually applied to the data. Currently, this is always "False", since the lossy compression is done by a post-processing script. """ - base_output_name = property_output_name.split("/")[-1] - compression = None - for _, prop in PropertyTable.full_property_list.items(): - if prop[0] == base_output_name: - compression = prop[6] - if compression is None: - - return {"Lossy compression filter": "None", "Is Compressed": False} - else: - return {"Lossy compression filter": compression, "Is Compressed": False} + # Output names can have / in them (for halo finder properties), so + # we need to loop over that case + split_name = property_output_name.split("/") + for i in range(len(split_name)): + output_name = "/".join(split_name[-1 - i :]) + for _, prop in PropertyTable.full_property_list.items(): + if prop.name == output_name: + return { + "Lossy compression filter": prop.lossy_compression_filter, + "Is Compressed": False, + } + raise RuntimeError(f"Compression filter not found for {split_name}") + + def set_property_filters(self, property_filters: Dict) -> None: + self.property_filters = property_filters def get_filter_metadata_for_property(self, property_output_name: str) -> Dict: """ @@ -137,18 +146,27 @@ def get_filter_metadata_for_property(self, property_output_name: str) -> Dict: Parameters: - property_output_name: str - Name of a halo property as it appears in the output file. + Full path to halo property as it appears in the output file. Returns a dictionary with the same format as those output by get_filter_metadata """ - base_output_name = property_output_name.split("/")[-1] - category = None - for _, prop in PropertyTable.full_property_list.items(): - if prop[0] == base_output_name: - category = prop[5] - # category=None corresponds to quantities outside the table - # (e.g. "density_in_search_radius") + # Convert from output file group name to parameter file name + # Return None for groups (e.g. InputHalos) which are never masked + base_halo_type = { + "BoundSubhalo": "SubhaloProperties", + "ProjectedAperture": "ProjectedApertureProperties", + "SO": "SOProperties", + "ExclusiveSphere": "ApertureProperties", + "InclusiveSphere": "ApertureProperties", + }.get(property_output_name.split("/")[0], None) + + if base_halo_type is not None: + output_name = property_output_name.split("/")[-1] + category = self.property_filters[base_halo_type][output_name] + else: + category = None + return self.get_filter_metadata(category) def get_filter_metadata(self, category: str) -> Dict: diff --git a/chunk_tasks.py b/SOAP/core/chunk_tasks.py similarity index 88% rename from chunk_tasks.py rename to SOAP/core/chunk_tasks.py index 71bb6be7..413c4088 100644 --- a/chunk_tasks.py +++ b/SOAP/core/chunk_tasks.py @@ -7,13 +7,13 @@ import numpy as np import unyt -import shared_mesh -import shared_array -from dataset_names import mass_dataset, ptypes_for_so_masses -from halo_tasks import process_halos -from mask_cells import mask_cells -import memory_use -import result_set +from . import shared_array +from . import shared_mesh +from . import result_set +from . import memory_use +from .dataset_names import mass_dataset, ptypes_for_so_masses +from .halo_tasks import process_halos +from .mask_cells import mask_cells # Will label messages with time since run start time_start = time.time() @@ -57,7 +57,7 @@ class ChunkTask: Each ChunkTask is called collectively on all of the MPI ranks in one compute node. The task imports the halos to be processed, reads in - the required patch of the snapshot and computes halo properties. + the required patch of the snapshot and computes halo properties. """ def __init__(self, halo_prop_list=None, chunk_nr=0, nr_chunks=1): @@ -95,7 +95,6 @@ def message(m): ) ) - # The first rank on this node import the halos to be processed. # It also checks if this chunk has already been processed (by # a previous SOAP run that crashed). @@ -113,17 +112,26 @@ def message(m): # Will need to broadcast names of the halo properties names = list(self.halo_arrays.keys()) + # Sort halos based on their number of bound particles + # We do this since larger halos will take longer to process + # and so we want to do them first + order = np.argsort(self.halo_arrays["nr_bound_part"])[::-1] + for name in names: + self.halo_arrays[name] = self.halo_arrays[name][order] + chunk_file_already_exists = False # Check if the chunk file exists, was fully written, and has the correct objects filename = scratch_file_format % {"file_nr": self.chunk_nr} if os.path.exists(filename): try: - with h5py.File(filename, 'r') as outfile: - chunk_file_already_exists = outfile.attrs.get('Write complete', False) - index = np.sort(outfile['InputHalos/HaloCatalogueIndex'][:]) + with h5py.File(filename, "r") as outfile: + chunk_file_already_exists = outfile.attrs.get( + "Write complete", False + ) + index = np.sort(outfile["InputHalos/HaloCatalogueIndex"][:]) file_calc_names = sorted(outfile.attrs["calc_names"].tolist()) # Check we have the correct halo indices - if not np.all(index == np.sort(self.halo_arrays['index'].value)): + if not np.all(index == np.sort(self.halo_arrays["index"].value)): chunk_file_already_exists = False # Check halo properties are the same calc_names = sorted([hp.name for hp in self.halo_prop_list]) @@ -138,7 +146,9 @@ def message(m): # File is valid, let's extracting the metadata from it if chunk_file_already_exists: - result_metadata = result_set.get_metadata_from_chunk_file(filename, self.halo_prop_list, cellgrid.snap_unit_registry) + result_metadata = result_set.get_metadata_from_chunk_file( + filename, self.halo_prop_list, cellgrid.snap_unit_registry + ) else: chunk_file_already_exists = None @@ -148,7 +158,7 @@ def message(m): chunk_file_already_exists = comm.bcast(chunk_file_already_exists) if chunk_file_already_exists: - message(f'Using pre-existing file for chunk') + message(f"using pre-existing file for chunk") return result_metadata # Then we copy the halo arrays into shared memory @@ -263,7 +273,7 @@ def message(m): for ptype in data: for name in data[ptype]: nr_bytes += data[ptype][name].full.nbytes - nr_mb = nr_bytes / (1024 ** 2) + nr_mb = nr_bytes / (1024**2) rate = nr_mb / (t1_read - t0_read) message( "read in %d particles in %.1fs = %.1fMB/s (uncompressed)" @@ -294,19 +304,16 @@ def message(m): mesh[ptype] = shared_mesh.SharedMesh(comm, pos, resolution) comm.barrier() t1_mesh = time.time() - message("constructing shared mesh took %.1fs" % (t1_mesh - t0_mesh)) - # Report remaining memory after particles have been read in and mesh has been built + msg = f"constructing shared mesh took {t1_mesh - t0_mesh:.1f}s" total_mem_gb, free_mem_gb = memory_use.get_memory_use() if total_mem_gb is not None: - message( - "node has %.1fGB of %.1fGB memory free" - % (free_mem_gb, total_mem_gb) - ) + msg += f", node has {free_mem_gb:.1f}GB of {total_mem_gb:.1f}GB memory free" + message(msg) # Calculate the halo properties t0_halos = time.time() - total_time, task_time, nr_left, nr_done = process_halos( + total_time, task_time, nr_left, nr_done, min_free_mem_gb = process_halos( comm, cellgrid.snap_unit_registry, data, @@ -333,6 +340,15 @@ def message(m): dead_time_fraction, ) ) + + # Report peak memory usage + total_mem_gb, free_mem_gb = memory_use.get_memory_use() + if total_mem_gb is not None: + message( + "at peak memory usage node had %.1fGB of %.1fGB memory free" + % (min_free_mem_gb, total_mem_gb) + ) + # Free the shared particle data for ptype in data: for name in data[ptype]: @@ -371,13 +387,13 @@ def message(m): # Write metadata in case this file is used for restarts if comm_rank == 0: - with h5py.File(filename, 'a') as outfile: + with h5py.File(filename, "a") as outfile: units = outfile.create_group("Units") for name, value in cellgrid.swift_units_group.items(): units.attrs[name] = [value] calc_names = sorted([hp.name for hp in self.halo_prop_list]) outfile.attrs["calc_names"] = calc_names - outfile.attrs['Write complete'] = True + outfile.attrs["Write complete"] = True # Return the names, dimensions and units of the quantities we computed # so that we can check they're consistent between chunks diff --git a/combine_args.py b/SOAP/core/combine_args.py similarity index 100% rename from combine_args.py rename to SOAP/core/combine_args.py diff --git a/combine_chunks.py b/SOAP/core/combine_chunks.py similarity index 60% rename from combine_chunks.py rename to SOAP/core/combine_chunks.py index ba04af96..1f969570 100644 --- a/combine_chunks.py +++ b/SOAP/core/combine_chunks.py @@ -1,19 +1,21 @@ #!/bin/env python +import os import socket import time -import numpy as np + import h5py from mpi4py import MPI - +import numpy as np import virgo.mpi.parallel_hdf5 as phdf5 import virgo.mpi.parallel_sort as psort from virgo.util.partial_formatter import PartialFormatter -from subhalo_rank import compute_subhalo_rank -import swift_units -from mpi_timer import MPITimer -from property_table import PropertyTable +from SOAP.catalogue_readers import read_hbtplus +from SOAP.property_calculation.subhalo_rank import compute_subhalo_rank +from SOAP.property_table import PropertyTable +from . import lustre, swift_units +from .mpi_timer import MPITimer def sub_snapnum(filename, snapnum): @@ -28,6 +30,37 @@ def sub_snapnum(filename, snapnum): return filename +def spatial_sort(halo_cofp, halo_index, cellgrid, comm): + """ + Sort the halos based on what cell they are in, then based + on their catalogue index + """ + cell_indices = (halo_cofp // cellgrid.cell_size).value.astype("int64") + assert cellgrid.dimension[0] >= cellgrid.dimension[1] >= cellgrid.dimension[2] + # Assert that all halos are within the box + if cell_indices.shape[0] > 0: + assert np.min(cell_indices) >= 0 + for i_cell in range(3): + assert np.max(cell_indices[:, i_cell]) < cellgrid.dimension[i_cell] + # Sort first based on position, then on catalogue index + sort_hash_dtype = [("cell_index", np.int64), ("catalogue_index", np.int64)] + sort_hash = np.zeros(cell_indices.shape[0], dtype=sort_hash_dtype) + sort_hash["cell_index"] += cell_indices[:, 0] * cellgrid.dimension[0] ** 2 + sort_hash["cell_index"] += cell_indices[:, 1] * cellgrid.dimension[1] + sort_hash["cell_index"] += cell_indices[:, 2] + sort_hash["catalogue_index"] = halo_index + order = psort.parallel_sort(sort_hash, return_index=True, comm=comm) + + # Calculate local count of halos in each cell, and combine on rank 0 + local_cell_counts = np.bincount( + sort_hash["cell_index"], minlength=cellgrid.nr_cells[0] + ).astype("int64") + assert local_cell_counts.shape[0] == np.prod(cellgrid.dimension) + cell_counts = comm.reduce(local_cell_counts) + + return order, cell_counts + + def combine_chunks( args, cellgrid, @@ -60,46 +93,80 @@ def combine_chunks( ): halo_cofp = scratch_file.read("InputHalos/HaloCentre") * cofp_units halo_index = scratch_file.read("InputHalos/HaloCatalogueIndex") - cell_indices = (halo_cofp // cellgrid.cell_size).value.astype("int64") - assert cellgrid.dimension[0] >= cellgrid.dimension[1] >= cellgrid.dimension[2] - # Sort first based on position, then on catalogue index - sort_hash_dtype = [("cell_index", np.int64), ("catalogue_index", np.int64)] - sort_hash = np.zeros(cell_indices.shape[0], dtype=sort_hash_dtype) - sort_hash["cell_index"] += cell_indices[:, 0] * cellgrid.dimension[0] ** 2 - sort_hash["cell_index"] += cell_indices[:, 1] * cellgrid.dimension[1] - sort_hash["cell_index"] += cell_indices[:, 2] - sort_hash["catalogue_index"] = halo_index - order = psort.parallel_sort(sort_hash, return_index=True, comm=comm_world) - del halo_cofp + order, cell_counts = spatial_sort(halo_cofp, halo_index, cellgrid, comm_world) - # Calculate local count of halos in each cell, and combine on rank 0 - local_cell_counts = np.bincount( - sort_hash["cell_index"], minlength=cellgrid.nr_cells[0] - ).astype("int64") - assert local_cell_counts.shape[0] == np.prod(cellgrid.dimension) - cell_counts = comm_world.reduce(local_cell_counts) + # Free up some memory + del halo_cofp + del halo_index # Determine total number of halos total_nr_halos = comm_world.allreduce(len(order)) - # Get metadata for derived quantities: these don't exist in the chunk + # Handle derived (SOAP) quantities: these don't exist in the chunk # output but will be computed by combining other halo properties. + soap_props = set() # SOAP properties which will be calculated + props_to_keep = set() # Properties required from the chunk files + if args.halo_format == "VR": + # Subhalo rank + soap_props.add("SOAP/SubhaloRankByBoundMass") + props_to_keep.update( + ( + "InputHalos/VR/ID", + "BoundSubhalo/TotalMass", + "InputHalos/VR/HostHaloID", + ) + ) + # Host halo index + soap_props.add("SOAP/HostHaloIndex") + props_to_keep.update(("InputHalos/VR/ID", "InputHalos/VR/HostHaloID")) + elif args.halo_format == "HBTplus": + # Subhalo rank + soap_props.add("SOAP/SubhaloRankByBoundMass") + props_to_keep.update( + ( + "InputHalos/HBTplus/HostFOFId", + "BoundSubhalo/TotalMass", + "InputHalos/HBTplus/TrackId", + ) + ) + # Host halo index + soap_props.add("SOAP/HostHaloIndex") + props_to_keep.update(("InputHalos/HBTplus/HostFOFId", "InputHalos/IsCentral")) + # Progenitor/Descendant SOAP index + soap_props.update(("SOAP/ProgenitorIndex", "SOAP/DescendantIndex")) + props_to_keep.add("InputHalos/HBTplus/TrackId") + + # Also keep M200c for calculating reduced_snapshot flag + if "reduced_snapshots" in args.calculations: + for metadata in ref_metadata: + if metadata[0] == "SO/200_crit/TotalMass": + soap_props.add("SOAP/IncludedInReducedSnapshot") + props_to_keep.add("SO/200_crit/TotalMass") + break + else: + if comm_world.Get_rank() == 0: + print("IncludedInReducedSnapshot is enabled, but M200c is missing") + + # Get metadata for derived quantities soap_metadata = [] - for soapkey in PropertyTable.soap_properties: - props = PropertyTable.full_property_list[f"{soapkey}"] - name = f"SOAP/{soapkey}" - size = props[1] + for key, prop in PropertyTable.full_property_list.items(): + if not key.startswith("SOAP/"): + continue + if not key in soap_props: + continue + name = prop.name + size = prop.shape if size == 1: # Scalar quantity size = () else: # Vector quantity size = (size,) - dtype = props[2] - unit = cellgrid.get_unit(props[3]) - description = props[4] - physical = props[9] - a_exponent = props[10] + dtype = prop.dtype + unit = cellgrid.get_unit(prop.unit) + description = prop.description + physical = prop.output_physical + a_exponent = prop.a_scale_exponent if not physical: unit = unit * cellgrid.get_unit("a") ** a_exponent soap_metadata.append( @@ -109,31 +176,40 @@ def combine_chunks( # Add metadata for FOF properties fof_metadata = [] if (args.fof_group_filename != "") and (args.halo_format == "HBTplus"): - for fofkey in ["Centres", "Masses", "Sizes"]: - props = PropertyTable.full_property_list[f"FOF/{fofkey}"] - name = f"InputHalos/FOF/{fofkey}" - size = props[1] + fof_keys = ["Centres", "Masses", "Sizes"] + if args.fof_radius_filename != "": + fof_keys.append("Radii") + for fofkey in fof_keys: + prop = PropertyTable.full_property_list[f"FOF/{fofkey}"] + name = f"InputHalos/{prop.name}" + size = prop.shape if size == 1: # Scalar quantity size = () else: # Vector quantity size = (size,) - dtype = props[2] - unit = cellgrid.get_unit(props[3]) - description = props[4] - physical = props[9] - a_exponent = props[10] + dtype = prop.dtype + unit = cellgrid.get_unit(prop.unit) + description = prop.description + physical = prop.output_physical + a_exponent = prop.a_scale_exponent if not physical: unit = unit * cellgrid.get_unit("a") ** a_exponent fof_metadata.append( (name, size, unit, dtype, description, physical, a_exponent) ) + # Keep the properties required for matching subhalos to FOFs + props_to_keep.update(("InputHalos/HBTplus/HostFOFId", "InputHalos/IsCentral")) + # First MPI rank sets up the output file with MPITimer("Creating output file", comm_world): output_file = sub_snapnum(args.output_file, args.snapshot_nr) if comm_world.Get_rank() == 0: + # Set striping on the file + lustre.setstripe(output_file) + # Create the file outfile = h5py.File(output_file, "w") @@ -153,10 +229,13 @@ def combine_chunks( if args.halo_indices is not None else np.ndarray(0, dtype=int) ) - recently_heated_gas_metadata = recently_heated_gas_filter.get_metadata() - recently_heated_gas_params = params.create_group("RecentlyHeatedGasFilter") - for at, val in recently_heated_gas_metadata.items(): - recently_heated_gas_params.attrs[at] = val + if recently_heated_gas_filter.initialised: + recently_heated_gas_metadata = recently_heated_gas_filter.get_metadata() + recently_heated_gas_params = params.create_group( + "RecentlyHeatedGasFilter" + ) + for at, val in recently_heated_gas_metadata.items(): + recently_heated_gas_params.attrs[at] = val if cold_dense_gas_filter.initialised: cold_dense_gas_params = params.create_group("ColdDenseGasFilter") for at, val in cold_dense_gas_filter.get_metadata().items(): @@ -166,6 +245,7 @@ def combine_chunks( code = outfile.create_group("Code") code.attrs["Code"] = "SOAP" code.attrs["git_hash"] = args.git_hash + code.attrs["Date"] = time.strftime("%Y-%m-%d %H:%M:%S") # Copy swift metadata params = cellgrid.copy_swift_metadata(outfile) @@ -238,16 +318,36 @@ def combine_chunks( name, size, unit, dtype, description, physical, a_exponent = metadata if description == "No description available": print(f"{name} not found in property table") + compression_metadata = { + "Lossy compression filter": "None", + "Is Compressed": False, + } + mask_metadata = category_filter.get_filter_metadata(None) + elif description.startswith("Time taken in seconds") or name in [ + "InputHalos/n_loop", + "InputHalos/n_process", + ]: + # Timing information + compression_metadata = { + "Lossy compression filter": "None", + "Is Compressed": False, + } + mask_metadata = category_filter.get_filter_metadata(None) + else: + compression_metadata = category_filter.get_compression_metadata( + name + ) + mask_metadata = category_filter.get_filter_metadata_for_property( + name + ) + shape = (total_nr_halos,) + size dataset = outfile.create_dataset( name, shape=shape, dtype=dtype, fillvalue=None ) - # Add units and description attrs = swift_units.attributes_from_units(unit, physical, a_exponent) attrs["Description"] = description - mask_metadata = category_filter.get_filter_metadata_for_property(name) attrs.update(mask_metadata) - compression_metadata = category_filter.get_compression_metadata(name) attrs.update(compression_metadata) for attr_name, attr_value in attrs.items(): dataset.attrs[attr_name] = attr_value @@ -270,31 +370,6 @@ def combine_chunks( # Reopen the output file in parallel mode outfile = h5py.File(output_file, "r+", driver="mpio", comm=comm_world) - - # Certain properties need to be kept for calculating the SOAP properties - subhalo_rank_props = { - "VR": ( - "InputHalos/VR/ID", - "BoundSubhalo/TotalMass", - "InputHalos/VR/HostHaloID", - ), - "HBTplus": ( - "InputHalos/HBTplus/HostFOFId", - "BoundSubhalo/TotalMass", - "InputHalos/HBTplus/TrackId", - ), - }.get(args.halo_format, ()) - host_halo_index_props = { - "VR": ("InputHalos/VR/ID", "InputHalos/VR/HostHaloID"), - "HBTplus": ("InputHalos/HBTplus/HostFOFId", "InputHalos/IsCentral"), - }.get(args.halo_format, ()) - fof_props = { - "HBTplus": ("InputHalos/HBTplus/HostFOFId", "InputHalos/IsCentral") - }.get(args.halo_format, ()) - props_to_keep = set((*subhalo_rank_props, *host_halo_index_props, *fof_props)) - # Also keep M200c for calculating reduced_snapshot flag - if "reduced_snapshots" in args.calculations: - props_to_keep.add("SO/200_crit/TotalMass") props_kept = {} with MPITimer("Writing output properties", comm_world): @@ -375,12 +450,12 @@ def combine_chunks( fof_com[keep] = psort.fetch_elements( fof_file.read("Groups/Centres"), indices, comm=comm_world ) - props = PropertyTable.full_property_list[f"FOF/Centres"] - soap_com_unit = cellgrid.get_unit(props[3]) - physical = props[9] - a_exponent = props[10] + prop = PropertyTable.full_property_list[f"FOF/Centres"] + soap_com_unit = cellgrid.get_unit(prop.unit) + physical = prop.output_physical + a_exponent = prop.a_scale_exponent if not physical: - soap_com_unit = soap_com_unit * cellgrid.get_unit('a') ** a_exponent + soap_com_unit = soap_com_unit * cellgrid.get_unit("a") ** a_exponent fof_com = (fof_com * fof_com_unit).to(soap_com_unit) phdf5.collective_write( outfile, @@ -394,12 +469,12 @@ def combine_chunks( fof_mass[keep] = psort.fetch_elements( fof_file.read("Groups/Masses"), indices, comm=comm_world ) - props = PropertyTable.full_property_list[f"FOF/Masses"] - soap_mass_unit = cellgrid.get_unit(props[3]) - physical = props[9] - a_exponent = props[10] + prop = PropertyTable.full_property_list[f"FOF/Masses"] + soap_mass_unit = cellgrid.get_unit(prop.unit) + physical = prop.output_physical + a_exponent = prop.a_scale_exponent if not physical: - soap_mass_unit = soap_mass_unit * cellgrid.get_unit('a') ** a_exponent + soap_mass_unit = soap_mass_unit * cellgrid.get_unit("a") ** a_exponent fof_mass = (fof_mass * fof_mass_unit).to(soap_mass_unit) phdf5.collective_write( outfile, @@ -421,43 +496,85 @@ def combine_chunks( comm=comm_world, ) + # Handle radius differently since SWIFT did not always output radius + # Assumes the FOF radii files are the same order as the other FOFs + read_radii = "InputHalos/FOF/Radii" in [m[0] for m in fof_metadata] + if read_radii: + # Open file in parallel + pf = PartialFormatter() + fof_filename = pf.format( + args.fof_radius_filename, snap_nr=args.snapshot_nr, file_nr=None + ) + fof_file = phdf5.MultiFile( + fof_filename, + file_nr_attr=("Header", "NumFilesPerSnapshot"), + comm=comm_world, + ) + + fof_radii = np.zeros(keep.shape[0], dtype=np.float32) + fof_radii[keep] = psort.fetch_elements( + fof_file.read("Groups/Radii"), indices, comm=comm_world + ) + prop = PropertyTable.full_property_list[f"FOF/Radii"] + soap_radii_unit = cellgrid.get_unit(prop.unit) + physical = prop.output_physical + a_exponent = prop.a_scale_exponent + if not physical: + soap_radii_unit = soap_radii_unit * cellgrid.get_unit("a") ** a_exponent + fof_radii = (fof_radii * fof_com_unit).to(soap_radii_unit) + phdf5.collective_write( + outfile, + "InputHalos/FOF/Radii", + fof_radii, + create_dataset=False, + comm=comm_world, + ) + else: + if comm_world.Get_rank() == 0: + print("Groups/Radii not found in FOF files") + # Calculate the index in the SOAP output of the host field halo (VR) or the central subhalo of the host FOF group (HBTplus) - if len(host_halo_index_props) > 0: + if "SOAP/HostHaloIndex" in soap_props: with MPITimer("Calculate and write host index of each satellite", comm_world): if args.halo_format == "VR": - sat_mask = props_kept["InputHalos/VR/HostHaloID"] != -1 - host_ids = props_kept["InputHalos/VR/HostHaloID"][sat_mask] - # If we run on an incomplete catalogue (e.g. for testing) some satellites will have an index == -1 - indices = psort.parallel_match( + # We don't need to worry about hostless halos for VR + host_ids = props_kept["InputHalos/VR/HostHaloID"].copy() + cen_mask = host_ids == -1 + host_ids[cen_mask] = props_kept["InputHalos/VR/ID"][cen_mask] + # If we run on an incomplete catalogue (e.g. for testing) + # some satellites might point to themselves + host_halo_index = psort.parallel_match( host_ids, props_kept["InputHalos/VR/ID"], comm=comm_world ) - host_halo_index = -1 * np.ones(sat_mask.shape[0], dtype=np.int64) - host_halo_index[sat_mask] = indices elif args.halo_format == "HBTplus": - # Create array where FOF IDs are only set for centrals, so we can match to it - cen_fof_id = props_kept["InputHalos/HBTplus/HostFOFId"].copy() + host_fof_id = props_kept["InputHalos/HBTplus/HostFOFId"] + # Create array where FOF IDs are only set for centrals + # so that we can match to it + cen_fof_id = host_fof_id.copy() sat_mask = props_kept["InputHalos/IsCentral"] == 0 cen_fof_id[sat_mask] = -1 - host_ids = props_kept["InputHalos/HBTplus/HostFOFId"][sat_mask] - # If we run on an incomplete catalogue (e.g. for testing) some satellites will have an index == -1 + # We don't want to set HostHaloIndex for hostless halos + has_host_mask = host_fof_id != -1 + host_ids = host_fof_id[has_host_mask] + # If we run on an incomplete catalogue (e.g. for testing) + # some satellites might point to themselves indices = psort.parallel_match(host_ids, cen_fof_id, comm=comm_world) host_halo_index = -1 * np.ones(sat_mask.shape[0], dtype=np.int64) - host_halo_index[sat_mask] = indices + host_halo_index[has_host_mask] = indices + + phdf5.collective_write( + outfile, + "SOAP/HostHaloIndex", + host_halo_index, + create_dataset=False, + comm=comm_world, + ) else: - # Set default value - host_halo_index = -1 * np.ones(order.shape[0], dtype=np.int64) if comm_world.Get_rank() == 0: print("Not calculating host halo index") - phdf5.collective_write( - outfile, - "SOAP/HostHaloIndex", - host_halo_index, - create_dataset=False, - comm=comm_world, - ) # Write out subhalo ranking by mass within host halos, if we have all the required quantities. - if len(subhalo_rank_props) > 0: + if "SOAP/SubhaloRankByBoundMass" in soap_props: with MPITimer("Calculate and write subhalo ranking by mass", comm_world): if args.halo_format == "VR": # Set field halos to be their own host (VR sets hostid=-1 in this case) @@ -474,23 +591,19 @@ def combine_chunks( subhalo_rank = compute_subhalo_rank( host_id, props_kept["BoundSubhalo/TotalMass"], comm_world ) + phdf5.collective_write( + outfile, + "SOAP/SubhaloRankByBoundMass", + subhalo_rank, + create_dataset=False, + comm=comm_world, + ) else: - # Set default value - subhalo_rank = -1 * np.ones(order.shape[0], dtype=np.int32) if comm_world.Get_rank() == 0: print("Not calculating subhalo ranking by mass") - phdf5.collective_write( - outfile, - "SOAP/SubhaloRankByBoundMass", - subhalo_rank, - create_dataset=False, - comm=comm_world, - ) # Determine which objects should be saved in the reduced snapshot files - if ("reduced_snapshots" in args.calculations) and ( - "SO/200_crit/TotalMass" in props_kept - ): + if "SOAP/IncludedInReducedSnapshot" in soap_props: with MPITimer("Calculate and write reduced snapshot membership", comm_world): # Load parameters. We create mass bins with the lower limit of the smallest mass bin # given by "min_halo_mass". The size of the bins is set by "halo_bin_size_dex". @@ -548,18 +661,77 @@ def combine_chunks( assert n_keep[i_bin] <= np.sum(mask) keep_idx = np.random.choice(idx, size=n_keep[i_bin], replace=False) reduced_snapshot[keep_idx] = 1 + phdf5.collective_write( + outfile, + "SOAP/IncludedInReducedSnapshot", + reduced_snapshot, + create_dataset=False, + comm=comm_world, + ) else: - # Set default value - reduced_snapshot = np.zeros(order.shape[0], dtype=np.int32) if comm_world.Get_rank() == 0: print("Not calculating reduced snapshot membership") - phdf5.collective_write( - outfile, - "SOAP/IncludedInReducedSnapshot", - reduced_snapshot, - create_dataset=False, - comm=comm_world, - ) + + if "SOAP/ProgenitorIndex" in soap_props: + assert "SOAP/DescendantIndex" in soap_props + assert args.halo_format in ["HBTplus"] + + for name, snap_nr in [ + ("Progenitor", args.snapshot_nr - 1), + ("Descendant", args.snapshot_nr + 1), + ]: + + # Set the default value + prev_index = -1 * np.ones(order.shape[0], dtype=np.int32) + + # Check if the previous/next HBT catalogue is available + prev_basename = args.halo_basename.format(snap_nr=snap_nr) + if comm_world.Get_rank() == 0: + # Check for sorted and unsorted catalogues + prev_filename = prev_basename + ".0.hdf5" + if os.path.exists(prev_basename) or os.path.exists(prev_filename): + calculate_prev_index = True + else: + print(f"Can't find halo catalogues for calculating {name}Index") + calculate_prev_index = False + else: + calculate_prev_index = None + calculate_prev_index = comm_world.bcast(calculate_prev_index) + + if calculate_prev_index: + track_id = props_kept["InputHalos/HBTplus/TrackId"] + + # Load data from previous/next halo catalogue and sort it + # This assumes the metadata in the previous/next snapshot + # is the same as in the current snapshot + prev_data = read_hbtplus.read_hbtplus_catalogue( + comm_world, + prev_basename, + cellgrid.a_unit, + cellgrid.snap_unit_registry, + cellgrid.boxsize, + ) + prev_order, _ = spatial_sort( + prev_data["cofp"], + prev_data["index"], + cellgrid, + comm_world, + ) + prev_track_id = psort.fetch_elements( + prev_data["TrackId"], prev_order, comm=comm_world + ) + # Find where each TrackId appears in the previous/next snapshot + prev_index = psort.parallel_match( + track_id, prev_track_id, comm=comm_world + ) + + phdf5.collective_write( + outfile, + f"SOAP/{name}Index", + prev_index, + create_dataset=False, + comm=comm_world, + ) # Done. outfile.close() diff --git a/dataset_names.py b/SOAP/core/dataset_names.py similarity index 100% rename from dataset_names.py rename to SOAP/core/dataset_names.py diff --git a/domain_decomposition.py b/SOAP/core/domain_decomposition.py similarity index 95% rename from domain_decomposition.py rename to SOAP/core/domain_decomposition.py index e51836ce..d20f64d0 100644 --- a/domain_decomposition.py +++ b/SOAP/core/domain_decomposition.py @@ -17,7 +17,7 @@ def peano_decomposition(boxsize, local_halo, nr_chunks, comm): Sorts halos by chunk index and returns the number of halos in each chunk. local_halo is a dict of distributed unyt arrays with the halo properties. - + Will not work well for zoom simulations. Could use a grid which just covers the zoom region? """ @@ -27,9 +27,9 @@ def peano_decomposition(boxsize, local_halo, nr_chunks, comm): # Find size of grid to use to calculate PH keys centres = local_halo["cofp"] bits_per_dimension = 10 - cells_per_dimension = 2 ** bits_per_dimension + cells_per_dimension = 2**bits_per_dimension grid_size = boxsize / cells_per_dimension - nr_cells = cells_per_dimension ** 3 + nr_cells = cells_per_dimension**3 nr_halos = centres.shape[0] # number of halos on this rank total_nr_halos = comm.allreduce(nr_halos) # number on all ranks diff --git a/halo_centres.py b/SOAP/core/halo_centres.py similarity index 80% rename from halo_centres.py rename to SOAP/core/halo_centres.py index 7b88e269..08e606f6 100644 --- a/halo_centres.py +++ b/SOAP/core/halo_centres.py @@ -12,31 +12,28 @@ import virgo.mpi.gather_array as g import virgo.mpi.parallel_sort as psort -import domain_decomposition -import read_vr -import read_hbtplus -import read_subfind -import read_rockstar - -from mpi_tags import HALO_REQUEST_TAG, HALO_RESPONSE_TAG -from sleepy_recv import sleepy_recv +from SOAP.catalogue_readers import ( + read_vr, + read_hbtplus, + read_subfind, + read_subfind_eagle, + read_rockstar, +) +from . import domain_decomposition +from .combine_chunks import sub_snapnum +from .mpi_tags import HALO_REQUEST_TAG, HALO_RESPONSE_TAG +from .sleepy_recv import sleepy_recv class SOCatalogue: def __init__( self, comm, - halo_basename, - halo_format, a_unit, registry, boxsize, - max_halos, - centrals_only, - halo_indices, halo_prop_list, - nr_chunks, - min_read_radius_cmpc, + args, ): """ This reads in the halo catalogue and stores the halo properties in a @@ -75,24 +72,29 @@ def __init__( "nr_bound_part", "nr_unbound_part", ) - if halo_format == "VR": + halo_basename = sub_snapnum(args.halo_basename, args.snapshot_nr) + if args.halo_format == "VR": halo_data = read_vr.read_vr_catalogue( comm, halo_basename, a_unit, registry, boxsize ) - elif halo_format == "HBTplus": + elif args.halo_format == "HBTplus": halo_data = read_hbtplus.read_hbtplus_catalogue( comm, halo_basename, a_unit, registry, boxsize ) - elif halo_format == "Subfind": + elif args.halo_format == "Subfind": halo_data = read_subfind.read_gadget4_catalogue( comm, halo_basename, a_unit, registry, boxsize ) - elif halo_format == "Rockstar": + elif args.halo_format == "SubfindEagle": + halo_data = read_subfind_eagle.read_subfind_catalogue( + comm, halo_basename, a_unit, registry, boxsize + ) + elif args.halo_format == "Rockstar": halo_data = read_rockstar.read_rockstar_catalogue( comm, halo_basename, a_unit, registry, boxsize ) else: - raise RuntimeError(f"Halo format {format} not recognised!") + raise RuntimeError(f"Halo format {args.halo_format} not recognised!") # Add halo finder prefix to halo finder specific quantities: # This in case different finders use the same property names. @@ -101,12 +103,12 @@ def __init__( if name in common_props: local_halo[name] = halo_data[name] else: - local_halo[f"{halo_format}/{name}"] = halo_data[name] + local_halo[f"{args.halo_format}/{name}"] = halo_data[name] del halo_data # Only keep halos in the supplied list of halo IDs. - if (halo_indices is not None) and (local_halo["index"].shape[0]): - halo_indices = np.asarray(halo_indices, dtype=np.int64) + if (args.halo_indices is not None) and (local_halo["index"].shape[0]): + halo_indices = np.asarray(args.halo_indices, dtype=np.int64) keep = np.zeros_like(local_halo["index"], dtype=bool) matching_index = virgo.util.match.match(halo_indices, local_halo["index"]) have_match = matching_index >= 0 @@ -115,16 +117,16 @@ def __init__( local_halo[name] = local_halo[name][keep, ...] # Discard satellites, if necessary - if centrals_only: + if args.centrals_only: keep = local_halo["is_central"] == 1 for name in local_halo: local_halo[name] = local_halo[name][keep, ...] # For testing: limit number of halos processed - if max_halos > 0: + if args.max_halos > 0: nr_halos_local = len(local_halo["index"]) nr_halos_prev = comm.scan(nr_halos_local) - nr_halos_local - nr_keep_local = max_halos - nr_halos_prev + nr_keep_local = args.max_halos - nr_halos_prev if nr_keep_local < 0: nr_keep_local = 0 if nr_keep_local > nr_halos_local: @@ -150,7 +152,7 @@ def __init__( comm.Abort(1) # Reduce the number of chunks if necessary so that all chunks have at least one halo - nr_chunks = min(nr_chunks, self.nr_halos) + nr_chunks = min(args.chunks, self.nr_halos) self.nr_chunks = nr_chunks # Assign halos to chunk tasks: @@ -161,7 +163,7 @@ def __init__( # Compute initial radius to read in about each halo local_halo["read_radius"] = local_halo["search_radius"].copy() - min_radius = min_read_radius_cmpc * swift_cmpc + min_radius = args.min_read_radius_cmpc * swift_cmpc local_halo["read_radius"] = local_halo["read_radius"].clip(min=min_radius) # Find minimum physical radius to read in @@ -177,6 +179,43 @@ def __init__( physical_radius_mpc, units=swift_pmpc ) + if args.record_halo_timings: + # Total amount of time spent processing this halo + local_halo["process_time"] = unyt.unyt_array( + np.zeros(local_halo["index"].shape[0], dtype=np.float32), + units=unyt.dimensionless, + registry=registry, + ) + # Number of loops before target density was reached + local_halo["n_loop"] = unyt.unyt_array( + np.zeros(local_halo["index"].shape[0], dtype=np.float32), + units=unyt.dimensionless, + registry=registry, + ) + # Number of times this halo was processed (a halo will have to be + # reprocessed if it's target density is not reached with the region + # currently loaded in memory) + local_halo["n_process"] = unyt.unyt_array( + np.zeros(local_halo["index"].shape[0], dtype=np.float32), + units=unyt.dimensionless, + registry=registry, + ) + for halo_prop in halo_prop_list: + # Total time taken for this halo prop + local_halo[f"{halo_prop.name}_total_time"] = unyt.unyt_array( + np.zeros(local_halo["index"].shape[0], dtype=np.float32), + units=unyt.dimensionless, + registry=registry, + ) + # Time taken the final time we calculated this halo_prop. This is needed + # certain halo_prop can throw exceptions which require them to be reprocessed, + # and so we want to know how often that happens + local_halo[f"{halo_prop.name}_final_time"] = unyt.unyt_array( + np.zeros(local_halo["index"].shape[0], dtype=np.float32), + units=unyt.dimensionless, + registry=registry, + ) + # Ensure that both the initial search radius and the radius to read in # are >= the minimum physical radius required by property calculations local_halo["read_radius"] = local_halo["read_radius"].clip( diff --git a/halo_tasks.py b/SOAP/core/halo_tasks.py similarity index 68% rename from halo_tasks.py rename to SOAP/core/halo_tasks.py index 465aafcf..53f91598 100644 --- a/halo_tasks.py +++ b/SOAP/core/halo_tasks.py @@ -5,10 +5,10 @@ import numpy as np import unyt -from dataset_names import mass_dataset, ptypes_for_so_masses -from halo_properties import SearchRadiusTooSmallError -import shared_array -from property_table import PropertyTable +from SOAP.core import memory_use, shared_array +from SOAP.core.dataset_names import mass_dataset, ptypes_for_so_masses +from SOAP.particle_selection.halo_properties import SearchRadiusTooSmallError +from SOAP.property_table import PropertyTable # Factor by which to increase search radius when looking for density threshold @@ -34,13 +34,14 @@ def process_single_halo( ): """ This computes properties for one halo and runs on a single - MPI rank. Result is a dict of properties of the form + MPI rank. The first output is a dict of properties of the form halo_result[property_name] = (unyt_array, description) where the property_name will be used as the HDF5 dataset name in the output and the units of the unyt_array determine the unit - attributes. + attributes. The second output contains information about how long + it took to process the halo. Two radii are passed in: @@ -50,13 +51,13 @@ def process_single_halo( the density within read_radius is above the threshold, then we didn't read in a large enough region. - Returns None if we need to try again with a larger region. + Returns halo_result=None if we need to try again with a larger region. """ swift_mpc = unyt.Unit("swift_mpc", registry=unit_registry) snap_length = unyt.Unit("snap_length", registry=unit_registry) snap_mass = unyt.Unit("snap_mass", registry=unit_registry) - snap_density = snap_mass / (snap_length ** 3) + snap_density = snap_mass / (snap_length**3) # Record which calculations are still to do for this halo halo_prop_done = np.zeros(len(halo_prop_list), dtype=bool) @@ -64,9 +65,15 @@ def process_single_halo( # Dict to store the results halo_result = {} + # Dict to store timing information for this iteration of + # attempting to process this halo + t0_halo = time.time() + timings = {"n_process": 1} + # Loop until we fall below the required density current_radius = input_halo["search_radius"] while True: + timings["n_loop"] = timings.get("n_loop", 0) + 1 # Sanity checks on the radius assert current_radius <= input_halo["read_radius"] @@ -88,7 +95,7 @@ def process_single_halo( mass_total += np.sum(mass.full[idx[ptype]], dtype=float) # Find mean density in the search radius - density = mass_total / (4.0 / 3.0 * np.pi * current_radius ** 3) + density = mass_total / (4.0 / 3.0 * np.pi * current_radius**3) # If we've reached the target density, we can try to compute halo properties max_physical_radius_mpc = ( @@ -116,6 +123,7 @@ def process_single_halo( # Already have the result for this one continue try: + t0_halo_prop = time.time() halo_prop.calculate( input_halo, current_radius, particle_data, halo_result ) @@ -124,6 +132,11 @@ def process_single_halo( max_physical_radius_mpc = max( max_physical_radius_mpc, halo_prop.physical_radius_mpc ) + timings[f"{halo_prop.name}_total_time"] = ( + timings.get(f"{halo_prop.name}_total_time", 0) + + time.time() + - t0_halo_prop + ) break except Exception as e: # Calculation caused an unexpected error. @@ -135,6 +148,17 @@ def process_single_halo( else: # The property calculation worked! halo_prop_done[prop_nr] = True + # The total_time value stored in timings will include all previous failed attempts to + # calculate this halo prop, so we also store how long the successful attempt took + timings[f"{halo_prop.name}_total_time"] = ( + timings.get(f"{halo_prop.name}_total_time", 0) + + time.time() + - t0_halo_prop + ) + if f"{halo_prop.name}_final_time" in input_halo: + input_halo[f"{halo_prop.name}_final_time"] += ( + time.time() - t0_halo_prop + ) # If we computed all of the properties, we're done with this halo if np.all(halo_prop_done): @@ -148,12 +172,14 @@ def process_single_halo( # which we read in, so we can't process the halo on this iteration regardless # of the current radius. input_halo["search_radius"] = max(search_radius, required_radius) - return None + timings["process_time"] = time.time() - t0_halo + return None, timings elif current_radius >= input_halo["read_radius"]: # The current radius has exceeded the region read in. Will need to redo this # halo using current_radius as the starting point for the next iteration. input_halo["search_radius"] = max(search_radius, current_radius) - return None + timings["process_time"] = time.time() - t0_halo + return None, timings else: # We still have a large enough region in memory that we can try a larger radius current_radius = min( @@ -161,30 +187,64 @@ def process_single_halo( ) current_radius = max(current_radius, required_radius) - # In case we're not doing any calculations with a target density - if target_density is None: - target_density = density * 0.0 + # Store timings + for k in timings: + if k in input_halo: + input_halo[k] += timings[k] + if "process_time" in input_halo: + input_halo["process_time"] += time.time() - t0_halo # Store input halo quantites for name in input_halo: - if name not in ("done", "task_id", "read_radius", "search_radius"): + # Skip internal properties we don't need to output + if name in ("done", "task_id", "read_radius", "search_radius"): + continue + + # Timing information + if ("_time" in name) or (name in ["n_loop", "n_process"]): + dataset_name = f"InputHalos/{name}" + arr = input_halo[name] + physical = True + a_exponent = None + if "_total_time" in name: + description = ( + f"Time taken in seconds spent on {name.replace('_total_time', '')}" + ) + elif "_final_time" in name: + description = ( + f"Time taken in seconds spent on {name.replace('_final_time', '')}" + "the final time it was calculated" + ) + else: + description = { + "process_time": "Time taken in seconds in total processing this halo", + "n_loop": "Number of loops before target density was reached", + "n_process": ( + "Number of times this halo was processed (a halo " + "will have to be reprocessed if it's target density " + "is not reached with the region currently loaded " + "in memory)" + ), + }[name] + + # Check the property table + else: try: prop = PropertyTable.full_property_list[name] - # Don't remove halo finder prefix - # e.g. don't want "VR/ID" replaced with "ID" + dataset_name = prop.name + # We want to store halo finder properties within the InputHalos group + # Identify them using the fact that they have a prefix if "/" in name: - group = name.split("/")[0] - dataset_name = f"{group}/{prop[0]}" - else: - dataset_name = prop[0] - dtype = prop[2] - unit = unyt.Unit(prop[3], registry=unit_registry) - description = prop[4] - physical = prop[9] - a_exponent = prop[10] + dataset_name = f"InputHalos/{dataset_name}" + + dtype = prop.dtype + unit = unyt.Unit(prop.unit, registry=unit_registry) + description = prop.description + physical = prop.output_physical + a_exponent = prop.a_scale_exponent if not physical: unit = unit * unyt.Unit("a", registry=unit_registry) ** a_exponent - # unyt_array.to outputs a float64 array, which is dangerous for integers + # unyt_array.to() outputs a float64 array, which is dangerous for integers # so don't allow this to happen if np.issubdtype(input_halo[name].dtype, np.integer) or np.issubdtype( dtype, np.integer @@ -193,22 +253,25 @@ def process_single_halo( assert input_halo[name].units == unit else: arr = input_halo[name].to(unit).astype(dtype) - # Property not present in PropertyTable. We log this fact to the output - # within combine_chunks, rather than here. + + # This property not present in PropertyTable. We log this fact + # to stdout during combine_chunks, rather than doing it here. except KeyError: - dataset_name = name + dataset_name = f"InputHalos/{name}" arr = input_halo[name] description = "No description available" physical = True a_exponent = None - halo_result[f"InputHalos/{dataset_name}"] = ( - arr, - description, - physical, - a_exponent, - ) - return halo_result + # Store the value + halo_result[dataset_name] = ( + arr, + description, + physical, + a_exponent, + ) + + return halo_result, None def process_halos( @@ -267,6 +330,7 @@ def process_halos( # Start the clock comm.barrier() t0_all = time.time() + min_free_mem_gb = float("inf") # Count halos to do nr_halos_left = comm.allreduce(np.sum(halo_arrays["done"].local.value == 0)) @@ -278,6 +342,12 @@ def process_halos( task_time = 0.0 while True: + # Check memory usage + if comm.Get_rank() == 0: + _, free_mem_gb = memory_use.get_memory_use() + if free_mem_gb is not None: + min_free_mem_gb = min(min_free_mem_gb, free_mem_gb) + # Get a task by atomic incrementing the counter. Don't know how to do # an atomic fetch and add in python, so will use MPI RMA calls! task_to_do = np.ndarray(1, dtype=np.int64) @@ -294,13 +364,13 @@ def process_halos( # Skip halos we already did if halo_arrays["done"].full[task_to_do].value == 0: - # Extract this halo's VR information (centre, radius, index etc) + # Extract the halofinder information for this object (centre, radius, index etc) input_halo = {} for name in halo_arrays: input_halo[name] = halo_arrays[name].full[task_to_do, ...].copy() # Fetch the results for this particular halo - halo_result = process_single_halo( + halo_result, timings = process_single_halo( mesh, unit_registry, data, @@ -309,13 +379,14 @@ def process_halos( mean_density, boxsize, input_halo, - target_density if input_halo['is_central'] == 1 else None, + target_density if input_halo["is_central"] == 1 else None, ) if halo_result is not None: # Store results and flag this halo as done results.append(halo_result) nr_done_this_rank += 1 halo_arrays["done"].full[task_to_do] = 1 + # No need to store timing information, it is contained in halo_result else: # We didn't read in a large enough region. Update the shared radius # arrays so that we read a larger region next time and start the @@ -330,9 +401,12 @@ def process_halos( halo_arrays["search_radius"].full[task_to_do] = input_halo[ "search_radius" ] + # Store the timing information for the next rank that picks up this halo + for k in timings: + if k in input_halo: + halo_arrays[k].full[task_to_do] += timings[k] - t1_task = time.time() - task_time += t1_task - t0_task + task_time += time.time() - t0_task else: # We ran out of halos to do break @@ -348,4 +422,10 @@ def process_halos( comm.barrier() t1_all = time.time() - return t1_all - t0_all, task_time, nr_halos_left, comm.allreduce(nr_done_this_rank) + return ( + t1_all - t0_all, + task_time, + nr_halos_left, + comm.allreduce(nr_done_this_rank), + min_free_mem_gb, + ) diff --git a/lazy_properties.py b/SOAP/core/lazy_properties.py similarity index 100% rename from lazy_properties.py rename to SOAP/core/lazy_properties.py diff --git a/SOAP/core/lustre.py b/SOAP/core/lustre.py new file mode 100644 index 00000000..70feaccd --- /dev/null +++ b/SOAP/core/lustre.py @@ -0,0 +1,34 @@ +#!/bin/env python + +import subprocess +import os + + +def setstripe(filename, stripe_size=32, stripe_count=32): + """ + Try to set Lustre striping on a file + """ + # Remove file if it already exists (or lfs will error) + try: + os.remove(filename) + except FileNotFoundError: + pass + + # Only set striping for snap8 on cosma + if not filename.startswith("/snap8/scratch"): + print(f"Not setting lustre striping on {filename}") + return + + args = [ + "lfs", + "setstripe", + f"--stripe-count={stripe_count}", + f"--stripe-size={stripe_size}M", + filename, + ] + try: + subprocess.run(args) + except (FileNotFoundError, subprocess.CalledProcessError): + # if the 'lfs' command is not available, this will generate a + # FileNotFoundError + print(f"WARNING: failed to set lustre striping on {filename}") diff --git a/mask_cells.py b/SOAP/core/mask_cells.py similarity index 100% rename from mask_cells.py rename to SOAP/core/mask_cells.py diff --git a/memory_use.py b/SOAP/core/memory_use.py similarity index 95% rename from memory_use.py rename to SOAP/core/memory_use.py index be94b1dc..22943f53 100644 --- a/memory_use.py +++ b/SOAP/core/memory_use.py @@ -15,7 +15,7 @@ def get_memory_use(): if psutil is None: return None, None - GB = 1024 ** 3 + GB = 1024**3 mem = psutil.virtual_memory() total_mem_gb = mem.total / GB diff --git a/mpi_tags.py b/SOAP/core/mpi_tags.py similarity index 100% rename from mpi_tags.py rename to SOAP/core/mpi_tags.py diff --git a/mpi_timer.py b/SOAP/core/mpi_timer.py similarity index 100% rename from mpi_timer.py rename to SOAP/core/mpi_timer.py diff --git a/parameter_file.py b/SOAP/core/parameter_file.py similarity index 56% rename from parameter_file.py rename to SOAP/core/parameter_file.py index 9c9536eb..909a9b39 100644 --- a/parameter_file.py +++ b/SOAP/core/parameter_file.py @@ -13,7 +13,7 @@ from typing import Dict, Union, List, Tuple import yaml -import property_table +from SOAP import property_table class ParameterFile: @@ -26,6 +26,9 @@ class ParameterFile: # parameter dictionary parameters: Dict + # Whether to record timings for calculations + record_property_timings: bool = False + def __init__( self, file_name: Union[None, str] = None, @@ -58,6 +61,9 @@ def __init__( self.parameters = {} self.snipshot = snipshot + self.aliases = None + + self.property_filters = {} def get_parameters(self) -> Dict: """ @@ -76,19 +82,15 @@ def write_parameters(self, file_name: str = "SOAP.used_parameters.yml"): with open(file_name, "w") as handle: yaml.safe_dump(self.parameters, handle) - def get_property_mask(self, halo_type: str, full_list: List[str]) -> Dict: + def get_property_filters(self, base_halo_type: str, full_list: List[str]) -> Dict: """ - Get a dictionary with True/False values indicating which properties - should actually be computed for the given halo type. The dictionary - keys are based on the contents of the given list of properties. If - a property in the list is missing from the parameter file, it is - assumed that this property needs to be calculated. - - Note that we currently do not check for properties in the parameter - file that are not in the list. + Get a dictionary with the filter that should be applied to each + property for the given halo type. If a property should be not be + computed for this halo type then False is return. The dictionary + keys are based on the contents of the given list of properties. Parameters: - - halo_type: str + - base_halo_type: str Halo type identifier in the parameter file, can be one of ApertureProperties, ProjectedApertureProperties, SOProperties or SubhaloProperties. @@ -97,41 +99,59 @@ def get_property_mask(self, halo_type: str, full_list: List[str]) -> Dict: particular halo type (as defined in the corresponding HaloProperty specialisation). - Returns a dictionary with True or False for each property in full_list. + Returns a dictionary where the keys are each property in full_list. The + values are either False (if the property should not be calculated) or a + string (the name of the filter to apply to the property). """ - if not halo_type in self.parameters: - self.parameters[halo_type] = {} - if not "properties" in self.parameters[halo_type]: - self.parameters[halo_type]["properties"] = {} + # Save the filters as they are needed in combine chunks + self.property_filters[base_halo_type] = self.property_filters.get( + base_halo_type, {} + ) + + if not base_halo_type in self.parameters: + self.parameters[base_halo_type] = {} + # Handle the case where no properties are listed for the halo type + if not "properties" in self.parameters[base_halo_type]: + self.parameters[base_halo_type]["properties"] = {} for property in full_list: - self.parameters[halo_type]["properties"][ + self.parameters[base_halo_type]["properties"][ property ] = self.calculate_missing_properties() - mask = {} + filters = {} for property in full_list: - # Property is listed in the parameter file for this halo_type - if property in self.parameters[halo_type]["properties"]: - should_calculate = self.parameters[halo_type]["properties"][property] - # should_calculate is a dict if we want different behaviour for snapshots/snipshots - if isinstance(should_calculate, dict): + # Check if property is listed in the parameter file for this base_halo_type + if property in self.parameters[base_halo_type]["properties"]: + filter_name = self.parameters[base_halo_type]["properties"][property] + # filter_name will a dict if we want different behaviour + # for snapshots/snipshots + if isinstance(filter_name, dict): if self.snipshot: - mask[property] = should_calculate["snipshot"] + filter_name = filter_name["snipshot"] else: - mask[property] = should_calculate["snapshot"] - # otherwise should_calculate is a bool - else: - mask[property] = should_calculate - # Property is not listed in the parameter file for this halo_type + filter_name = filter_name["snapshot"] + # if a filter is not specified in the snapshots + # then we default to "basic" + if filter_name == True: + filter_name = "basic" + filters[property] = filter_name + # Property is not listed in the parameter file for this base_halo_type else: if self.calculate_missing_properties(): - mask[property] = True - self.parameters[halo_type]["properties"][property] = True + filters[property] = "basic" + self.parameters[base_halo_type]["properties"][property] = "basic" if self.unregistered_parameters is not None: - self.unregistered_parameters.add((halo_type, property)) + self.unregistered_parameters.add((base_halo_type, property)) else: - mask[property] = False - assert isinstance(mask[property], bool) - return mask + filters[property] = False + if isinstance(filters[property], str): + assert (filters[property] in self.parameters.get("filters", {})) or ( + filters[property] == "basic" + ), f'Filter "{filters[property]}" is not defined in paramter file' + else: + assert filters[property] == False + + self.property_filters[base_halo_type][property] = filters[property] + return filters def print_unregistered_properties(self) -> None: """ @@ -145,34 +165,43 @@ def print_unregistered_properties(self) -> None: print( "The following properties were not found in the parameter file, but will be calculated:" ) - for halo_type, property in self.unregistered_parameters: - print(f" {halo_type.ljust(30)}{property}") + for base_halo_type, property in self.unregistered_parameters: + print(f" {base_halo_type.ljust(30)}{property}") - def print_invalid_properties(self) -> None: + def print_invalid_properties(self, halo_prop_list) -> None: """ Print a list of any properties in the parameter file that are not present in the property table. This doesn't check if the property is defined for a specific halo type. """ - invalid_properties = [] - full_property_list = property_table.PropertyTable.full_property_list - valid_properties = [prop[0] for prop in full_property_list.values()] + invalid_properties = set() for key in self.parameters: # Skip keys which aren't halo types if "properties" not in self.parameters[key]: continue + # Add all properties to the invalid list for prop in self.parameters[key]["properties"]: - if prop not in valid_properties: - invalid_properties.append(prop) + invalid_properties.add((key, prop)) + # Remove those which are valid for this particle halo type + for halo_type in halo_prop_list: + if key != halo_type.base_halo_type: + continue + valid_properties = [ + prop.name for prop in halo_type.property_list.values() + ] + for prop in self.parameters[key]["properties"]: + if prop in valid_properties: + invalid_properties.discard((key, prop)) if len(invalid_properties): + invalid_properties = sorted(invalid_properties, key=lambda x: (x[0], x[1])) print( "The following properties were found in the parameter file, but are invalid:" ) - for prop in invalid_properties: - print(f" {prop}") + for base_halo_type, prop in invalid_properties: + print(f" {base_halo_type} {prop}") def get_halo_type_variations( - self, halo_type: str, default_variations: Dict + self, base_halo_type: str, default_variations: Dict ) -> Dict: """ Get a dictionary of variations for the given halo type. @@ -184,7 +213,7 @@ def get_halo_type_variations( variations are specified, the default variations are used. Parameters: - - halo_type: str + - base_halo_type: str Halo type identifier in the parameter file, can be one of ApertureProperties, ProjectedApertureProperties, SOProperties or SubhaloProperties. @@ -195,15 +224,15 @@ def get_halo_type_variations( Returns a dictionary from which different versions of the corresponding HaloProperty specialisation can be constructed. """ - if not halo_type in self.parameters: - self.parameters[halo_type] = {} - if not "variations" in self.parameters[halo_type]: - self.parameters[halo_type]["variations"] = {} + if not base_halo_type in self.parameters: + self.parameters[base_halo_type] = {} + if not "variations" in self.parameters[base_halo_type]: + self.parameters[base_halo_type]["variations"] = {} for variation in default_variations: - self.parameters[halo_type]["variations"][variation] = dict( + self.parameters[base_halo_type]["variations"][variation] = dict( default_variations[variation] ) - return dict(self.parameters[halo_type]["variations"]) + return dict(self.parameters[base_halo_type]["variations"]) def get_particle_property(self, property_name: str) -> Tuple[str, str]: """ @@ -223,9 +252,9 @@ def get_particle_property(self, property_name: str) -> Tuple[str, str]: Returns a tuple with the path of the actual dataset in the snapshot, e.g. ("PartType4", "Masses"). """ - if "aliases" in self.parameters: - if property_name in self.parameters["aliases"]: - property_name = self.parameters["aliases"][property_name] + aliases = self.get_aliases() + if property_name in aliases: + property_name = aliases[property_name] parts = property_name.split("/") if not len(parts) == 2: raise RuntimeError( @@ -240,10 +269,20 @@ def get_aliases(self) -> Dict: Returns the dictionary of aliases or an empty dictionary if no aliases were defined (there are no default aliases). """ - if "aliases" in self.parameters: - return dict(self.parameters["aliases"]) - else: - return dict() + if self.aliases is None: + if "aliases" in self.parameters: + if "snipshot" in self.parameters["aliases"]: + if self.snipshot: + self.aliases = dict(self.parameters["aliases"]["snipshot"]) + else: + aliases = dict(self.parameters["aliases"]) + del aliases["snipshot"] + self.aliases = aliases + else: + self.aliases = dict(self.parameters["aliases"]) + else: + self.aliases = dict() + return self.aliases def get_filters(self, default_filters: Dict) -> Dict: """ @@ -289,3 +328,41 @@ def calculate_missing_properties(self) -> bool: """ calculations = self.parameters.get("calculations", {}) return calculations.get("calculate_missing_properties", True) + + def strict_halo_copy(self) -> bool: + """ + Returns a bool indicating if approximate properties should be copied + over from small ExclusiveSphere/ProjectedApertures. Defaults to false + """ + calculations = self.parameters.get("calculations", {}) + return calculations.get("strict_halo_copy", False) + + def renclose_enabled(self) -> bool: + """ + Returns a bool indicating if BoundSubhalo/EncloseRadius is enabled + """ + return self.parameters["SubhaloProperties"]["properties"].get( + "EncloseRadius", False + ) + + def get_cold_dense_params(self) -> Dict: + """ + Returns a dict of the parameters required for initialising + the ColdDenseGasFilter object + """ + + try: + raw_params = self.parameters["calculations"]["cold_dense_gas_filter"] + return { + "maximum_temperature_K": float(raw_params["maximum_temperature_K"]), + "minimum_hydrogen_number_density_cm3": float( + raw_params["minimum_hydrogen_number_density_cm3"] + ), + "initialised": True, + } + except KeyError as e: + return { + "maximum_temperature_K": 0, + "minimum_hydrogen_number_density_cm3": 0, + "initialised": False, + } diff --git a/result_set.py b/SOAP/core/result_set.py similarity index 89% rename from result_set.py rename to SOAP/core/result_set.py index cc83a4c7..1261deda 100644 --- a/result_set.py +++ b/SOAP/core/result_set.py @@ -1,16 +1,15 @@ #!/bin/env python +import os + import h5py import numpy as np +from mpi4py import MPI import unyt - import virgo.mpi.parallel_hdf5 as phdf5 import virgo.mpi.parallel_sort as psort -import create_groups -import swift_units - -from mpi4py import MPI +from . import swift_units def concatenate(result_sets): @@ -120,9 +119,9 @@ def append(self, results): ] # Find the array to store this result - result_array, result_description, result_physical, result_a_exponent = self.result_arrays[ - result_name - ] + result_array, result_description, result_physical, result_a_exponent = ( + self.result_arrays[result_name] + ) # Consistency check: data type, units and shape should match the existing array if result_data.units != result_array.units: @@ -220,6 +219,27 @@ def parallel_sort(self, key, comm): ) del idx + @staticmethod + def find_groups_to_create(paths): + """ + Given a list of paths to HDF5 objects, return a list of the names + of the groups which must be created in the order in which to create + them. + """ + + groups_to_create = set() + for path in paths: + dirname = path + while True: + dirname = os.path.dirname(dirname) + if len(dirname) > 0: + groups_to_create.add(dirname) + else: + break + groups_to_create = list(groups_to_create) + groups_to_create.sort(key=lambda x: len(x.split("/"))) + return groups_to_create + def collective_write(self, outfile, comm): """ Write the results to a file in collective mode @@ -232,7 +252,7 @@ def collective_write(self, outfile, comm): names = comm.bcast(list(self.result_arrays.keys())) # Ensure any HDF5 groups we need exist - group_names = comm.bcast(create_groups.find_groups_to_create(names)) + group_names = comm.bcast(self.find_groups_to_create(names)) for group_name in group_names: outfile.create_group(group_name) @@ -303,27 +323,28 @@ def get_metadata(self, comm): else: return None + def get_metadata_from_chunk_file(filename, halo_prop_list, reg): metadata = [] - with h5py.File(filename, 'r') as file: + with h5py.File(filename, "r") as file: # Get the names of the datasets datasets = [] for halo_prop in halo_prop_list: - if 'ProjectedAperture' in halo_prop.group_name: + if "ProjectedAperture" in halo_prop.group_name: for proj_name in ["projx", "projy", "projz"]: - group_name = f'{halo_prop.group_name}/{proj_name}' + group_name = f"{halo_prop.group_name}/{proj_name}" for key in file[group_name]: - name = f'{group_name}/{key}' + name = f"{group_name}/{key}" datasets.append(name) else: for key in file[halo_prop.group_name]: - name = f'{halo_prop.group_name}/{key}' + name = f"{halo_prop.group_name}/{key}" datasets.append(name) - for key in file['InputHalos']: - name = f'InputHalos/{key}' + for key in file["InputHalos"]: + name = f"InputHalos/{key}" if isinstance(file[name], h5py.Group): for group_key in file[name]: - datasets.append(f'{name}/{group_key}') + datasets.append(f"{name}/{group_key}") else: datasets.append(name) @@ -335,10 +356,10 @@ def get_metadata_from_chunk_file(filename, halo_prop_list, reg): size = (file[name].shape[1],) unit = swift_units.units_from_attributes(file[name].attrs, reg) dtype = file[name].dtype - description = file[name].attrs['Description'] - physical = file[name].attrs['Value stored as physical'][0] == 1 - a_exponent = file[name].attrs['a-scale exponent'][0] - if not file[name].attrs['Property can be converted to comoving']: + description = file[name].attrs["Description"] + physical = file[name].attrs["Value stored as physical"][0] == 1 + a_exponent = file[name].attrs["a-scale exponent"][0] + if not file[name].attrs["Property can be converted to comoving"]: a_exponent = None metadata.append( (name, size, unit, dtype, description, physical, a_exponent) diff --git a/shared_array.py b/SOAP/core/shared_array.py similarity index 100% rename from shared_array.py rename to SOAP/core/shared_array.py diff --git a/shared_mesh.py b/SOAP/core/shared_mesh.py similarity index 52% rename from shared_mesh.py rename to SOAP/core/shared_mesh.py index b49be47d..ab830fbd 100644 --- a/shared_mesh.py +++ b/SOAP/core/shared_mesh.py @@ -1,10 +1,10 @@ #!/bin/env python import numpy as np -import shared_array -import virgo.mpi.parallel_sort as ps from mpi4py import MPI -import pytest +import virgo.mpi.parallel_sort as ps + +from SOAP.core import shared_array class SharedMesh: @@ -62,7 +62,7 @@ def __init__(self, comm, pos, resolution): # Determine the cell size self.resolution = int(resolution) - nr_cells = self.resolution ** 3 + nr_cells = self.resolution**3 self.cell_size = (self.pos_max - self.pos_min) / self.resolution # Determine which cell each particle in the local part of pos belongs to @@ -73,7 +73,7 @@ def __init__(self, comm, pos, resolution): cell_idx = ( cell_idx[:, 0] + self.resolution * cell_idx[:, 1] - + (self.resolution ** 2) * cell_idx[:, 2] + + (self.resolution**2) * cell_idx[:, 2] ) # Count local particles per cell @@ -139,7 +139,7 @@ def periodic_distance_squared(pos, centre): dr = pos - centre[None, :] dr[dr > 0.5 * boxsize] -= boxsize dr[dr < -0.5 * boxsize] += boxsize - return np.sum(dr ** 2, axis=1) + return np.sum(dr**2, axis=1) # Find the coordinates in the grid to search in each dimension. Here we deal with the # periodic box by also considering periodic copies of the search centre and radius. @@ -183,7 +183,7 @@ def periodic_distance_squared(pos, centre): for k in cell_coords[2]: for j in cell_coords[1]: for i in cell_coords[0]: - cell_nr = i + self.resolution * j + (self.resolution ** 2) * k + cell_nr = i + self.resolution * j + (self.resolution**2) * k start = self.cell_offset.full[cell_nr] count = self.cell_count.full[cell_nr] if count > 0: @@ -198,226 +198,3 @@ def periodic_distance_squared(pos, centre): return np.concatenate(idx) else: return np.ndarray(0, dtype=int) - - -def make_test_dataset(boxsize, total_nr_points, centre, radius, box_wrap, comm): - """ - Make a set of random test points - - boxsize - periodic box size (unyt scalar) - total_nr_points - number of points in the box over all MPI ranks - centre - centre of the particle distribution - radius - half side length of the particle distribution - box_wrap - True if points should be wrapped into the box - comm - MPI communicator to use - - Returns a (total_nr_points,3) SharedArray instance. - """ - comm_size = comm.Get_size() - comm_rank = comm.Get_rank() - - # Determine number of points per rank - nr_points = total_nr_points // comm_size - if comm_rank < (total_nr_points % comm_size): - nr_points += 1 - assert comm.allreduce(nr_points) == total_nr_points - - # Make some test data - pos = shared_array.SharedArray( - local_shape=(nr_points, 3), dtype=np.float64, units=radius.units, comm=comm - ) - if comm_rank == 0: - # Rank 0 initializes all elements to avoid parallel RNG issues - pos.full[:, :] = 2 * radius * np.random.random_sample(pos.full.shape) - radius - pos.full[:, :] += centre[None, :].to(radius.units) - if box_wrap: - pos.full[:, :] = pos.full[:, :] % boxsize - assert np.all((pos.full >= 0.0) & (pos.full < boxsize)) - pos.sync() - comm.barrier() - return pos - - -def _test_periodic_box( - total_nr_points, - centre, - radius, - boxsize, - box_wrap, - nr_queries, - resolution, - max_search_radius, -): - """ - Test case where points fill the periodic box. - - Creates a shared mesh from random points, queries for points near random - centres and checks the results against a simple brute force method. - """ - - from mpi4py import MPI - - comm = MPI.COMM_WORLD - comm_size = comm.Get_size() - comm_rank = comm.Get_rank() - - if comm_rank == 0: - print( - f"Test with {total_nr_points} points, resolution {resolution} and {nr_queries} queries" - ) - print( - f" Boxsize {boxsize}, centre {centre}, radius {radius}, box_wrap {box_wrap}" - ) - - def periodic_distance_squared(pos, centre): - dr = pos - centre[None, :] - dr[dr > 0.5 * boxsize] -= boxsize - dr[dr < -0.5 * boxsize] += boxsize - return np.sum(dr ** 2, axis=1) - - # Generate random test points - pos = make_test_dataset(boxsize, total_nr_points, centre, radius, box_wrap, comm) - - # Construct the shared mesh - mesh = SharedMesh(comm, pos, resolution=resolution) - - # Each MPI rank queries random points and verifies the result - nr_failures = 0 - for query_nr in range(nr_queries): - - # Pick a centre and radius - search_centre = (np.random.random_sample((3,)) * 2 * radius) - radius + centre - search_radius = np.random.random_sample(()) * max_search_radius - - # Query the mesh for point indexes - idx = mesh.query_radius_periodic(search_centre, search_radius, pos, boxsize) - - # Check that the indexes are unique - if len(idx) != len(np.unique(idx)): - print( - f" Duplicate IDs for centre={search_centre}, radius={search_radius}" - ) - nr_failures += 1 - else: - # Flag the points in the returned index array - in_idx = np.zeros(pos.full.shape[0], dtype=bool) - in_idx[idx] = True - # Find radii of all points - r2 = periodic_distance_squared(pos.full, search_centre) - # Check for any flagged points outside the radius - if np.any(r2[in_idx] > search_radius * search_radius): - print( - f" Returned point outside radius for centre={search_centre}, radius={search_radius}" - ) - nr_failures += 1 - # Check for any non-flagged points inside the radius - missed = (in_idx == False) & (r2 < search_radius * search_radius) - if np.any(missed): - print(r2[missed]) - print( - f" Missed point inside radius for centre={search_centre}, radius={search_radius}, rank={comm_rank}" - ) - nr_failures += 1 - - # Tidy up before possibly throwing an exception - pos.free() - mesh.free() - - nr_failures = comm.allreduce(nr_failures) - - comm.barrier() - if comm_rank == 0: - if nr_failures == 0: - print(f" OK") - else: - print(f" {nr_failures} of {nr_queries*comm_size} queries FAILED") - comm.Abort(1) - - -@pytest.mark.mpi -def test_shared_mesh(): - - import unyt - - # Use a different, reproducible seed on each rank - from mpi4py import MPI - - comm = MPI.COMM_WORLD - np.random.seed(comm.Get_rank()) - - resolutions = (1, 2, 4, 8, 16, 32) - - # Test a particle distribution which fills the box, searching up to 0.25 box size - for resolution in resolutions: - centre = 0.5 * np.ones(3, dtype=np.float64) * unyt.m - radius = 0.5 * unyt.m - centre, radius = comm.bcast((centre, radius)) - boxsize = 1.0 * unyt.m - _test_periodic_box( - 1000, - centre, - radius, - boxsize, - box_wrap=False, - nr_queries=100, - resolution=resolution, - max_search_radius=0.25 * boxsize, - ) - - # Test populating some random sub-regions, which may extend outside the box or be wrapped back in - nr_regions = 10 - boxsize = 1.0 * unyt.m - for box_wrap in (True, False): - for resolution in resolutions: - for region_nr in range(nr_regions): - centre = np.random.random_sample((3,)) * boxsize - radius = 0.25 * np.random.random_sample(()) * boxsize - centre, radius = comm.bcast((centre, radius)) - _test_periodic_box( - 1000, - centre, - radius, - boxsize, - box_wrap=box_wrap, - nr_queries=10, - resolution=resolution, - max_search_radius=radius, - ) - - # Zero particles in the box - for resolution in resolutions: - centre = 0.5 * np.ones(3, dtype=np.float64) * unyt.m - radius = 0.5 * unyt.m - centre, radius = comm.bcast((centre, radius)) - boxsize = 1.0 * unyt.m - _test_periodic_box( - 0, - centre, - radius, - boxsize, - box_wrap=False, - nr_queries=100, - resolution=resolution, - max_search_radius=0.25 * boxsize, - ) - - # One particle in the box - for resolution in resolutions: - centre = 0.5 * np.ones(3, dtype=np.float64) * unyt.m - radius = 0.5 * unyt.m - centre, radius = comm.bcast((centre, radius)) - boxsize = 1.0 * unyt.m - _test_periodic_box( - 1, - centre, - radius, - boxsize, - box_wrap=False, - nr_queries=100, - resolution=resolution, - max_search_radius=0.25 * boxsize, - ) - - -if __name__ == "__main__": - test_shared_mesh() diff --git a/sleepy_recv.py b/SOAP/core/sleepy_recv.py similarity index 100% rename from sleepy_recv.py rename to SOAP/core/sleepy_recv.py diff --git a/snapshot_datasets.py b/SOAP/core/snapshot_datasets.py similarity index 67% rename from snapshot_datasets.py rename to SOAP/core/snapshot_datasets.py index 1311d186..7fe07010 100644 --- a/snapshot_datasets.py +++ b/SOAP/core/snapshot_datasets.py @@ -19,7 +19,7 @@ from typing import Dict from numpy.typing import NDArray -import property_table +from SOAP import property_table class SnapshotDatasets: @@ -36,8 +36,6 @@ class SnapshotDatasets: dataset_map: Dict # mapping from dataset + column names to column index named_columns: Dict - # grain compositions in the dust model (currently not used) - dust_grain_composition: NDArray[float] # constants defined in the parameter file defined_constants: Dict @@ -72,28 +70,24 @@ def __init__(self, filenames: list): "NamedColumns" not in file_handle["SubgridScheme"] ): continue + # As the snapshot filename is done first, if one of the extra-input # files has a named column entry in common with the snapshot then - # we use then one from the extra-input file. - for name in file_handle["SubgridScheme"]["NamedColumns"]: - column_names = file_handle["SubgridScheme"]["NamedColumns"][name][:] - self.named_columns[name] = {} - # turn the list into a dictionary that maps a column name to - # a colum index - for iname, colname in enumerate(column_names): - self.named_columns[name][colname.decode("utf-8")] = iname - - try: - self.dust_grain_composition = file_handle["SubgridScheme"][ - "GrainToElementMapping" - ][:] - except KeyError: - try: - self.dust_grain_composition = file_handle["SubgridScheme"][ - "DustMassFractionsToElementMassFractionsMapping" - ][:] - except KeyError: - pass + # we use then one from the extra-input file. We append the ptype + # in case there of conflict (e.g. we use PartType0/SpeciesFractions + # from the snapshot file, but PartType4/SpeciesFractions from the + # extra-input files. + for dset in file_handle["SubgridScheme"]["NamedColumns"]: + column_names = file_handle["SubgridScheme"]["NamedColumns"][dset][:] + for group, datasets in self.datasets_in_file.items(): + if not dset in datasets: + continue + name = f"{group}/{dset}" + self.named_columns[name] = {} + # turn the list into a dictionary that maps a column name to + # a colum index + for iname, colname in enumerate(column_names): + self.named_columns[name][colname.decode("utf-8")] = iname def setup_aliases(self, aliases: Dict): """ @@ -130,10 +124,8 @@ def setup_aliases(self, aliases: Dict): SOAP_ptype, SOAP_dset = alias.split("/") snap_ptype, snap_dset = aliases[alias].split("/") self.dataset_map[alias] = (snap_ptype, snap_dset) - if (snap_dset in self.named_columns) and ( - SOAP_dset not in self.named_columns - ): - self.named_columns[SOAP_dset] = dict(self.named_columns[snap_dset]) + if aliases[alias] in self.named_columns: + self.named_columns[alias] = dict(self.named_columns[aliases[alias]]) def setup_defined_constants(self, defined_constants: Dict): """ @@ -182,45 +174,18 @@ def get_dataset(self, name: str, data_dict: Dict) -> unyt.unyt_array: try: ptype, dset = self.dataset_map[name] except KeyError as e: - print(f'Dataset "{name}" not found!') - print("The following properties require this dataset:") - full_property_list = property_table.PropertyTable.full_property_list - for k, v in full_property_list.items(): - if name in v[8]: - print(k) - raise e + # This should never occur since swift_cells.check_datasets_exist + # checks all the properties we require are indeed available + raise KeyError(f"Failed to read {name} from input files!") return data_dict[ptype][dset] - def get_dataset_column( - self, name: str, column_name: str, data_dict: Dict - ) -> unyt.unyt_array: - """ - Get the data for the given named column in the dataset with the given - generic name. - - Parameters: - - name: str - Generic name of a dataset, as used by halo property calculations. - - column_name: str - Name of a named column, as defined in the snapshot metadata and used - by halo property calculations. - - data_dict: Dict - Dictionary of particle properties, as read from the snapshot. - - Returns the corresponding data, taking into account potential - aliases and the named column metadata. - """ - ptype, dset = self.dataset_map[name] - column_index = self.named_columns[dset][column_name] - return data_dict[ptype][dset][:, column_index] - - def get_column_index(self, dset: str, column_name: str) -> int: + def get_column_index(self, name: str, column_name: str) -> int: """ Get the index of the given named column of the dataset with the given name. Parameters: - - dset: str + - name: str Generic name of a dataset, as used by halo property calculations. - column_name: str Name of a named column, as defined in the snapshot metadata and @@ -230,20 +195,4 @@ def get_column_index(self, dset: str, column_name: str) -> int: access that specific column in a data array that was obtained earlier using get_dataset(). """ - return self.named_columns[dset][column_name] - - def get_dust_grain_composition(self, grain_name: str) -> NDArray[float]: - """ - Get the composition of the grain with the given name. - - Currently not used. - - Parameters: - - grain_name: str - Name of a dust grain. - - Returns the corresponding elemental composition of the grain. - """ - return self.dust_grain_composition[ - self.named_columns["DustMassFractions"][grain_name] - ] + return self.named_columns[name][column_name] diff --git a/soap_args.py b/SOAP/core/soap_args.py similarity index 82% rename from soap_args.py rename to SOAP/core/soap_args.py index 4bcd6e46..4e974e27 100644 --- a/soap_args.py +++ b/SOAP/core/soap_args.py @@ -8,7 +8,7 @@ from virgo.mpi.util import MPIArgumentParser -import combine_args +from . import combine_args def get_git_hash() -> str: @@ -54,6 +54,16 @@ def get_soap_args(comm): parser.add_argument( "--centrals-only", action="store_true", help="Only process central halos" ) + parser.add_argument( + "--record-halo-timings", + action="store_true", + help="Record time taken to process each halo", + ) + parser.add_argument( + "--record-property-timings", + action="store_true", + help="Record time taken to process each property", + ) parser.add_argument( "--max-halos", metavar="N", @@ -111,11 +121,17 @@ def get_soap_args(comm): args.scratch_dir = all_args["HaloProperties"]["chunk_dir"] args.halo_basename = all_args["HaloFinder"]["filename"] args.halo_format = all_args["HaloFinder"]["type"] + args.read_potential_energies = all_args["HaloFinder"].get( + "read_potential_energies", False + ) args.fof_group_filename = all_args["HaloFinder"].get("fof_filename", "") + args.fof_radius_filename = all_args["HaloFinder"].get("fof_radius_filename", "") args.output_file = all_args["HaloProperties"]["filename"] args.snapshot_nr = all_args["Parameters"]["snap_nr"] args.chunks = all_args["Parameters"]["chunks"] args.centrals_only = all_args["Parameters"]["centrals_only"] + args.record_halo_timings = all_args["Parameters"]["record_halo_timings"] + args.record_property_timings = all_args["Parameters"]["record_property_timings"] args.dmo = all_args["Parameters"]["dmo"] args.max_halos = all_args["Parameters"]["max_halos"] args.halo_indices = all_args["Parameters"]["halo_indices"] @@ -124,7 +140,7 @@ def get_soap_args(comm): args.max_ranks_reading = all_args["Parameters"]["max_ranks_reading"] args.output_parameters = all_args["Parameters"]["output_parameters"] args.git_hash = all_args["git_hash"] - args.min_read_radius_cmpc = all_args["calculations"]["min_read_radius_cmpc"] + args.min_read_radius_cmpc = all_args["calculations"].get("min_read_radius_cmpc", 0) args.calculations = all_args["calculations"] # Extra-input files which are optionally passed in the parameter file are @@ -165,7 +181,15 @@ def get_soap_args(comm): snap_nr=args.snapshot_nr, file_nr=0 ) if not os.path.exists(fof_filename): - print("FOF group catalogues do not exist") + print(f"Could not find FOF group catalogue: {fof_filename}") + comm.Abort(1) + if args.fof_radius_filename != "": + assert args.fof_group_filename != "" + fof_filename = args.fof_radius_filename.format( + snap_nr=args.snapshot_nr, file_nr=0 + ) + if not os.path.exists(fof_filename): + print(f"Could not find FOF radius catalogue: {fof_filename}") comm.Abort(1) return args diff --git a/swift_cells.py b/SOAP/core/swift_cells.py similarity index 95% rename from swift_cells.py rename to SOAP/core/swift_cells.py index 8bff8efe..30061fe8 100644 --- a/swift_cells.py +++ b/SOAP/core/swift_cells.py @@ -2,18 +2,18 @@ import collections -import numpy as np import h5py from mpi4py import MPI -import unyt +import numpy as np import scipy.spatial import virgo.mpi.parallel_hdf5 as phdf5 +import unyt -import swift_units -import task_queue -import shared_array -from snapshot_datasets import SnapshotDatasets -import property_table +from . import swift_units +from . import task_queue +from . import shared_array +from .snapshot_datasets import SnapshotDatasets +from SOAP import property_table # HDF5 chunk cache parameters: # SWIFT writes datasets with large chunks so the default 1Mb may be too small @@ -212,7 +212,7 @@ def __init__( # Determine if this is a snapshot or snipshot self.snipshot = ( - self.swift_header_group["SelectOutput"].decode() == "Snipshot" + self.swift_header_group.get("SelectOutput", b"").decode() == "Snipshot" ) # Read the critical density and attach units @@ -255,11 +255,12 @@ def __init__( # mean density. H0 = self.cosmology["H0 [internal units]"] G = self.constants_internal["newton_G"] - critical_density_z0_internal = 3 * (H0 ** 2) / (8 * np.pi * G) - mean_density_z0_internal = ( - critical_density_z0_internal * self.cosmology["Omega_m"] - ) - mean_density_internal = mean_density_z0_internal / (self.a ** 3) + critical_density_z0_internal = 3 * (H0**2) / (8 * np.pi * G) + # We use non-relativistic neutrinos when we compute the mass with + # an SO, so consider them when we calculate the reference mean density + omega_m = self.cosmology["Omega_m"] + self.cosmology.get("Omega_nu_0", 0) + mean_density_z0_internal = critical_density_z0_internal * omega_m + mean_density_internal = mean_density_z0_internal / (self.a**3) self.mean_density = unyt.unyt_quantity( mean_density_internal, units=internal_density_unit ) @@ -268,10 +269,10 @@ def __init__( Omega_k = self.cosmology["Omega_k"] Omega_Lambda = self.cosmology["Omega_lambda"] Omega_m = self.cosmology["Omega_m"] - bnx = -(Omega_k / self.a ** 2 + Omega_Lambda) / ( - Omega_k / self.a ** 2 + Omega_m / self.a ** 3 + Omega_Lambda + bnx = -(Omega_k / self.a**2 + Omega_Lambda) / ( + Omega_k / self.a**2 + Omega_m / self.a**3 + Omega_Lambda ) - self.virBN98 = 18.0 * np.pi ** 2 + 82.0 * bnx - 39.0 * bnx ** 2 + self.virBN98 = 18.0 * np.pi**2 + 82.0 * bnx - 39.0 * bnx**2 if self.virBN98 < 50.0 or self.virBN98 > 1000.0: raise RuntimeError("Invalid value for virBN98!") @@ -438,7 +439,7 @@ def verify_extra_input(self, comm): npart_extra = extra_file[f"{parttype}/{dset}"].shape[0] if npart_snapshot[parttype] != npart_extra: print(f"Incorrect number of {parttype} in {extra_filename}") - comm.Abort() + comm.Abort(1) def check_datasets_exist(self, required_datasets, halo_prop_list): # Check we have all the fields needed for each property @@ -446,6 +447,8 @@ def check_datasets_exist(self, required_datasets, halo_prop_list): # to output a list of properties that require the missing fields for ptype in set(self.ptypes).intersection(set(required_datasets.keys())): for name in required_datasets[ptype]: + # Note that the field names in required_datasets have already had + # any aliases applied, so we can check the raw files themselves in_extra = (self.extra_filenames is not None) and ( name in self.extra_metadata_combined[ptype] ) @@ -456,12 +459,12 @@ def check_datasets_exist(self, required_datasets, halo_prop_list): full_property_list = property_table.PropertyTable.full_property_list for k, v in full_property_list.items(): # Skip property if it doesn't require this dataset - if dataset not in v[8]: + if dataset not in v.particle_properties: continue # Only print if the property is being calculated for some halo type for halo_prop in halo_prop_list: - if halo_prop.property_mask.get(v[0], False): - print(f" {v[0]}") + if halo_prop.property_filters.get(v.name, False): + print(f" {v.name}") break raise KeyError( f"Can't find required dataset {dataset} in input file(s)!" @@ -492,7 +495,7 @@ def prepare_read(self, ptype, mask): cells_to_read = cells_to_read[idx] # Merge adjacent cells - max_size = 20 * 1024 ** 2 + max_size = 20 * 1024**2 nr_to_read = len(cells_to_read) for cell_nr in range(nr_to_read - 1): cell1 = cells_to_read[cell_nr] @@ -572,7 +575,7 @@ def read_masked_cells_to_shared_memory( for file_nr in all_file_nrs: for ptype in reads_for_type: if file_nr in reads_for_type[ptype]: - for (file_offset, mem_offset, count) in reads_for_type[ptype][ + for file_offset, mem_offset, count in reads_for_type[ptype][ file_nr ]: nr_parts[ptype] += count @@ -591,9 +594,9 @@ def read_masked_cells_to_shared_memory( and dataset not in self.extra_metadata_combined[ptype] ): if file_nr in reads_for_type[ptype]: - for (file_offset, mem_offset, count) in reads_for_type[ - ptype - ][file_nr]: + for file_offset, mem_offset, count in reads_for_type[ptype][ + file_nr + ]: all_tasks.append( ReadTask( filename, @@ -765,7 +768,7 @@ def complete_radius_from_mask(self, mask): # All types use the same grid, so just use cell arrays for the first type ptype = list(self.cell.keys())[0] cell_centre = self.cell[ptype]["centre"] - cell_diagonal = np.sqrt(np.sum(self.cell_size.value ** 2)) + cell_diagonal = np.sqrt(np.sum(self.cell_size.value**2)) # Output array cell_complete_radius = np.zeros(self.dimension) diff --git a/swift_units.py b/SOAP/core/swift_units.py similarity index 95% rename from swift_units.py rename to SOAP/core/swift_units.py index 32bedd75..7e0fff62 100644 --- a/swift_units.py +++ b/SOAP/core/swift_units.py @@ -76,7 +76,7 @@ def unit_registry_from_snapshot(snap): ) unyt.define_unit( "newton_G", - physical_constants_cgs["newton_G"] * unyt.cm ** 3 / unyt.g / unyt.s ** 2, + physical_constants_cgs["newton_G"] * unyt.cm**3 / unyt.g / unyt.s**2, registry=reg, ) @@ -112,9 +112,9 @@ def units_from_attributes(attrs, registry): u = u * unit elif exponent != 0.0: if u is unyt.dimensionless: - u = unit ** exponent + u = unit**exponent else: - u = u * (unit ** exponent) + u = u * (unit**exponent) # Add expansion factor a_scale_exponent = attrs["a-scale exponent"][0] @@ -166,7 +166,7 @@ def attributes_from_units(units, physical, a_exponent): if physical: assert a_exponent_in_units == 0 else: - assert a_exponent_in_units == a_exponent + assert float(a_exponent_in_units) == a_exponent # Get h exponent h_unit = unyt.Unit("h", registry=units.registry) @@ -180,7 +180,7 @@ def attributes_from_units(units, physical, a_exponent): # Set the attributes attrs["Conversion factor to CGS (not including cosmological corrections)"] = [ - float(cgs_factor / (a_val ** a_exponent_in_units) / (h_val ** h_exponent)) + float(cgs_factor / (a_val**a_exponent_in_units) / (h_val**h_exponent)) ] attrs["Conversion factor to physical CGS (including cosmological corrections)"] = [ float(cgs_factor) diff --git a/task_queue.py b/SOAP/core/task_queue.py similarity index 98% rename from task_queue.py rename to SOAP/core/task_queue.py index 598fc4e4..d40d6195 100644 --- a/task_queue.py +++ b/SOAP/core/task_queue.py @@ -1,13 +1,13 @@ #!/bin/env python +import collections import time import threading from mpi4py import MPI -import collections import numpy as np -from mpi_tags import REQUEST_TASK_TAG, ASSIGN_TASK_TAG -from sleepy_recv import sleepy_recv +from .mpi_tags import REQUEST_TASK_TAG, ASSIGN_TASK_TAG +from .sleepy_recv import sleepy_recv def distribute_tasks(tasks, comm): diff --git a/group_membership.py b/SOAP/group_membership.py similarity index 75% rename from group_membership.py rename to SOAP/group_membership.py index 9d0b308d..1b0db8ff 100644 --- a/group_membership.py +++ b/SOAP/group_membership.py @@ -2,20 +2,19 @@ import time import socket -import numpy as np -import h5py +import h5py +import numpy as np +from mpi4py import MPI import virgo.mpi.parallel_hdf5 as phdf5 import virgo.mpi.parallel_sort as psort from virgo.util.partial_formatter import PartialFormatter -import lustre -import combine_args -import read_vr -import read_hbtplus -import read_subfind -import read_rockstar -from mpi4py import MPI +from SOAP.core import combine_args, swift_units +from SOAP.catalogue_readers import read_vr +from SOAP.catalogue_readers import read_hbtplus +from SOAP.catalogue_readers import read_subfind +from SOAP.catalogue_readers import read_rockstar comm = MPI.COMM_WORLD comm_rank = comm.Get_rank() @@ -28,10 +27,11 @@ def process_particle_type( ids_bound, grnr_bound, rank_bound, - ids_unbound, + potential_energies, fof_ptypes, fof_file, create_file, + output_filename, ): """ Compute group membership for one particle type @@ -78,23 +78,22 @@ def process_particle_type( swift_rank_bound = np.ndarray(len(swift_ids), dtype=rank_bound.dtype) swift_rank_bound[matched] = psort.fetch_elements(rank_bound, ptr[matched]) swift_rank_bound[matched == False] = -1 - del ptr - del matched - if ids_unbound is not None: + if potential_energies is not None: if comm_rank == 0: - print(" Matching SWIFT particle IDs to unbound IDs") - ptr = psort.parallel_match(swift_ids, ids_unbound) + print(" Assigning potential energy to SWIFT particles") + if potential_energies.shape[0] > 0: + assert np.max(potential_energies) <= 0 + swift_potential_energies = np.ndarray( + len(swift_ids), dtype=potential_energies.dtype + ) + swift_potential_energies[matched] = psort.fetch_elements( + potential_energies, ptr[matched] + ) + swift_potential_energies[matched == False] = 0 - if comm_rank == 0: - print(" Assigning unbound group membership to SWIFT particles") - matched = ptr >= 0 - swift_grnr_unbound = np.ndarray(len(swift_ids), dtype=grnr_unbound.dtype) - swift_grnr_unbound[matched] = psort.fetch_elements(grnr_unbound, ptr[matched]) - swift_grnr_unbound[matched == False] = -1 - swift_grnr_all = np.maximum(swift_grnr_bound, swift_grnr_unbound) - del ptr - del matched + del ptr + del matched # Determine if we need to create a new output file set if create_file: @@ -113,6 +112,8 @@ def process_particle_type( "U_T exponent": [0.0], "a-scale exponent": [0.0], "h-scale exponent": [0.0], + "Property can be converted to comoving": [0], + "Value stored as physical": [1], } attrs = { "GroupNr_bound": { @@ -121,8 +122,8 @@ def process_particle_type( "Rank_bound": { "Description": "Ranking by binding energy of the bound particles (first in halo=0), or -1 if not bound" }, - "GroupNr_all": { - "Description": "Index of halo in which this particle is a member (bound or unbound), or -1 if none" + "SpecificPotentialEnergies": { + "Description": "Specific potential energy of the bound particles" }, "FOFGroupIDs": { "Description": "Friends-Of-Friends ID of the group in which this particle is a member, of -1 if none" @@ -130,7 +131,6 @@ def process_particle_type( } attrs["GroupNr_bound"].update(unit_attrs) attrs["Rank_bound"].update(unit_attrs) - attrs["GroupNr_all"].update(unit_attrs) attrs["FOFGroupIDs"].update(unit_attrs) # Write these particles out with the same layout as the input snapshot @@ -140,21 +140,25 @@ def process_particle_type( output = {"GroupNr_bound": swift_grnr_bound} if rank_bound is not None: output["Rank_bound"] = swift_rank_bound - if ids_unbound is not None: - output["GroupNr_all"] = swift_grnr_all + if potential_energies is not None: + unit_attrs = swift_units.attributes_from_units( + potential_energies.units, True, None + ) + attrs["SpecificPotentialEnergies"].update(unit_attrs) + output["SpecificPotentialEnergies"] = swift_potential_energies if ptype in fof_ptypes: output["FOFGroupIDs"] = swift_fof_group_ids snap_file.write( output, elements_per_file, - filenames=output_file, + filenames=output_filename, mode=mode, group=ptype, attrs=attrs, ) -if __name__ == "__main__": +def main(): # Read parameters from command line and config file from virgo.mpi.util import MPIArgumentParser @@ -177,31 +181,36 @@ def process_particle_type( swift_filename = args["Snapshots"]["filename"] halo_format = args["HaloFinder"]["type"] halo_basename = args["HaloFinder"]["filename"] - output_file = args["GroupMembership"]["filename"] + read_potential_energies = args["HaloFinder"].get("read_potential_energies", False) + output_filename = args["GroupMembership"]["filename"] if comm_rank == 0: - print(f'Input snapshot is {swift_filename}') - print(f'Halo basename is {halo_basename}') - print(f'Snapshot number is {snap_nr}') + print(f"Input snapshot is {swift_filename}") + print(f"Halo basename is {halo_basename}") + print(f"Snapshot number is {snap_nr}") # Substitute in the snapshot number where necessary pf = PartialFormatter() swift_filename = pf.format(swift_filename, snap_nr=snap_nr, file_nr=None) fof_filename = pf.format(fof_filename, snap_nr=snap_nr, file_nr=None) halo_basename = pf.format(halo_basename, snap_nr=snap_nr, file_nr=None) - output_file = pf.format(output_file, snap_nr=snap_nr, file_nr=None) + output_filename = pf.format(output_filename, snap_nr=snap_nr, file_nr=None) # Check both swift and output filenames are (not) chunk files if "file_nr" in swift_filename: - assert "file_nr" in output_file, "Membership filenames require {file_nr}" + assert "file_nr" in output_filename, "Membership filenames require {file_nr}" else: assert ( - "file_nr" not in output_file + "file_nr" not in output_filename ), "Membership filenames shouldn't have {file_nr}" # Ensure output dir exists if comm_rank == 0: - lustre.ensure_output_dir(output_file) + try: + os.makedirs(os.path.dirname(output_filename), exist_ok=True) + except OSError as e: + print(f"Error creating output directory: {e}") + comm.Abort(1) comm.barrier() # Find group number for each particle ID in the halo finder output @@ -212,32 +221,48 @@ def process_particle_type( ids_bound, grnr_bound, rank_bound, - ids_unbound, - grnr_unbound, ) = read_vr.read_vr_groupnr(halo_basename) + potential_energies = None elif halo_format == "HBTplus": # Read HBTplus output - total_nr_halos, ids_bound, grnr_bound, rank_bound = read_hbtplus.read_hbtplus_groupnr( - halo_basename - ) - ids_unbound = None # HBTplus does not output unbound particles - grnr_unbound = None + if read_potential_energies: + if comm_rank == 0: + with h5py.File(swift_filename.format(file_nr=0), "r") as file: + registry = swift_units.unit_registry_from_snapshot(file) + else: + registry = None + registry = comm.bcast(registry) + total_nr_halos, ids_bound, grnr_bound, rank_bound, potential_energies = ( + read_hbtplus.read_hbtplus_groupnr( + halo_basename, + read_potential_energies=True, + registry=registry, + ) + ) + else: + if comm_rank == 0: + print("Not reading in potential energies") + total_nr_halos, ids_bound, grnr_bound, rank_bound = ( + read_hbtplus.read_hbtplus_groupnr( + halo_basename, + ) + ) + potential_energies = None + elif halo_format == "Subfind": # Read Gadget-4 subfind output total_nr_halos, ids_bound, grnr_bound = read_subfind.read_gadget4_groupnr( halo_basename ) - ids_unbound = None - grnr_unbound = None rank_bound = None + potential_energies = None elif halo_format == "Rockstar": # Read Rockstar output total_nr_halos, ids_bound, grnr_bound = read_rockstar.read_rockstar_groupnr( halo_basename ) - ids_unbound = None - grnr_unbound = None rank_bound = None + potential_energies = None else: raise RuntimeError(f"Unrecognised halo finder name: {halo_format}") @@ -322,10 +347,11 @@ def process_particle_type( ids_bound, grnr_bound, rank_bound, - ids_unbound, + potential_energies, fof_ptypes, fof_file, create_file, + output_filename, ) create_file = False comm.barrier() @@ -337,11 +363,15 @@ def process_particle_type( for file_nr in range( first_file[comm_rank], first_file[comm_rank] + files_on_rank[comm_rank] ): - with h5py.File(output_file.format(file_nr=file_nr), "r+") as infile: - group = infile.create_group('Header') + with h5py.File(output_filename.format(file_nr=file_nr), "r+") as infile: + group = infile.create_group("Header") for k, v in header.items(): group.attrs[k] = v comm.barrier() if comm_rank == 0: print("Done.") + + +if __name__ == "__main__": + main() diff --git a/cold_dense_gas_filter.py b/SOAP/particle_filter/cold_dense_gas_filter.py similarity index 100% rename from cold_dense_gas_filter.py rename to SOAP/particle_filter/cold_dense_gas_filter.py diff --git a/recently_heated_gas_filter.py b/SOAP/particle_filter/recently_heated_gas_filter.py similarity index 95% rename from recently_heated_gas_filter.py rename to SOAP/particle_filter/recently_heated_gas_filter.py index 597ca479..8cdb7721 100644 --- a/recently_heated_gas_filter.py +++ b/SOAP/particle_filter/recently_heated_gas_filter.py @@ -16,15 +16,13 @@ requires knowledge of the cosmology. """ -import numpy as np -import unyt from astropy.cosmology import w0waCDM, z_at_value import astropy.constants as const import astropy.units as astropy_units - -from swift_cells import SWIFTCellGrid +import numpy as np from numpy.typing import NDArray +import unyt class RecentlyHeatedGasFilter: @@ -51,7 +49,7 @@ class RecentlyHeatedGasFilter: def __init__( self, - cellgrid: SWIFTCellGrid, + cellgrid, delta_time: unyt.unyt_quantity, use_AGN_delta_T: bool, initialised: bool, @@ -84,6 +82,8 @@ def __init__( file. If this is false and the filter is called, it will throw an error. """ self.initialised = initialised + if not self.initialised: + return H0 = unyt.unyt_quantity( cellgrid.cosmology["H0 [internal units]"], @@ -102,13 +102,13 @@ def __init__( # expressions taken directly from astropy, since they do no longer # allow access to these attributes (since version 5.1+) critdens_const = (3.0 / (8.0 * np.pi * const.G)).cgs.value - a_B_c2 = (4.0 * const.sigma_sb / const.c ** 3).cgs.value + a_B_c2 = (4.0 * const.sigma_sb / const.c**3).cgs.value # SWIFT provides Omega_g, but we need a consistent Tcmb0 for astropy. # This is an exact inversion of the procedure performed in astropy. critical_density_0 = astropy_units.Quantity( critdens_const * H0.to("1/s").value ** 2, - astropy_units.g / astropy_units.cm ** 3, + astropy_units.g / astropy_units.cm**3, ) Tcmb0 = (Omega_g * critical_density_0.value / a_B_c2) ** (1.0 / 4.0) @@ -143,8 +143,8 @@ def __init__( self.use_AGN_delta_T = use_AGN_delta_T if use_AGN_delta_T: AGN_delta_T = cellgrid.AGN_delta_T - self.Tmin = AGN_delta_T * 10.0 ** delta_logT_min - self.Tmax = AGN_delta_T * 10.0 ** delta_logT_max + self.Tmin = AGN_delta_T * 10.0**delta_logT_min + self.Tmax = AGN_delta_T * 10.0**delta_logT_max self.metadata["AGN_delta_T_in_K"] = (AGN_delta_T.to("K").value,) self.metadata["delta_logT_min"] = (delta_logT_min,) self.metadata["delta_logT_max"] = (delta_logT_max,) diff --git a/SO_properties.py b/SOAP/particle_selection/SO_properties.py similarity index 83% rename from SO_properties.py rename to SOAP/particle_selection/SO_properties.py index a1a90d30..7faaf700 100644 --- a/SO_properties.py +++ b/SOAP/particle_selection/SO_properties.py @@ -17,32 +17,34 @@ somewhat more involved than simply using a fixed aperture. Contrary to the other halo types, spherical overdensities are only -calculated for central halos. SO properties are also only calculated if +calculated for central halos. SO properties are also only calculated if an SO radius could be determined. """ +import time +from typing import Tuple, Dict, List + import numpy as np -import unyt +from numpy.typing import NDArray from scipy.optimize import brentq +import unyt -from halo_properties import HaloProperty, SearchRadiusTooSmallError -from kinematic_properties import ( +from .halo_properties import HaloProperty, SearchRadiusTooSmallError +from SOAP.property_calculation.kinematic_properties import ( get_angular_momentum, - get_angular_momentum_and_kappa_corot, + get_angular_momentum_and_kappa_corot_mass_weighted, + get_angular_momentum_and_kappa_corot_luminosity_weighted, get_vmax, - get_inertia_tensor, ) -from recently_heated_gas_filter import RecentlyHeatedGasFilter -from property_table import PropertyTable -from dataset_names import mass_dataset -from lazy_properties import lazy_property -from category_filter import CategoryFilter -from parameter_file import ParameterFile -from snapshot_datasets import SnapshotDatasets -from swift_cells import SWIFTCellGrid - -from typing import Tuple, Dict, List -from numpy.typing import NDArray +from SOAP.property_calculation.inertia_tensors import get_inertia_tensor_mass_weighted +from SOAP.particle_filter.recently_heated_gas_filter import RecentlyHeatedGasFilter +from SOAP.property_table import PropertyTable +from SOAP.core.dataset_names import mass_dataset +from SOAP.core.lazy_properties import lazy_property +from SOAP.core.category_filter import CategoryFilter +from SOAP.core.parameter_file import ParameterFile +from SOAP.core.snapshot_datasets import SnapshotDatasets +from SOAP.core.swift_cells import SWIFTCellGrid def cumulative_mass_intersection(r: float, rho_dim: float, slope_dim: float) -> float: @@ -72,7 +74,7 @@ def cumulative_mass_intersection(r: float, rho_dim: float, slope_dim: float) -> Returns the value of the intersection equation for the given r (and using the given rho_dim and slope_dim boundary conditions). """ - return 4.0 * np.pi / 3.0 * rho_dim * r ** 3 - slope_dim * r + slope_dim - 1.0 + return 4.0 * np.pi / 3.0 * rho_dim * r**3 - slope_dim * r + slope_dim - 1.0 def find_SO_radius_and_mass( @@ -172,7 +174,7 @@ def find_SO_radius_and_mass( / (np.pi * ordered_radius[ipos] * reference_density) ) SO_mass = cumulative_mass[ipos] * SO_r / ordered_radius[ipos] - return SO_r, SO_mass, 4.0 * np.pi / 3.0 * SO_r ** 3 + return SO_r, SO_mass, 4.0 * np.pi / 3.0 * SO_r**3 # We now have the intersecting interval. Get the limits. r1 = ordered_radius[i - 1] @@ -201,13 +203,13 @@ def find_SO_radius_and_mass( # compute the dimensionless quantities that enter the intersection equation # remember, we are simply solving # 4*pi/3*r^3*rho = M1 + (M2-M1)/(r2-r1)*(r-r1) - rho_dim = reference_density * r1 ** 3 / M1 + rho_dim = reference_density * r1**3 / M1 slope_dim = (M2 - M1) / (r2 - r1) * (r1 / M1) SO_r = r1 * brentq( cumulative_mass_intersection, 1.0, r2 / r1, args=(rho_dim, slope_dim) ) - SO_volume = 4.0 / 3.0 * np.pi * SO_r ** 3 + SO_volume = 4.0 / 3.0 * np.pi * SO_r**3 # compute the SO mass by requiring that the mean density in the SO is the # target density SO_mass = SO_volume * reference_density @@ -249,6 +251,7 @@ def __init__( virial_definition: bool, search_radius: unyt.unyt_quantity, cosmology: dict, + boxsize: unyt.unyt_quantity, ): """ Constructor. @@ -283,6 +286,8 @@ def __init__( this radius. - cosmology: dict Cosmological parameters required for SO calculation + - boxsize: unyt.unyt_quantity + Boxsize for correcting periodic boundary conditions """ self.input_halo = input_halo self.data = data @@ -296,6 +301,7 @@ def __init__( self.virial_definition = virial_definition self.search_radius = search_radius self.cosmology = cosmology + self.boxsize = boxsize self.compute_basics() def get_dataset(self, name: str) -> unyt.unyt_array: @@ -329,7 +335,7 @@ def compute_basics(self): mass.append(self.get_dataset(f"{ptype}/{mass_dataset(ptype)}")) pos = self.get_dataset(f"{ptype}/Coordinates") - self.centre[None, :] position.append(pos) - r = np.sqrt(np.sum(pos ** 2, axis=1)) + r = np.sqrt(np.sum(pos**2, axis=1)) radius.append(r) velocity.append(self.get_dataset(f"{ptype}/Velocities")) typearr = int(ptype[-1]) * np.ones(r.shape, dtype=np.int32) @@ -376,7 +382,7 @@ def compute_SO_radius_and_mass( "PartType6/Weights" ) pos = self.get_dataset("PartType6/Coordinates") - self.centre[None, :] - nur = np.sqrt(np.sum(pos ** 2, axis=1)) + nur = np.sqrt(np.sum(pos**2, axis=1)) self.nu_mass = numass self.nu_radius = nur self.nu_softening = ( @@ -396,7 +402,7 @@ def compute_SO_radius_and_mass( ) # add mean neutrino mass cumulative_mass += ( - self.cosmology["nu_density"] * 4.0 / 3.0 * np.pi * ordered_radius ** 3 + self.cosmology["nu_density"] * 4.0 / 3.0 * np.pi * ordered_radius**3 ) # Determine FOF ID of object using the central non-neutrino particle non_neutrino_order = order[order < self.radius.shape[0]] @@ -411,7 +417,7 @@ def compute_SO_radius_and_mass( ordered_radius = ordered_radius[nskip:] cumulative_mass = cumulative_mass[nskip:] nr_parts = len(ordered_radius) - density = cumulative_mass / (4.0 / 3.0 * np.pi * ordered_radius ** 3) + density = cumulative_mass / (4.0 / 3.0 * np.pi * ordered_radius**3) # Check if we ever reach the density threshold if reference_density > 0: @@ -423,10 +429,10 @@ def compute_SO_radius_and_mass( except SearchRadiusTooSmallError: raise SearchRadiusTooSmallError("SO radius multiple was too small!") else: - self.SO_volume = 0 * ordered_radius.units ** 3 + self.SO_volume = 0 * ordered_radius.units**3 elif physical_radius > 0: self.SO_r = physical_radius - self.SO_volume = 4.0 * np.pi / 3.0 * self.SO_r ** 3 + self.SO_volume = 4.0 * np.pi / 3.0 * self.SO_r**3 if nr_parts > 0: # find the enclosed mass using interpolation outside_radius = ordered_radius > self.SO_r @@ -460,20 +466,20 @@ def compute_SO_radius_and_mass( ) if SO_exists: - # Calculate DMO mass fraction found at SO_r + # Estimate DMO mass fraction found at SO_r # This is used when computing concentration_dmo dm_r = self.radius[self.types == 1] dm_m = self.mass[self.types == 1] - order = np.argsort(dm_r) - ordered_dm_r = dm_r[order] - outside_radius = ordered_dm_r > self.SO_r + outside_radius = dm_r > self.SO_r self.dm_missed_mass = 0 * self.mass.units if np.any(outside_radius): - i = np.argmax(outside_radius) - if i != 0: # We have DM particles inside the SO radius - r1 = ordered_dm_r[i - 1] - r2 = ordered_dm_r[i] - self.dm_missed_mass = (self.SO_r - r1) / (r2 - r1) * dm_m[order][i] + inside_radius = np.logical_not(outside_radius) + if np.any(inside_radius): + r1 = np.max(dm_r[inside_radius]) + i = np.argmin(dm_r[outside_radius]) + r2 = dm_r[outside_radius][i] + m2 = dm_m[outside_radius][i] + self.dm_missed_mass = m2 * (self.SO_r - r1) / (r2 - r1) # Removing particles outside SO radius self.all_selection = self.radius < self.SO_r @@ -553,7 +559,9 @@ def com(self) -> unyt.unyt_array: """ Centre of mass of all particles in the spherical overdensity. """ - return (self.mass_fraction[:, None] * self.position).sum(axis=0) + self.centre + return ( + (self.mass_fraction[:, None] * self.position).sum(axis=0) + self.centre + ) % self.boxsize @lazy_property def vcom(self) -> unyt.unyt_array: @@ -562,6 +570,21 @@ def vcom(self) -> unyt.unyt_array: """ return (self.mass_fraction[:, None] * self.velocity).sum(axis=0) + @lazy_property + def R_vmax_soft(self) -> unyt.unyt_quantity: + """ + Radius at which the maximum circular velocity of the halo is reached. + Particles are set to have minimum radius equal to their softening length. + + This includes contributions from all particle types. + """ + if self.Mtotpart == 0: + return None + if not hasattr(self, "vmax_soft"): + soft_r = np.maximum(self.softening, self.radius) + self.r_vmax_soft, self.vmax_soft = get_vmax(self.mass, soft_r) + return self.r_vmax_soft + @lazy_property def Vmax_soft(self): """ @@ -605,7 +628,7 @@ def TotalInertiaTensor(self) -> unyt.unyt_array: return None mass = np.concatenate([self.mass, self.surrounding_mass], axis=0) position = np.concatenate([self.position, self.surrounding_position], axis=0) - return get_inertia_tensor( + return get_inertia_tensor_mass_weighted( mass, position, self.SO_r, search_radius=self.search_radius ) @@ -620,7 +643,7 @@ def TotalInertiaTensorReduced(self) -> unyt.unyt_array: return None mass = np.concatenate([self.mass, self.surrounding_mass], axis=0) position = np.concatenate([self.position, self.surrounding_position], axis=0) - return get_inertia_tensor( + return get_inertia_tensor_mass_weighted( mass, position, self.SO_r, search_radius=self.search_radius, reduced=True ) @@ -632,7 +655,9 @@ def TotalInertiaTensorNoniterative(self) -> unyt.unyt_array: """ if self.Mtotpart == 0: return None - return get_inertia_tensor(self.mass, self.position, self.SO_r, max_iterations=1) + return get_inertia_tensor_mass_weighted( + self.mass, self.position, self.SO_r, max_iterations=1 + ) @lazy_property def TotalInertiaTensorReducedNoniterative(self) -> unyt.unyt_array: @@ -642,7 +667,7 @@ def TotalInertiaTensorReducedNoniterative(self) -> unyt.unyt_array: """ if self.Mtotpart == 0: return None - return get_inertia_tensor( + return get_inertia_tensor_mass_weighted( self.mass, self.position, self.SO_r, reduced=True, max_iterations=1 ) @@ -712,9 +737,9 @@ def com_gas(self) -> unyt.unyt_array: """ if self.Mgas == 0: return None - return (self.gas_mass_fraction[:, None] * self.gas_pos).sum( - axis=0 - ) + self.centre + return ( + (self.gas_mass_fraction[:, None] * self.gas_pos).sum(axis=0) + self.centre + ) % self.boxsize @lazy_property def vcom_gas(self) -> unyt.unyt_array: @@ -736,11 +761,11 @@ def compute_Lgas_props(self): self.internal_Lgas, _, self.internal_Mcountrot_gas, - ) = get_angular_momentum_and_kappa_corot( + ) = get_angular_momentum_and_kappa_corot_mass_weighted( self.gas_masses, self.gas_pos, self.gas_vel, - ref_velocity=self.vcom_gas, + reference_velocity=self.vcom_gas, do_counterrot_mass=True, ) @@ -778,7 +803,7 @@ def gas_inertia_tensor(self, **kwargs) -> unyt.unyt_array: surrounding_position = self.surrounding_position[self.surrounding_types == 0] mass = np.concatenate([self.gas_masses, surrounding_mass], axis=0) position = np.concatenate([self.gas_pos, surrounding_position], axis=0) - return get_inertia_tensor( + return get_inertia_tensor_mass_weighted( mass, position, self.SO_r, search_radius=self.search_radius, **kwargs ) @@ -812,7 +837,7 @@ def GasInertiaTensorNoniterative(self) -> unyt.unyt_array: """ if self.Mgas == 0: return None - return get_inertia_tensor( + return get_inertia_tensor_mass_weighted( self.gas_masses, self.gas_pos, self.SO_r, max_iterations=1 ) @@ -824,7 +849,7 @@ def GasInertiaTensorReducedNoniterative(self) -> unyt.unyt_array: """ if self.Mgas == 0: return None - return get_inertia_tensor( + return get_inertia_tensor_mass_weighted( self.gas_masses, self.gas_pos, self.SO_r, reduced=True, max_iterations=1 ) @@ -895,7 +920,7 @@ def dm_inertia_tensor(self, **kwargs) -> unyt.unyt_array: surrounding_position = self.surrounding_position[self.surrounding_types == 1] mass = np.concatenate([self.dm_masses, surrounding_mass], axis=0) position = np.concatenate([self.dm_pos, surrounding_position], axis=0) - return get_inertia_tensor( + return get_inertia_tensor_mass_weighted( mass, position, self.SO_r, search_radius=self.search_radius, **kwargs ) @@ -929,7 +954,7 @@ def DarkMatterInertiaTensorNoniterative(self) -> unyt.unyt_array: """ if self.Mdm == 0: return None - return get_inertia_tensor( + return get_inertia_tensor_mass_weighted( self.dm_masses, self.dm_pos, self.SO_r, max_iterations=1 ) @@ -941,7 +966,7 @@ def DarkMatterInertiaTensorReducedNoniterative(self) -> unyt.unyt_array: """ if self.Mdm == 0: return None - return get_inertia_tensor( + return get_inertia_tensor_mass_weighted( self.dm_masses, self.dm_pos, self.SO_r, reduced=True, max_iterations=1 ) @@ -991,9 +1016,9 @@ def com_star(self) -> unyt.unyt_array: """ if self.Mstar == 0: return None - return (self.star_mass_fraction[:, None] * self.star_pos).sum( - axis=0 - ) + self.centre + return ( + (self.star_mass_fraction[:, None] * self.star_pos).sum(axis=0) + self.centre + ) % self.boxsize @lazy_property def vcom_star(self) -> unyt.unyt_array: @@ -1014,14 +1039,40 @@ def compute_Lstar_props(self): self.internal_Lstar, _, self.internal_Mcountrot_star, - ) = get_angular_momentum_and_kappa_corot( + ) = get_angular_momentum_and_kappa_corot_mass_weighted( self.star_masses, self.star_pos, self.star_vel, - ref_velocity=self.vcom_star, + reference_velocity=self.vcom_star, do_counterrot_mass=True, ) + def compute_Lstar_luminosity_weighted_props(self): + """ + Compute the angular momentum and related properties for star particles, + weighted by their luminosity in a given GAMA band. + + We need this method because Lstar, kappa_star and Mcountrot_star are + computed together. + """ + + # Contrary to compute_Lstar_props, each of the output arrays contains a + # value for each GAMA filter, hence they will have shape (9,) + ( + self.internal_Lstar_luminosity_weighted, + _, + self.internal_Mcountrot_star_luminosity_weighted, + self.internal_Lcountrot_star_luminosity_weighted, + ) = get_angular_momentum_and_kappa_corot_luminosity_weighted( + self.star_masses, + self.star_pos, + self.star_vel, + self.get_dataset("PartType4/Luminosities")[self.star_selection], + reference_velocity=self.vcom_star, + do_counterrot_mass=True, + do_counterrot_luminosity=True, + ) + @lazy_property def Lstar(self) -> unyt.unyt_array: """ @@ -1035,6 +1086,23 @@ def Lstar(self) -> unyt.unyt_array: self.compute_Lstar_props() return self.internal_Lstar + @lazy_property + def Lstar_luminosity_weighted(self) -> unyt.unyt_array: + """ + Luminosity-weighted angular momentum of star particles for different + luminosity bands. NOTE: we reshape the 2D array of shape + (number_luminosity_bans, 3) to a 1D array of shape (number_luminosity_bans * 3,) + + This is computed together with Lstar_luminosity_weighted, kappa_star_luminosity_weighted, + Mcountrot_star_luminosity_weighted and Lcountrot_star_luminosity_weighted + by compute_Lstar_luminosity_weighted_props(). + """ + if self.Nstar == 0: + return None + if not hasattr(self, "internal_Lstar_luminosity_weighted"): + self.compute_Lstar_luminosity_weighted_props() + return self.internal_Lstar_luminosity_weighted.flatten() + @lazy_property def DtoTstar(self) -> unyt.unyt_quantity: """ @@ -1048,6 +1116,47 @@ def DtoTstar(self) -> unyt.unyt_quantity: self.compute_Lstar_props() return 1.0 - 2.0 * self.internal_Mcountrot_star / self.Mstar + @lazy_property + def DtoTstar_luminosity_weighted_luminosity_ratio(self) -> unyt.unyt_array: + """ + Disk to total luminosity ratio for all provided stellar luminosity bands. + Each band uses the luminosity-weighted angular momentum as defined in that + band. + + This is computed together with Lstar_luminosity_weighted, kappa_star_luminosity_weighted, + Mcountrot_star_luminosity_weighted and Lcountrot_star_luminosity_weighted + by compute_Lstar_luminosity_weighted_props(). + """ + if self.Nstar == 0: + return None + if not hasattr(self, "internal_Lcountrot_star_luminosity_weighted"): + self.compute_Lstar_luminosity_weighted_props() + + return ( + 1.0 + - 2.0 + * self.internal_Lcountrot_star_luminosity_weighted + / self.StellarLuminosity + ) + + @lazy_property + def DtoTstar_luminosity_weighted_mass_ratio(self) -> unyt.unyt_array: + """ + Disk to total mass ratio for all provided stellar luminosity bands. + Each band uses the luminosity-weighted angular momentum as defined in that + band. + + This is computed together with Lstar_luminosity_weighted, kappa_star_luminosity_weighted, + Mcountrot_star_luminosity_weighted and Lcountrot_star_luminosity_weighted + by compute_Lstar_luminosity_weighted_props(). + """ + if self.Nstar == 0: + return None + if not hasattr(self, "internal_Mcountrot_star_luminosity_weighted"): + self.compute_Lstar_luminosity_weighted_props() + + return 1.0 - 2.0 * self.internal_Mcountrot_star_luminosity_weighted / self.Mstar + def stellar_inertia_tensor(self, **kwargs) -> unyt.unyt_array: """ Helper function for calculating stellar inertia tensors @@ -1056,7 +1165,7 @@ def stellar_inertia_tensor(self, **kwargs) -> unyt.unyt_array: surrounding_position = self.surrounding_position[self.surrounding_types == 4] mass = np.concatenate([self.star_masses, surrounding_mass], axis=0) position = np.concatenate([self.star_pos, surrounding_position], axis=0) - return get_inertia_tensor( + return get_inertia_tensor_mass_weighted( mass, position, self.SO_r, search_radius=self.search_radius, **kwargs ) @@ -1090,7 +1199,7 @@ def StellarInertiaTensorNoniterative(self) -> unyt.unyt_array: """ if self.Mstar == 0: return None - return get_inertia_tensor( + return get_inertia_tensor_mass_weighted( self.star_masses, self.star_pos, self.SO_r, max_iterations=1 ) @@ -1102,7 +1211,7 @@ def StellarInertiaTensorReducedNoniterative(self) -> unyt.unyt_array: """ if self.Mstar == 0: return None - return get_inertia_tensor( + return get_inertia_tensor_mass_weighted( self.star_masses, self.star_pos, self.SO_r, reduced=True, max_iterations=1 ) @@ -1307,7 +1416,7 @@ def gasOfrac(self) -> unyt.unyt_quantity: * self.get_dataset("PartType0/ElementMassFractions")[self.gas_selection][ :, self.snapshot_datasets.get_column_index( - "ElementMassFractions", "Oxygen" + "PartType0/ElementMassFractions", "Oxygen" ), ] ).sum() / self.Mgas @@ -1326,7 +1435,9 @@ def gasFefrac(self) -> unyt.unyt_quantity: self.gas_masses * self.get_dataset("PartType0/ElementMassFractions")[self.gas_selection][ :, - self.snapshot_datasets.get_column_index("ElementMassFractions", "Iron"), + self.snapshot_datasets.get_column_index( + "PartType0/ElementMassFractions", "Iron" + ), ] ).sum() / self.Mgas @@ -1407,7 +1518,7 @@ def Tgas_cy_weighted_core_excision(self) -> unyt.unyt_quantity: @lazy_property def Tgas_cy_weighted_core_excision_no_agn(self) -> unyt.unyt_quantity: """ - ComptonY-weighted average gas temperature, excluding the inner core and + ComptonY-weighted average gas temperature, excluding the inner core and gas recently heated by AGN. """ if self.Ngas == 0: @@ -1458,7 +1569,7 @@ def Tgas_no_agn_core_excision(self) -> unyt.unyt_quantity: def Tgas_no_cool_core_excision(self) -> unyt.unyt_quantity: """ Mass-weighted average gas temperature, excluding the inner core and cool - gas. + gas. """ if self.Ngas_no_cool_core_excision == 0: return None @@ -1681,7 +1792,7 @@ def compY_unit(self) -> unyt.unyt_quantity: if self.Ngas == 0: return None unit = 1.0 * self.gas_compY.units - new_unit = unit.to(PropertyTable.full_property_list["compY"][3]) + new_unit = unit.to(PropertyTable.full_property_list["compY"].unit) return new_unit @lazy_property @@ -1712,7 +1823,7 @@ def gas_no_agn(self) -> NDArray[bool]: @lazy_property def Xraylum_no_agn(self) -> unyt.unyt_array: """ - Total observer-frame X-ray luminosities of gas particles, excluding + Total observer-frame X-ray luminosities of gas particles, excluding contributions from gas particles that were recently heated by AGN feedback. Note that this is an array, since we have multiple luminosity bands. @@ -1724,7 +1835,7 @@ def Xraylum_no_agn(self) -> unyt.unyt_array: @lazy_property def Xrayphlum_no_agn(self) -> unyt.unyt_array: """ - Total observer-frame X-ray photon luminosities of gas particles, + Total observer-frame X-ray photon luminosities of gas particles, excluding contributions from gas particles that were recently heated by AGN feedback. Note that this is an array, since we have multiple luminosity bands. @@ -1748,7 +1859,7 @@ def Xraylum_restframe_no_agn(self) -> unyt.unyt_array: @lazy_property def Xrayphlum_restframe_no_agn(self) -> unyt.unyt_array: """ - Total rest-frame X-ray photon luminosities of gas particles, + Total rest-frame X-ray photon luminosities of gas particles, excluding contributions from gas particles that were recently heated by AGN feedback. Note that this is an array, since we have multiple luminosity bands. @@ -1824,10 +1935,47 @@ def Xraylum_core_excision(self) -> unyt.unyt_array: return None return self.gas_xraylum[self.gas_selection_core_excision].sum(axis=0) + @lazy_property + def gas_selection_exclude_sat(self): + """ + Mask which removes particles bound to satellites + """ + if self.Ngas == 0: + return None + groupnr_bound = self.get_dataset("PartType0/GroupNr_bound")[self.gas_selection] + return (groupnr_bound == self.index) | (groupnr_bound == -1) + + @lazy_property + def XRayLuminosityNoSat(self) -> unyt.unyt_array: + """ + Total observer-frame X-ray luminosities of gas, + excluding those bound to satellites. + + Note that this is an array, since there are multiple luminosity bands. + """ + if self.Ngas == 0: + return None + return self.gas_xraylum[self.gas_selection_exclude_sat].sum(axis=0) + + @lazy_property + def XRayLuminosityCoreExcisionNoSat(self) -> unyt.unyt_array: + """ + Total observer-frame X-ray luminosities of gas, + excluding contributions from gas particles in the inner core, + and those bound to satellites. + + Note that this is an array, since there are multiple luminosity bands. + """ + + if self.Ngas_core_excision == 0: + return None + mask = self.gas_selection_exclude_sat & self.gas_selection_core_excision + return self.gas_xraylum[mask].sum(axis=0) + @lazy_property def Xrayphlum_core_excision(self) -> unyt.unyt_array: """ - Total observer-frame X-ray photon luminosities of gas particles, + Total observer-frame X-ray photon luminosities of gas particles, excluding contributions from gas particles in the inner core. Note that this is an array, since we have multiple luminosity bands. @@ -1839,7 +1987,7 @@ def Xrayphlum_core_excision(self) -> unyt.unyt_array: @lazy_property def Xraylum_no_agn_core_excision(self) -> unyt.unyt_array: """ - Total observer-frame X-ray luminosities of gas particles, + Total observer-frame X-ray luminosities of gas particles, excluding contributions from gas particles in the inner core and those recently heated by AGN. @@ -1852,7 +2000,7 @@ def Xraylum_no_agn_core_excision(self) -> unyt.unyt_array: @lazy_property def Xrayphlum_no_agn_core_excision(self) -> unyt.unyt_array: """ - Total observer-frame X-ray photon luminosities of gas particles, + Total observer-frame X-ray photon luminosities of gas particles, excluding contributions from gas particles in the inner core and those recently heated by AGN. @@ -1865,7 +2013,7 @@ def Xrayphlum_no_agn_core_excision(self) -> unyt.unyt_array: @lazy_property def Xraylum_restframe_core_excision(self) -> unyt.unyt_array: """ - Total rest-frame X-ray luminosities of gas particles, + Total rest-frame X-ray luminosities of gas particles, excluding contributions from gas particles in the inner core. Note that this is an array, since we have multiple luminosity bands. @@ -1877,7 +2025,7 @@ def Xraylum_restframe_core_excision(self) -> unyt.unyt_array: @lazy_property def Xrayphlum_restframe_core_excision(self) -> unyt.unyt_array: """ - Total rest-frame X-ray photon luminosities of gas particles, + Total rest-frame X-ray photon luminosities of gas particles, excluding contributions from gas particles in the inner core. Note that this is an array, since we have multiple luminosity bands. @@ -1891,7 +2039,7 @@ def Xrayphlum_restframe_core_excision(self) -> unyt.unyt_array: @lazy_property def Xraylum_restframe_no_agn_core_excision(self) -> unyt.unyt_array: """ - Total rest-frame X-ray luminosities of gas particles, + Total rest-frame X-ray luminosities of gas particles, excluding contributions from gas particles in the inner core and those recently heated by AGN. @@ -1906,7 +2054,7 @@ def Xraylum_restframe_no_agn_core_excision(self) -> unyt.unyt_array: @lazy_property def Xrayphlum_restframe_no_agn_core_excision(self) -> unyt.unyt_array: """ - Total rest-frame X-ray photon luminosities of gas particles, + Total rest-frame X-ray photon luminosities of gas particles, excluding contributions from gas particles in the inner core and those recently heated by AGN. @@ -2054,23 +2202,15 @@ def SpectroscopicLikeTemperature_no_agn_core_excision(self) -> unyt.unyt_quantit return numerator / denominator @lazy_property - def Ekin_gas(self) -> unyt.unyt_quantity: + def KineticEnergyGas(self) -> unyt.unyt_quantity: """ Total kinetic energy of gas particles. - - Note that we need to be careful with the units here to avoid numerical - overflow. """ if self.Ngas == 0: return None - # below we need to force conversion to np.float64 before summing up particles - # to avoid overflow - ekin_gas = self.gas_masses * ((self.gas_vel - self.vcom_gas[None, :]) ** 2).sum( - axis=1 - ) - ekin_gas = unyt.unyt_array( - ekin_gas.value, dtype=np.float64, units=ekin_gas.units - ) + v_gas = self.gas_vel - self.vcom[None, :] + v_gas += self.gas_pos * self.cosmology["H"] + ekin_gas = self.gas_masses * (v_gas**2).sum(axis=1) return 0.5 * ekin_gas.sum() @lazy_property @@ -2087,7 +2227,7 @@ def gas_electron_number_densities(self): return self.get_dataset("PartType0/ElectronNumberDensities")[self.gas_selection] @lazy_property - def Etherm_gas(self) -> unyt.unyt_array: + def ThermalEnergyGas(self) -> unyt.unyt_array: """ Total thermal energy of gas particles. @@ -2096,9 +2236,6 @@ def Etherm_gas(self) -> unyt.unyt_array: P = (gamma-1) * rho * u (with gamma=5/3) because some simulations (read: FLAMINGO) do not output the internal energies. - - Note that we need to be careful with the units here to avoid numerical - overflow. """ if self.Ngas == 0: return None @@ -2108,9 +2245,6 @@ def Etherm_gas(self) -> unyt.unyt_array: * self.get_dataset("PartType0/Pressures")[self.gas_selection] / self.gas_densities ) - etherm_gas = unyt.unyt_array( - etherm_gas.value, dtype=np.float64, units=etherm_gas.units - ) return etherm_gas.sum() @lazy_property @@ -2134,7 +2268,7 @@ def DopplerB(self) -> unyt.unyt_quantity: # to make them absolute again before subtracting the observer # position relpos = self.gas_pos + self.centre[None, :] - self.observer_position[None, :] - distance = np.sqrt((relpos ** 2).sum(axis=1)) + distance = np.sqrt((relpos**2).sum(axis=1)) # we need to exclude particles at zero distance # (we assume those have no relative velocity) vr = unyt.unyt_array( @@ -2150,7 +2284,7 @@ def DopplerB(self) -> unyt.unyt_quantity: ) / distance[has_distance] fac = unyt.sigma_thompson / unyt.c volumes = self.gas_masses / self.gas_densities - area = np.pi * self.SO_r ** 2 + area = np.pi * self.SO_r**2 return (fac * ne * vr * (volumes / area)).sum().to("dimensionless") @lazy_property @@ -2204,7 +2338,7 @@ def starOfrac(self) -> unyt.unyt_quantity: * self.get_dataset("PartType4/ElementMassFractions")[self.star_selection][ :, self.snapshot_datasets.get_column_index( - "ElementMassFractions", "Oxygen" + "PartType4/ElementMassFractions", "Oxygen" ), ] ).sum() / self.Mstar @@ -2222,7 +2356,9 @@ def starFefrac(self) -> unyt.unyt_quantity: self.star_masses * self.get_dataset("PartType4/ElementMassFractions")[self.star_selection][ :, - self.snapshot_datasets.get_column_index("ElementMassFractions", "Iron"), + self.snapshot_datasets.get_column_index( + "PartType4/ElementMassFractions", "Iron" + ), ] ).sum() / self.Mstar @@ -2240,23 +2376,16 @@ def StellarLuminosity(self) -> unyt.unyt_array: ) @lazy_property - def Ekin_star(self) -> unyt.unyt_quantity: + def KineticEnergyStars(self) -> unyt.unyt_quantity: """ Total kinetic energy of star particles. - - Note that we need to be careful with units here to avoid numerical - overflow. """ if self.Nstar == 0: return None - # below we need to force conversion to np.float64 before summing up particles - # to avoid overflow - ekin_star = self.star_masses * ( - (self.star_vel - self.vcom_star[None, :]) ** 2 - ).sum(axis=1) - ekin_star = unyt.unyt_array( - ekin_star.value, dtype=np.float64, units=ekin_star.units - ) + + v_star = self.star_vel - self.vcom[None, :] + v_star += self.star_pos * self.cosmology["H"] + ekin_star = self.star_masses * (v_star**2).sum(axis=1) return 0.5 * ekin_star.sum() @lazy_property @@ -2603,7 +2732,7 @@ def concentration_from_R1(R1): c += b * np.log10(R1.to("dimensionless")) ** i # Cap concentration values, as polynomial is only valid for 1 unyt.unyt_array: Centre of mass velocity of all particles within 0.1 R_SO. """ mask = self.radius < 0.1 * self.SO_r - return (self.mass[mask, None] * self.velocity[mask]).sum(axis=0) / self.mass[mask].sum() + if not np.sum(mask): + return None + return (self.mass[mask, None] * self.velocity[mask]).sum(axis=0) / self.mass[ + mask + ].sum() @lazy_property def vcom_thirty_percent(self) -> unyt.unyt_array: @@ -2673,7 +2806,11 @@ def vcom_thirty_percent(self) -> unyt.unyt_array: Centre of mass velocity of all particles within 0.3 R_SO. """ mask = self.radius < 0.3 * self.SO_r - return (self.mass[mask, None] * self.velocity[mask]).sum(axis=0) / self.mass[mask].sum() + if not np.sum(mask): + return None + return (self.mass[mask, None] * self.velocity[mask]).sum(axis=0) / self.mass[ + mask + ].sum() def calculate_flow_rate( self, @@ -2696,7 +2833,7 @@ def calculate_flow_rate( centered at R_SO extends beyond R_SO. """ # Calculate particle radii - radii = np.sqrt(np.sum(positions ** 2, axis=1)) + radii = np.sqrt(np.sum(positions**2, axis=1)) # Specify radii to calculate flow rates for R_fracs = [0.1, 0.3, 1] @@ -2722,6 +2859,11 @@ def calculate_flow_rate( 1: self.vcom, }[R_frac] + # This shouldn't happen, but HaloCentre doesn't actually need to + # be on top of a particle + if vcom is None: + vcom = np.zeros(3) * self.vcom.units + # Calculate radial velocity by subtracting CoM velocity and # taking dot product with r_hat r_hat = positions[r_mask] / np.stack(3 * [radii[r_mask]], axis=1) @@ -2748,7 +2890,7 @@ def calculate_flow_rate( elif flow_type == "energy": # Subtract CoM velocity proper_vel = velocities[r_mask] - vcom[None, :] - kinetic = 0.5 * np.sqrt(np.sum(proper_vel ** 2, axis=1)) ** 2 + kinetic = 0.5 * np.sqrt(np.sum(proper_vel**2, axis=1)) ** 2 flow_rate = ( masses[r_mask] * np.abs(v_r) * (kinetic + internal_energies[r_mask]) ) @@ -2757,7 +2899,7 @@ def calculate_flow_rate( gamma = 5.0 / 3.0 sq_sound_speed = (gamma - 1) * gamma * internal_energies[r_mask] # Calculate momentum flux, second term accounts for pressure - flow_rate = masses[r_mask] * (v_r ** 2 + (sq_sound_speed / gamma)) + flow_rate = masses[r_mask] * (v_r**2 + (sq_sound_speed / gamma)) # Determine total outflow/inflow rates inflow = np.sum(flow_rate[v_r < 0]) / dR @@ -2818,9 +2960,11 @@ def HIMassFlowRate(self) -> unyt.unyt_array: pos = self.get_dataset(f"PartType0/Coordinates") - self.centre[None, :] vel = self.get_dataset("PartType0/Velocities") i_H = self.snapshot_datasets.get_column_index( - "ElementMassFractions", "Hydrogen" + "PartType0/ElementMassFractions", "Hydrogen" + ) + i_HI = self.snapshot_datasets.get_column_index( + "PartType0/SpeciesFractions", "HI" ) - i_HI = self.snapshot_datasets.get_column_index("SpeciesFractions", "HI") mass = ( self.get_dataset("PartType0/Masses") * self.get_dataset("PartType0/ElementMassFractions")[:, i_H] @@ -2842,9 +2986,11 @@ def H2MassFlowRate(self) -> unyt.unyt_array: pos = self.get_dataset(f"PartType0/Coordinates") - self.centre[None, :] vel = self.get_dataset("PartType0/Velocities") i_H = self.snapshot_datasets.get_column_index( - "ElementMassFractions", "Hydrogen" + "PartType0/ElementMassFractions", "Hydrogen" + ) + i_H2 = self.snapshot_datasets.get_column_index( + "PartType0/SpeciesFractions", "H2" ) - i_H2 = self.snapshot_datasets.get_column_index("SpeciesFractions", "H2") # Factor of two needed since we want mass fraction not number mass = ( self.get_dataset("PartType0/Masses") @@ -2867,7 +3013,9 @@ def MetalMassFlowRate(self) -> unyt.unyt_array: # flow through the SO radius pos = self.get_dataset(f"PartType0/Coordinates") - self.centre[None, :] vel = self.get_dataset("PartType0/Velocities") - mass = self.get_dataset("PartType0/Masses") * self.get_dataset("PartType0/MetalMassFractions") + mass = self.get_dataset("PartType0/Masses") * self.get_dataset( + "PartType0/MetalMassFractions" + ) return self.calculate_flow_rate("mass", pos, mass, vel) @@ -2875,7 +3023,7 @@ def _calculate_temperature_constrained_gas_flow_rate( self, flow_type, Tmin=None, Tmax=None ): """ - Helper function for calculating the flow rate of gas particles + Helper function for calculating the flow rate of gas particles masked based on their temperature. """ @@ -3081,9 +3229,10 @@ class SOProperties(HaloProperty): Each property should have a corresponding method/property/lazy_property in the SOParticleData class above. """ - property_list = [ - (prop, *PropertyTable.full_property_list[prop]) - for prop in [ + base_halo_type = "SOProperties" + property_list = { + name: PropertyTable.full_property_list[name] + for name in [ "r", "Mtot", "Ngas", @@ -3129,13 +3278,13 @@ class SOProperties(HaloProperty): "com", "vcom", "Vmax_soft", + "R_vmax_soft", "Mfrac_satellites", "Mfrac_external", "Mgas", "Lgas", "com_gas", "vcom_gas", - # "veldisp_matrix_gas", "gasmetalfrac", "Mhotgas", "Tgas", @@ -3156,20 +3305,19 @@ class SOProperties(HaloProperty): "Xraylum_restframe_no_agn", "Xrayphlum_restframe_no_agn", "compY_no_agn", - "Ekin_gas", - "Etherm_gas", + "KineticEnergyGas", + "ThermalEnergyGas", "Mdm", "Ldm", - # "veldisp_matrix_dm", "Mstar", "com_star", "vcom_star", - # "veldisp_matrix_star", "Lstar", + "Lstar_luminosity_weighted", "Mstar_init", "starmetalfrac", "StellarLuminosity", - "Ekin_star", + "KineticEnergyStars", "Lbaryons", "Mbh_dynamical", "Mbh_subgrid", @@ -3207,6 +3355,8 @@ class SOProperties(HaloProperty): "gasFefrac", "DtoTgas", "DtoTstar", + "DtoTstar_luminosity_weighted_luminosity_ratio", + "DtoTstar_luminosity_weighted_mass_ratio", "starOfrac", "starFefrac", "gasmetalfrac_SF", @@ -3215,7 +3365,7 @@ class SOProperties(HaloProperty): "concentration_dmo_unsoft", "concentration_dmo_soft", ] - ] + } def __init__( self, @@ -3259,8 +3409,8 @@ def __init__( super().__init__(cellgrid) - self.property_mask = parameters.get_property_mask( - "SOProperties", [prop[1] for prop in self.property_list] + self.property_filters = parameters.get_property_filters( + "SOProperties", [prop.name for prop in self.property_list.values()] ) if not type in ["mean", "crit", "physical", "BN98"]: @@ -3272,7 +3422,9 @@ def __init__( self.category_filter = category_filter self.snapshot_datasets = cellgrid.snapshot_datasets self.halo_filter = halo_filter + self.record_timings = parameters.record_property_timings self.observer_position = cellgrid.observer_position + self.boxsize = cellgrid.boxsize self.cosmology = {} # in the neutrino model, the mean neutrino density is implicitly @@ -3288,7 +3440,7 @@ def __init__( / cellgrid.cosmology["H [internal units]"] ) ** 2 - / cellgrid.a ** 3 + / cellgrid.a**3 ) # We need the following for inflow/outflow calculations @@ -3396,14 +3548,17 @@ def __init__( ], "PartType6": ["Coordinates", "Masses", "Weights"], } - for prop in self.property_list: - outputname = prop[1] - if not self.property_mask[outputname]: + # add additional particle properties based on the selected halo + # properties in the parameter file + for name, prop in self.property_list.items(): + outputname = prop.name + # Skip if this property is disabled in the parameter file + if not self.property_filters[outputname]: continue - is_dmo = prop[8] - if self.category_filter.dmo and not is_dmo: + # Skip non-DMO properties when in DMO run mode + if self.category_filter.dmo and not prop.dmo_property: continue - partprops = prop[9] + partprops = prop.particle_properties for partprop in partprops: pgroup, dset = parameters.get_particle_property(partprop) if not pgroup in self.particle_properties: @@ -3435,26 +3590,27 @@ def calculate( registry = input_halo["cofp"].units.registry SO = {} + timings = {} + # declare all the variables we will compute # we set them to 0 in case a particular variable cannot be computed # all variables are defined with physical units and an appropriate dtype # we need to use the custom unit registry so that everything can be converted # back to snapshot units in the end - for prop in self.property_list: - outputname = prop[1] - # skip properties that are masked - if not self.property_mask[outputname]: + for name, prop in self.property_list.items(): + outputname = prop.name + # Skip if this property is disabled in the parameter file + filter_name = self.property_filters[outputname] + if not filter_name: continue - # skip non-DMO properties in DMO run mode - is_dmo = prop[8] - if self.category_filter.dmo and not is_dmo: + # Skip non-DMO properties when in DMO run mode + if self.category_filter.dmo and not prop.dmo_property: continue - name = prop[0] - shape = prop[2] - dtype = prop[3] - unit = unyt.Unit(prop[4], registry=registry) - physical = prop[10] - a_exponent = prop[11] + shape = prop.shape + dtype = prop.dtype + unit = unyt.Unit(prop.unit, registry=registry) + physical = prop.output_physical + a_exponent = prop.a_scale_exponent if shape > 1: val = [0] * shape else: @@ -3483,6 +3639,7 @@ def calculate( self.virial_definition, search_radius, self.cosmology, + self.boxsize, ) # we need to make sure the physical radius uses the correct unit @@ -3500,25 +3657,23 @@ def calculate( ) if SO_exists: - for prop in self.property_list: - outputname = prop[1] - # skip properties that are masked - if not self.property_mask[outputname]: + for name, prop in self.property_list.items(): + outputname = prop.name + # Skip if this property is disabled in the parameter file + filter_name = self.property_filters[outputname] + if not filter_name: continue - # skip non-DMO properties in DMO run mode - is_dmo = prop[8] - if do_calculation["DMO"] and not is_dmo: + # Skip non-DMO properties when in DMO run mode + if self.category_filter.dmo and not prop.dmo_property: continue - name = prop[0] - dtype = prop[3] - unit = prop[4] - unit = unyt.Unit(prop[4], registry=registry) - category = prop[6] - physical = prop[10] - a_exponent = prop[11] + dtype = prop.dtype + unit = unyt.Unit(prop.unit, registry=registry) + physical = prop.output_physical + a_exponent = prop.a_scale_exponent if not physical: unit = unit * unyt.Unit("a", registry=registry) ** a_exponent - if do_calculation[category]: + if do_calculation[filter_name]: + t0_calc = time.time() val = getattr(part_props, name) if val is not None: assert ( @@ -3536,49 +3691,62 @@ def calculate( registry=registry, ) else: - err = f'Overflow for halo {input_halo["index"]} when' + err = f'Overflow for halo {input_halo["index"]} when ' err += f"calculating {name} in SO_properties" assert np.max(np.abs(val.to(unit).value)) < float( "inf" ), err SO[name] += val + timings[name] = time.time() - t0_calc # Return value should be a dict containing unyt_arrays and descriptions. # The dict keys will be used as HDF5 dataset names in the output. - for prop in self.property_list: - outputname = prop[1] - # skip properties that are masked - if not self.property_mask[outputname]: + for name, prop in self.property_list.items(): + outputname = prop.name + # Skip if this property is disabled in the parameter file + if not self.property_filters[outputname]: continue - # skip non-DMO properties in DMO run mode - is_dmo = prop[8] - if self.category_filter.dmo and not is_dmo: + # Skip non-DMO properties when in DMO run mode + if self.category_filter.dmo and not prop.dmo_property: continue - name = prop[0] - description = prop[5] - physical = prop[10] - a_exponent = prop[11] halo_result.update( { - f"SO/{self.SO_name}/{outputname}": ( + f"{self.group_name}/{outputname}": ( SO[name], - description.format( + prop.description.format( label=self.label, core_excision=self.core_excision_string ), - physical, - a_exponent, + prop.output_physical, + prop.a_scale_exponent, ) } ) + if self.record_timings: + arr = unyt.unyt_array( + timings.get(name, 0), + dtype=np.float32, + units=unyt.dimensionless, + registry=registry, + ) + halo_result.update( + { + f"{self.group_name}/{outputname}_time": ( + arr, + "Time taken in seconds", + True, + None, + ) + } + ) return class CoreExcisedSOProperties(SOProperties): # Add the extra core excised properties we want from the table - property_list = SOProperties.property_list + [ - (prop, *PropertyTable.full_property_list[prop]) - for prop in [ + property_list = SOProperties.property_list | { + name: PropertyTable.full_property_list[name] + for name in [ "Tgas_core_excision", "Tgas_no_cool_core_excision", "Tgas_no_agn_core_excision", @@ -3588,6 +3756,8 @@ class CoreExcisedSOProperties(SOProperties): "SpectroscopicLikeTemperature_core_excision", "SpectroscopicLikeTemperature_no_agn_core_excision", "Xraylum_core_excision", + "XRayLuminosityNoSat", + "XRayLuminosityCoreExcisionNoSat", "Xraylum_no_agn_core_excision", "Xrayphlum_core_excision", "Xrayphlum_no_agn_core_excision", @@ -3596,7 +3766,7 @@ class CoreExcisedSOProperties(SOProperties): "Xrayphlum_restframe_core_excision", "Xrayphlum_restframe_no_agn_core_excision", ] - ] + } def __init__( self, @@ -3636,7 +3806,7 @@ class RadiusMultipleSOProperties(SOProperties): # since the halo_result dictionary contains the name of the dataset as it # appears in the output, we have to get that name from the property table # to access the radius - radius_name = PropertyTable.full_property_list["r"][0] + radius_name = PropertyTable.full_property_list["r"].name def __init__( self, @@ -3754,444 +3924,3 @@ def calculate( super().calculate(input_halo, search_radius, data, halo_result) return - - -def test_SO_properties_random_halo(): - """ - Unit test for the SO property calculation. - - We generate 100 random halos and check that the various SO halo - calculations return the expected results and do not lead to any - errors. - """ - from dummy_halo_generator import DummyHaloGenerator - - dummy_halos = DummyHaloGenerator(4251) - gas_filter = dummy_halos.get_recently_heated_gas_filter() - cat_filter = CategoryFilter(dummy_halos.get_filters({"general": 100})) - parameters = ParameterFile( - parameter_dictionary={ - "aliases": { - "PartType0/ElementMassFractions": "PartType0/SmoothedElementMassFractions", - "PartType4/ElementMassFractions": "PartType4/SmoothedElementMassFractions", - "PartType0/XrayLuminositiesRestframe": "PartType0/XrayLuminositiesRestframe", - "PartType0/XrayPhotonLuminositiesRestframe": "PartType0/XrayPhotonLuminositiesRestframe", - } - } - ) - dummy_halos.get_cell_grid().snapshot_datasets.setup_aliases( - parameters.get_aliases() - ) - parameters.get_halo_type_variations( - "SOProperties", - { - "50_kpc": {"value": 50.0, "type": "physical"}, - "2500_mean": {"value": 2500.0, "type": "mean"}, - "2500_crit": {"value": 2500.0, "type": "crit"}, - "BN98": {"value": 0.0, "type": "BN98"}, - "5xR2500_mean": {"value": 2500.0, "type": "mean", "radius_multiple": 5.0}, - }, - ) - - property_calculator_50kpc = SOProperties( - dummy_halos.get_cell_grid(), - parameters, - gas_filter, - cat_filter, - "basic", - 50.0, - "physical", - ) - property_calculator_2500mean = SOProperties( - dummy_halos.get_cell_grid(), - parameters, - gas_filter, - cat_filter, - "basic", - 2500.0, - "mean", - ) - property_calculator_2500crit = SOProperties( - dummy_halos.get_cell_grid(), - parameters, - gas_filter, - cat_filter, - "basic", - 2500.0, - "crit", - ) - property_calculator_BN98 = SOProperties( - dummy_halos.get_cell_grid(), - parameters, - gas_filter, - cat_filter, - "basic", - 0.0, - "BN98", - ) - property_calculator_5x2500mean = RadiusMultipleSOProperties( - dummy_halos.get_cell_grid(), - parameters, - gas_filter, - cat_filter, - "basic", - 2500.0, - 5.0, - "mean", - ) - - # Create a filter that no halos will satisfy - fail_filter = CategoryFilter(dummy_halos.get_filters({"general": 10000000})) - property_calculator_filter_test = SOProperties( - dummy_halos.get_cell_grid(), - parameters, - gas_filter, - fail_filter, - "general", - 200.0, - "crit", - ) - property_calculator_filter_test.SO_name = "filter_test" - - for i in range(100): - ( - input_halo, - data, - rmax, - Mtot, - Npart, - particle_numbers, - ) = dummy_halos.get_random_halo([2, 10, 100, 1000, 10000], has_neutrinos=True) - halo_result_template = dummy_halos.get_halo_result_template(particle_numbers) - rho_ref = Mtot / (4.0 / 3.0 * np.pi * rmax ** 3) - - # force the SO radius to be outside the search sphere and check that - # we get a SearchRadiusTooSmallError - property_calculator_2500mean.reference_density = 0.01 * rho_ref - property_calculator_2500crit.reference_density = 0.01 * rho_ref - property_calculator_BN98.reference_density = 0.01 * rho_ref - for prop_calc in [ - property_calculator_2500mean, - property_calculator_2500crit, - property_calculator_BN98, - ]: - fail = False - try: - halo_result = dict(halo_result_template) - prop_calc.calculate(input_halo, rmax, data, halo_result) - except SearchRadiusTooSmallError: - fail = True - # 1 particle halos don't fail, since we always assume that the first - # particle is at the centre of potential (which means we exclude it - # in the SO calculation) - # non-centrals don't fail, since we do not calculate any SO - # properties and simply return zeros in this case - - # TODO: This can fail due to how we calculate the SO if the - # first particle is a neutrino with negative mass. In that case - # we linearly interpolate the mass of the first non-negative particle - # outwards. - assert (Npart == 1) or input_halo["is_central"] == 0 or fail - - # force the radius multiple to trip over not having computed the - # required radius - fail = False - try: - halo_result = dict(halo_result_template) - property_calculator_5x2500mean.calculate( - input_halo, rmax, data, halo_result - ) - except RuntimeError: - fail = True - assert fail - - # force the radius multiple to trip over the search radius - fail = False - try: - halo_result = dict(halo_result_template) - halo_result.update( - { - f"SO/2500_mean/{property_calculator_5x2500mean.radius_name}": ( - 0.1 * rmax, - "Dummy value.", - ) - } - ) - property_calculator_5x2500mean.calculate( - input_halo, 0.2 * rmax, data, halo_result - ) - except SearchRadiusTooSmallError: - fail = True - assert fail - - # force the SO radius to be within the search sphere - property_calculator_2500mean.reference_density = 2.0 * rho_ref - property_calculator_2500crit.reference_density = 2.0 * rho_ref - property_calculator_BN98.reference_density = 2.0 * rho_ref - - for SO_name, prop_calc in [ - ("50_kpc", property_calculator_50kpc), - ("2500_mean", property_calculator_2500mean), - ("2500_crit", property_calculator_2500crit), - ("BN98", property_calculator_BN98), - ("5xR_2500_mean", property_calculator_5x2500mean), - ("filter_test", property_calculator_filter_test), - ]: - halo_result = dict(halo_result_template) - # make sure the radius multiple is found this time - if SO_name == "5xR_2500_mean": - halo_result[ - f"SO/2500_mean/{property_calculator_5x2500mean.radius_name}" - ] = (0.1 * rmax, "Dummy value to force correct behaviour") - input_data = {} - for ptype in prop_calc.particle_properties: - if ptype in data: - input_data[ptype] = {} - for dset in prop_calc.particle_properties[ptype]: - input_data[ptype][dset] = data[ptype][dset] - # Adding Restframe luminosties as they are calculated in halo_tasks - if "PartType0" in input_data: - for dset in [ - "XrayLuminositiesRestframe", - "XrayPhotonLuminositiesRestframe", - ]: - input_data["PartType0"][dset] = data["PartType0"][dset] - input_data["PartType0"][dset] = data["PartType0"][dset] - halo_result[ - f"SO/2500_mean/{property_calculator_5x2500mean.radius_name}" - ] = (0.1 * rmax, "Dummy value to force correct behaviour") - input_halo_copy = input_halo.copy() - input_data_copy = input_data.copy() - prop_calc.calculate(input_halo, rmax, input_data, halo_result) - # make sure the calculation does not change the input - assert input_halo_copy == input_halo - assert input_data_copy == input_data - - for prop in prop_calc.property_list: - outputname = prop[1] - size = prop[2] - dtype = prop[3] - unit_string = prop[4] - full_name = f"SO/{SO_name}/{outputname}" - assert full_name in halo_result - result = halo_result[full_name][0] - assert (len(result.shape) == 0 and size == 1) or result.shape[0] == size - assert result.dtype == dtype - unit = unyt.Unit(unit_string, registry=dummy_halos.unit_registry) - assert result.units.same_dimensions_as(unit.units) - - # Check properties were not calculated for filtered halos - if SO_name == "filter_test": - for prop in prop_calc.property_list: - outputname = prop[1] - size = prop[2] - full_name = f"SO/{SO_name}/{outputname}" - assert np.all(halo_result[full_name][0].value == np.zeros(size)) - - # Now test the calculation for each property individually, to make sure that - # all properties read all the datasets they require - all_parameters = parameters.get_parameters() - for property in all_parameters["SOProperties"]["properties"]: - print(f"Testing only {property}...") - single_property = dict(all_parameters) - for other_property in all_parameters["SOProperties"]["properties"]: - single_property["SOProperties"]["properties"][other_property] = ( - other_property == property - ) or other_property.startswith("NumberOf") - single_parameters = ParameterFile(parameter_dictionary=single_property) - - property_calculator_50kpc = SOProperties( - dummy_halos.get_cell_grid(), - single_parameters, - gas_filter, - cat_filter, - "basic", - 50.0, - "physical", - ) - property_calculator_2500mean = SOProperties( - dummy_halos.get_cell_grid(), - single_parameters, - gas_filter, - cat_filter, - "basic", - 2500.0, - "mean", - ) - property_calculator_2500crit = SOProperties( - dummy_halos.get_cell_grid(), - single_parameters, - gas_filter, - cat_filter, - "basic", - 2500.0, - "crit", - ) - property_calculator_BN98 = SOProperties( - dummy_halos.get_cell_grid(), - single_parameters, - gas_filter, - cat_filter, - "basic", - 0.0, - "BN98", - ) - property_calculator_5x2500mean = RadiusMultipleSOProperties( - dummy_halos.get_cell_grid(), - single_parameters, - gas_filter, - cat_filter, - "basic", - 2500.0, - 5.0, - "mean", - ) - - halo_result_template = dummy_halos.get_halo_result_template(particle_numbers) - rho_ref = Mtot / (4.0 / 3.0 * np.pi * rmax ** 3) - - # force the SO radius to be within the search sphere - property_calculator_2500mean.reference_density = 2.0 * rho_ref - property_calculator_2500crit.reference_density = 2.0 * rho_ref - property_calculator_BN98.reference_density = 2.0 * rho_ref - - for SO_name, prop_calc in [ - ("50_kpc", property_calculator_50kpc), - ("2500_mean", property_calculator_2500mean), - ("2500_crit", property_calculator_2500crit), - ("BN98", property_calculator_BN98), - ("5xR_2500_mean", property_calculator_5x2500mean), - ]: - - halo_result = dict(halo_result_template) - # make sure the radius multiple is found this time - if SO_name == "5xR_2500_mean": - halo_result[ - f"SO/2500_mean/{property_calculator_5x2500mean.radius_name}" - ] = (0.1 * rmax, "Dummy value to force correct behaviour") - input_data = {} - for ptype in prop_calc.particle_properties: - if ptype in data: - input_data[ptype] = {} - for dset in prop_calc.particle_properties[ptype]: - input_data[ptype][dset] = data[ptype][dset] - # Adding Restframe luminosties as they are calculated in halo_tasks - if "PartType0" in input_data: - for dset in [ - "XrayLuminositiesRestframe", - "XrayPhotonLuminositiesRestframe", - ]: - input_data["PartType0"][dset] = data["PartType0"][dset] - input_data["PartType0"][dset] = data["PartType0"][dset] - input_halo_copy = input_halo.copy() - input_data_copy = input_data.copy() - prop_calc.calculate(input_halo, rmax, input_data, halo_result) - # make sure the calculation does not change the input - assert input_halo_copy == input_halo - assert input_data_copy == input_data - - for prop in prop_calc.property_list: - outputname = prop[1] - if not outputname == property: - continue - size = prop[2] - dtype = prop[3] - unit_string = prop[4] - physical = prop[10] - a_exponent = prop[11] - full_name = f"SO/{SO_name}/{outputname}" - assert full_name in halo_result - result = halo_result[full_name][0] - assert (len(result.shape) == 0 and size == 1) or result.shape[0] == size - assert result.dtype == dtype - unit = unyt.Unit(unit_string, registry=dummy_halos.unit_registry) - if not physical: - unit = ( - unit - * unyt.Unit("a", registry=dummy_halos.unit_registry) - ** a_exponent - ) - assert result.units == unit.units - - dummy_halos.get_cell_grid().snapshot_datasets.print_dataset_log() - - -def calculate_SO_properties_nfw_halo(seed, num_part, c): - """ - Generates a halo with an NFW profile, and calculates SO properties for it - """ - from dummy_halo_generator import DummyHaloGenerator - - dummy_halos = DummyHaloGenerator(seed) - gas_filter = dummy_halos.get_recently_heated_gas_filter() - cat_filter = CategoryFilter(dummy_halos.get_filters({"general": 100})) - parameters = ParameterFile( - parameter_dictionary={ - "aliases": { - "PartType0/ElementMassFractions": "PartType0/SmoothedElementMassFractions", - "PartType4/ElementMassFractions": "PartType4/SmoothedElementMassFractions", - "PartType0/XrayLuminositiesRestframe": "PartType0/XrayLuminositiesRestframe", - "PartType0/XrayPhotonLuminositiesRestframe": "PartType0/XrayPhotonLuminositiesRestframe", - } - } - ) - dummy_halos.get_cell_grid().snapshot_datasets.setup_aliases( - parameters.get_aliases() - ) - parameters.get_halo_type_variations( - "SOProperties", - { - "50_kpc": {"value": 50.0, "type": "physical"}, - "2500_mean": {"value": 2500.0, "type": "mean"}, - "2500_crit": {"value": 2500.0, "type": "crit"}, - "BN98": {"value": 0.0, "type": "BN98"}, - "5xR2500_mean": {"value": 2500.0, "type": "mean", "radius_multiple": 5.0}, - }, - ) - - property_calculator_200crit = SOProperties( - dummy_halos.get_cell_grid(), - parameters, - gas_filter, - cat_filter, - "basic", - 200.0, - "crit", - ) - - (input_halo, data, rmax, Mtot, Npart, particle_numbers) = dummy_halos.gen_nfw_halo( - 100, c, num_part - ) - - halo_result_template = dummy_halos.get_halo_result_template(particle_numbers) - - property_calculator_200crit.cosmology["nu_density"] *= 0 - property_calculator_200crit.calculate(input_halo, rmax, data, halo_result_template) - - return halo_result_template - - -def test_concentration_nfw_halo(): - """ - Test if the calculated concentration is close to the input value. - Only tests halos with for 10000 particles. - Fails due to noise for small particle numbers. - """ - n_part = 10000 - for seed in range(10): - for concentration in [5, 10]: - halo_result = calculate_SO_properties_nfw_halo(seed, n_part, concentration) - calculated = halo_result["SO/200_crit/Concentration"][0] - delta = np.abs(calculated - concentration) / concentration - assert delta < 0.1 - - -if __name__ == "__main__": - """ - Standalone mode for running tests. - """ - print("Calling test_SO_properties_random_halo()...") - test_SO_properties_random_halo() - print("Calling test_concentration_nfw_halo()...") - test_concentration_nfw_halo() - print("Tests passed.") diff --git a/aperture_properties.py b/SOAP/particle_selection/aperture_properties.py similarity index 70% rename from aperture_properties.py rename to SOAP/particle_selection/aperture_properties.py index fd482858..6a07f6cb 100644 --- a/aperture_properties.py +++ b/SOAP/particle_selection/aperture_properties.py @@ -133,30 +133,38 @@ def MetalFracStar(self): very messy and complex. But it is in fact quite neat and powerful. """ +import time import numpy as np +from numpy.typing import NDArray +from typing import Dict, List, Tuple import unyt -from halo_properties import HaloProperty, SearchRadiusTooSmallError -from dataset_names import mass_dataset -from half_mass_radius import get_half_mass_radius -from kinematic_properties import ( +from .halo_properties import HaloProperty, SearchRadiusTooSmallError +from SOAP.core.dataset_names import mass_dataset +from SOAP.property_calculation.half_mass_radius import ( + get_half_mass_radius, + get_half_light_radius, +) +from SOAP.property_calculation.kinematic_properties import ( get_velocity_dispersion_matrix, get_angular_momentum, - get_angular_momentum_and_kappa_corot, + get_angular_momentum_and_kappa_corot_mass_weighted, + get_angular_momentum_and_kappa_corot_luminosity_weighted, get_vmax, ) - -from swift_cells import SWIFTCellGrid -from recently_heated_gas_filter import RecentlyHeatedGasFilter -from stellar_age_calculator import StellarAgeCalculator -from cold_dense_gas_filter import ColdDenseGasFilter -from property_table import PropertyTable -from lazy_properties import lazy_property -from category_filter import CategoryFilter -from parameter_file import ParameterFile -from snapshot_datasets import SnapshotDatasets -from typing import Dict, List, Tuple -from numpy.typing import NDArray +from SOAP.property_calculation.inertia_tensors import ( + get_inertia_tensor_mass_weighted, + get_inertia_tensor_luminosity_weighted, +) +from SOAP.core.swift_cells import SWIFTCellGrid +from SOAP.property_calculation.stellar_age_calculator import StellarAgeCalculator +from SOAP.particle_filter.cold_dense_gas_filter import ColdDenseGasFilter +from SOAP.particle_filter.recently_heated_gas_filter import RecentlyHeatedGasFilter +from SOAP.property_table import PropertyTable +from SOAP.core.lazy_properties import lazy_property +from SOAP.core.category_filter import CategoryFilter +from SOAP.core.parameter_file import ParameterFile +from SOAP.core.snapshot_datasets import SnapshotDatasets class ApertureParticleData: @@ -195,6 +203,8 @@ def __init__( cold_dense_gas_filter: ColdDenseGasFilter, snapshot_datasets: SnapshotDatasets, softening_of_parttype: unyt.unyt_array, + boxsize: unyt.unyt_quantity, + cosmology: dict, ): """ Constructor. @@ -225,6 +235,10 @@ def __init__( appropriate aliases and column names. - softening_of_parttype: unyt.unyt_array Softening length of each particle types + - boxsize: unyt.unyt_quantity + Boxsize for correcting periodic boundary conditions + - cosmology: dict + Cosmological parameters required for SO calculation """ self.input_halo = input_halo self.data = data @@ -236,6 +250,8 @@ def __init__( self.cold_dense_gas_filter = cold_dense_gas_filter self.snapshot_datasets = snapshot_datasets self.softening_of_parttype = softening_of_parttype + self.boxsize = boxsize + self.cosmology = cosmology self.compute_basics() def get_dataset(self, name: str) -> unyt.unyt_array: @@ -389,6 +405,13 @@ def mass_gas(self) -> unyt.unyt_array: """ return self.mass[self.type == 0] + @lazy_property + def mass_dust(self) -> unyt.unyt_array: + """ + Dust mass of the gas particles. + """ + return self.gas_total_dust_mass_fractions * self.mass[self.type == 0] + @lazy_property def mass_dm(self) -> unyt.unyt_array: """ @@ -593,7 +616,7 @@ def star_mass_O(self) -> unyt.unyt_array: self.star_element_fractions[ :, self.snapshot_datasets.get_column_index( - "ElementMassFractions", "Oxygen" + "PartType4/ElementMassFractions", "Oxygen" ), ] * self.mass_star @@ -610,7 +633,7 @@ def star_mass_Mg(self) -> unyt.unyt_array: self.star_element_fractions[ :, self.snapshot_datasets.get_column_index( - "ElementMassFractions", "Magnesium" + "PartType4/ElementMassFractions", "Magnesium" ), ] * self.mass_star @@ -626,7 +649,9 @@ def star_mass_Fe(self) -> unyt.unyt_array: return ( self.star_element_fractions[ :, - self.snapshot_datasets.get_column_index("ElementMassFractions", "Iron"), + self.snapshot_datasets.get_column_index( + "PartType4/ElementMassFractions", "Iron" + ), ] * self.mass_star ) @@ -709,7 +734,8 @@ def stellar_age_lw(self) -> unyt.unyt_quantity: if self.Nstar == 0: return None Lr = self.stellar_luminosities[ - :, self.snapshot_datasets.get_column_index("Luminosities", "GAMA_r") + :, + self.snapshot_datasets.get_column_index("PartType4/Luminosities", "GAMA_r"), ] Lrtot = Lr.sum() if Lrtot == 0: @@ -1061,7 +1087,9 @@ def com(self) -> unyt.unyt_array: """ if self.Mtot == 0: return None - return (self.mass_fraction[:, None] * self.position).sum(axis=0) + self.centre + return ( + (self.mass_fraction[:, None] * self.position).sum(axis=0) + self.centre + ) % self.boxsize @lazy_property def vcom(self) -> unyt.unyt_array: @@ -1072,26 +1100,6 @@ def vcom(self) -> unyt.unyt_array: return None return (self.mass_fraction[:, None] * self.velocity).sum(axis=0) - @lazy_property - def spin_parameter(self) -> unyt.unyt_quantity: - """ - Spin parameter of all particles in the aperture. - - Computed as in Bullock et al. (2021): - lambda = |Ltot| / (sqrt(2) * M * v_max * R) - """ - if self.Mtot == 0: - return None - soft_r = np.maximum(self.softening, self.radius) - _, vmax_soft = get_vmax(self.mass, soft_r) - if vmax_soft > 0: - vrel = self.velocity - self.vcom[None, :] - Ltot = np.linalg.norm( - (self.mass[:, None] * np.cross(self.position, vrel)).sum(axis=0) - ) - return Ltot / (np.sqrt(2.0) * self.Mtot * self.aperture_radius * vmax_soft) - return None - @lazy_property def gas_mass_fraction(self) -> unyt.unyt_array: """ @@ -1122,11 +1130,11 @@ def compute_Lgas_props(self): self.internal_Lgas, self.internal_kappa_gas, self.internal_Mcountrot_gas, - ) = get_angular_momentum_and_kappa_corot( + ) = get_angular_momentum_and_kappa_corot_mass_weighted( self.mass_gas, self.pos_gas, self.vel_gas, - ref_velocity=self.vcom_gas, + reference_velocity=self.vcom_gas, do_counterrot_mass=True, ) @@ -1184,18 +1192,15 @@ def veldisp_matrix_gas(self) -> unyt.unyt_array: ) @lazy_property - def Ekin_gas(self) -> unyt.unyt_quantity: + def KineticEnergyGas(self) -> unyt.unyt_quantity: """ Kinetic energy of the gas. """ if self.Mgas == 0: return None - # below we need to force conversion to np.float64 before summing - # up particles to avoid overflow - ekin_gas = self.mass_gas * ((self.vel_gas - self.vcom_gas) ** 2).sum(axis=1) - ekin_gas = unyt.unyt_array( - ekin_gas.value, dtype=np.float64, units=ekin_gas.units - ) + v_gas = self.vel_gas - self.vcom[None, :] + v_gas += self.pos_gas * self.cosmology["H"] + ekin_gas = self.mass_gas * (v_gas**2).sum(axis=1) return 0.5 * ekin_gas.sum() @lazy_property @@ -1239,6 +1244,17 @@ def Ldm(self) -> unyt.unyt_array: self.mass_dm, self.pos_dm, self.vel_dm, ref_velocity=self.vcom_dm ) + @lazy_property + def com_dm(self) -> unyt.unyt_array: + """ + Centre of mass of DM particles in the aperture. + """ + if self.Mdm == 0: + return None + return ( + (self.dm_mass_fraction[:, None] * self.pos_dm).sum(axis=0) + self.centre + ) % self.boxsize + @lazy_property def com_star(self) -> unyt.unyt_array: """ @@ -1246,9 +1262,9 @@ def com_star(self) -> unyt.unyt_array: """ if self.Mstar == 0: return None - return (self.star_mass_fraction[:, None] * self.pos_star).sum( - axis=0 - ) + self.centre + return ( + (self.star_mass_fraction[:, None] * self.pos_star).sum(axis=0) + self.centre + ) % self.boxsize @lazy_property def vcom_star(self) -> unyt.unyt_array: @@ -1270,14 +1286,40 @@ def compute_Lstar_props(self): self.internal_Lstar, self.internal_kappa_star, self.internal_Mcountrot_star, - ) = get_angular_momentum_and_kappa_corot( + ) = get_angular_momentum_and_kappa_corot_mass_weighted( self.mass_star, self.pos_star, self.vel_star, - ref_velocity=self.vcom_star, + reference_velocity=self.vcom_star, do_counterrot_mass=True, ) + def compute_Lstar_luminosity_weighted_props(self): + """ + Compute the angular momentum and related properties for star particles, + weighted by their luminosity in a given GAMA band. + + We need this method because Lstar, kappa_star and Mcountrot_star are + computed together. + """ + + # Contrary to compute_Lstar_props, each of the output arrays contains a + # value for each GAMA filter, hence they will have shape (9,) + ( + self.internal_Lstar_luminosity_weighted, + self.internal_kappa_star_luminosity_weighted, + self.internal_Mcountrot_star_luminosity_weighted, + self.internal_Lcountrot_star_luminosity_weighted, + ) = get_angular_momentum_and_kappa_corot_luminosity_weighted( + self.mass_star, + self.pos_star, + self.vel_star, + self.stellar_luminosities, + reference_velocity=self.vcom_star, + do_counterrot_mass=True, + do_counterrot_luminosity=True, + ) + @lazy_property def Lstar(self) -> unyt.unyt_array: """ @@ -1292,6 +1334,23 @@ def Lstar(self) -> unyt.unyt_array: self.compute_Lstar_props() return self.internal_Lstar + @lazy_property + def Lstar_luminosity_weighted(self) -> unyt.unyt_array: + """ + Luminosity-weighted angular momentum of star particles for different + luminosity bands. NOTE: we reshape the 2D array of shape + (number_luminosity_bans, 3) to a 1D array of shape (number_luminosity_bans * 3,) + + This is computed together with Lstar_luminosity_weighted, kappa_star_luminosity_weighted, + Mcountrot_star_luminosity_weighted and Lcountrot_star_luminosity_weighted + by compute_Lstar_luminosity_weighted_props(). + """ + if self.Nstar == 0: + return None + if not hasattr(self, "internal_Lstar_luminosity_weighted"): + self.compute_Lstar_luminosity_weighted_props() + return self.internal_Lstar_luminosity_weighted.flatten() + @lazy_property def kappa_corot_star(self) -> unyt.unyt_quantity: """ @@ -1306,6 +1365,23 @@ def kappa_corot_star(self) -> unyt.unyt_quantity: self.compute_Lstar_props() return self.internal_kappa_star + @lazy_property + def kappa_corot_star_luminosity_weighted(self) -> unyt.unyt_array: + """ + Kinetic energy fraction of co-rotating star particles, measured for + different luminosity-weighted angular momentum vectors. + + This is computed together with Lstar_luminosity_weighted, kappa_star_luminosity_weighted, + Mcountrot_star_luminosity_weighted and Lcountrot_star_luminosity_weighted + by compute_Lstar_luminosity_weighted_props(). + """ + if self.Nstar == 0: + return None + if not hasattr(self, "internal_kappa_star_luminosity_weighted"): + self.compute_Lstar_luminosity_weighted_props() + + return self.internal_kappa_star_luminosity_weighted + @lazy_property def DtoTstar(self) -> unyt.unyt_quantity: """ @@ -1320,6 +1396,47 @@ def DtoTstar(self) -> unyt.unyt_quantity: self.compute_Lstar_props() return 1.0 - 2.0 * self.internal_Mcountrot_star / self.Mstar + @lazy_property + def DtoTstar_luminosity_weighted_luminosity_ratio(self) -> unyt.unyt_array: + """ + Disk to total luminosity ratio for all provided stellar luminosity bands. + Each band uses the luminosity-weighted angular momentum as defined in that + band. + + This is computed together with Lstar_luminosity_weighted, kappa_star_luminosity_weighted, + Mcountrot_star_luminosity_weighted and Lcountrot_star_luminosity_weighted + by compute_Lstar_luminosity_weighted_props(). + """ + if self.Nstar == 0: + return None + if not hasattr(self, "internal_Lcountrot_star_luminosity_weighted"): + self.compute_Lstar_luminosity_weighted_props() + + return ( + 1.0 + - 2.0 + * self.internal_Lcountrot_star_luminosity_weighted + / self.StellarLuminosity + ) + + @lazy_property + def DtoTstar_luminosity_weighted_mass_ratio(self) -> unyt.unyt_array: + """ + Disk to total mass ratio for all provided stellar luminosity bands. + Each band uses the luminosity-weighted angular momentum as defined in that + band. + + This is computed together with Lstar_luminosity_weighted, kappa_star_luminosity_weighted, + Mcountrot_star_luminosity_weighted and Lcountrot_star_luminosity_weighted + by compute_Lstar_luminosity_weighted_props(). + """ + if self.Nstar == 0: + return None + if not hasattr(self, "internal_Mcountrot_star_luminosity_weighted"): + self.compute_Lstar_luminosity_weighted_props() + + return 1.0 - 2.0 * self.internal_Mcountrot_star_luminosity_weighted / self.Mstar + @lazy_property def veldisp_matrix_star(self) -> unyt.unyt_array: """ @@ -1332,18 +1449,15 @@ def veldisp_matrix_star(self) -> unyt.unyt_array: ) @lazy_property - def Ekin_star(self) -> unyt.unyt_quantity: + def KineticEnergyStars(self) -> unyt.unyt_quantity: """ Kinetic energy of star particles. """ if self.Mstar == 0: return None - # below we need to force conversion to np.float64 before summing - # up particles to avoid overflow - ekin_star = self.mass_star * ((self.vel_star - self.vcom_star) ** 2).sum(axis=1) - ekin_star = unyt.unyt_array( - ekin_star.value, dtype=np.float64, units=ekin_star.units - ) + v_star = self.vel_star - self.vcom[None, :] + v_star += self.pos_star * self.cosmology["H"] + ekin_star = self.mass_star * (v_star**2).sum(axis=1) return 0.5 * ekin_star.sum() @lazy_property @@ -1375,11 +1489,11 @@ def compute_Lbar_props(self): ( self.internal_Lbar, self.internal_kappa_bar, - ) = get_angular_momentum_and_kappa_corot( + ) = get_angular_momentum_and_kappa_corot_mass_weighted( self.mass_baryons, self.pos_baryons, self.vel_baryons, - ref_velocity=self.vcom_bar, + reference_velocity=self.vcom_bar, ) @lazy_property @@ -1556,7 +1670,7 @@ def gas_MgasO(self) -> unyt.unyt_array: ][ :, self.snapshot_datasets.get_column_index( - "ElementMassFractions", "Oxygen" + "PartType0/ElementMassFractions", "Oxygen" ), ] ) @@ -1592,7 +1706,9 @@ def gas_MgasFe(self) -> unyt.unyt_array: self.gas_mask_ap ][ :, - self.snapshot_datasets.get_column_index("ElementMassFractions", "Iron"), + self.snapshot_datasets.get_column_index( + "PartType0/ElementMassFractions", "Iron" + ), ] ) @@ -1700,7 +1816,7 @@ def gas_mass_H(self) -> unyt.unyt_array: self.gas_element_fractions[ :, self.snapshot_datasets.get_column_index( - "ElementMassFractions", "Hydrogen" + "PartType0/ElementMassFractions", "Hydrogen" ), ] * self.mass_gas @@ -1717,7 +1833,7 @@ def gas_mass_He(self) -> unyt.unyt_array: self.gas_element_fractions[ :, self.snapshot_datasets.get_column_index( - "ElementMassFractions", "Helium" + "PartType0/ElementMassFractions", "Helium" ), ] * self.mass_gas @@ -1744,7 +1860,10 @@ def gas_mass_HI(self) -> unyt.unyt_array: return ( self.gas_mass_H * self.gas_species_fractions[ - :, self.snapshot_datasets.get_column_index("SpeciesFractions", "HI") + :, + self.snapshot_datasets.get_column_index( + "PartType0/SpeciesFractions", "HI" + ), ] ) @@ -1758,7 +1877,10 @@ def gas_mass_H2(self) -> unyt.unyt_array: return ( self.gas_mass_H * self.gas_species_fractions[ - :, self.snapshot_datasets.get_column_index("SpeciesFractions", "H2") + :, + self.snapshot_datasets.get_column_index( + "PartType0/SpeciesFractions", "H2" + ), ] * 2.0 ) @@ -1799,6 +1921,28 @@ def AtomicHydrogenMass(self) -> unyt.unyt_quantity: return None return self.gas_mass_HI.sum() + @lazy_property + def DustMass(self) -> unyt.unyt_quantity: + """ + Total dust mass in gas. + """ + if self.Ngas == 0: + return None + return self.mass_dust.sum() + + @lazy_property + def gas_total_dust_mass_fractions(self) -> unyt.unyt_array: + """ + Total dust mass fractions in gas particles. + """ + if self.Ngas == 0: + return None + mass_frac = self.get_dataset("PartType0/TotalDustMassFractions")[ + self.gas_mask_all + ][self.gas_mask_ap] + mass_frac[mass_frac < 10 ** (-10)] = 0 * unyt.dimensionless + return mass_frac + @lazy_property def gas_dust_mass_fractions(self) -> unyt.unyt_array: """ @@ -1806,9 +1950,11 @@ def gas_dust_mass_fractions(self) -> unyt.unyt_array: """ if self.Ngas == 0: return None - return self.get_dataset("PartType0/DustMassFractions")[self.gas_mask_all][ + mass_frac = self.get_dataset("PartType0/DustMassFractions")[self.gas_mask_all][ self.gas_mask_ap ] + mass_frac[mass_frac < 10 ** (-10)] = 0 * unyt.dimensionless + return mass_frac @lazy_property def gas_dust_mass_fractions_graphite_large(self) -> unyt.unyt_array: @@ -1820,7 +1966,7 @@ def gas_dust_mass_fractions_graphite_large(self) -> unyt.unyt_array: return self.gas_dust_mass_fractions[ :, self.snapshot_datasets.get_column_index( - "DustMassFractions", "GraphiteLarge" + "PartType0/DustMassFractions", "GraphiteLarge" ), ] @@ -1835,13 +1981,13 @@ def gas_dust_mass_fractions_silicates_large(self) -> unyt.unyt_array: self.gas_dust_mass_fractions[ :, self.snapshot_datasets.get_column_index( - "DustMassFractions", "MgSilicatesLarge" + "PartType0/DustMassFractions", "MgSilicatesLarge" ), ] + self.gas_dust_mass_fractions[ :, self.snapshot_datasets.get_column_index( - "DustMassFractions", "FeSilicatesLarge" + "PartType0/DustMassFractions", "FeSilicatesLarge" ), ] ) @@ -1856,7 +2002,7 @@ def gas_dust_mass_fractions_graphite_small(self) -> unyt.unyt_array: return self.gas_dust_mass_fractions[ :, self.snapshot_datasets.get_column_index( - "DustMassFractions", "GraphiteSmall" + "PartType0/DustMassFractions", "GraphiteSmall" ), ] @@ -1871,13 +2017,13 @@ def gas_dust_mass_fractions_silicates_small(self) -> unyt.unyt_array: self.gas_dust_mass_fractions[ :, self.snapshot_datasets.get_column_index( - "DustMassFractions", "MgSilicatesSmall" + "PartType0/DustMassFractions", "MgSilicatesSmall" ), ] + self.gas_dust_mass_fractions[ :, self.snapshot_datasets.get_column_index( - "DustMassFractions", "FeSilicatesSmall" + "PartType0/DustMassFractions", "FeSilicatesSmall" ), ] ) @@ -1957,7 +2103,16 @@ def DustGraphiteMassInAtomicGas(self) -> unyt.unyt_quantity: """ if self.Ngas == 0: return None - return (self.gas_graphite_mass_fractions * self.gas_mass_HI).sum() + atomic_gas_mass = ( + self.gas_species_fractions[ + :, + self.snapshot_datasets.get_column_index( + "PartType0/SpeciesFractions", "HI" + ), + ] + * self.mass_gas + ) + return (self.gas_graphite_mass_fractions * atomic_gas_mass).sum() @lazy_property def DustGraphiteMassInMolecularGas(self) -> unyt.unyt_quantity: @@ -1966,7 +2121,17 @@ def DustGraphiteMassInMolecularGas(self) -> unyt.unyt_quantity: """ if self.Ngas == 0: return None - return (self.gas_graphite_mass_fractions * self.gas_mass_H2).sum() + molecular_gas_mass = ( + 2 + * self.gas_species_fractions[ + :, + self.snapshot_datasets.get_column_index( + "PartType0/SpeciesFractions", "H2" + ), + ] + * self.mass_gas + ) + return (self.gas_graphite_mass_fractions * molecular_gas_mass).sum() @lazy_property def DustGraphiteMassInColdDenseGas(self) -> unyt.unyt_quantity: @@ -1996,7 +2161,16 @@ def DustSilicatesMassInAtomicGas(self) -> unyt.unyt_quantity: """ if self.Ngas == 0: return None - return (self.gas_silicates_mass_fractions * self.gas_mass_HI).sum() + atomic_gas_mass = ( + self.gas_species_fractions[ + :, + self.snapshot_datasets.get_column_index( + "PartType0/SpeciesFractions", "HI" + ), + ] + * self.mass_gas + ) + return (self.gas_silicates_mass_fractions * atomic_gas_mass).sum() @lazy_property def DustSilicatesMassInMolecularGas(self) -> unyt.unyt_quantity: @@ -2005,7 +2179,17 @@ def DustSilicatesMassInMolecularGas(self) -> unyt.unyt_quantity: """ if self.Ngas == 0: return None - return (self.gas_silicates_mass_fractions * self.gas_mass_H2).sum() + molecular_gas_mass = ( + 2 + * self.gas_species_fractions[ + :, + self.snapshot_datasets.get_column_index( + "PartType0/SpeciesFractions", "H2" + ), + ] + * self.mass_gas + ) + return (self.gas_silicates_mass_fractions * molecular_gas_mass).sum() @lazy_property def DustSilicatesMassInColdDenseGas(self) -> unyt.unyt_quantity: @@ -2035,7 +2219,17 @@ def DustLargeGrainMassInMolecularGas(self) -> unyt.unyt_quantity: """ if self.Ngas == 0: return None - return (self.gas_large_dust_mass_fractions * self.gas_mass_H2).sum() + molecular_gas_mass = ( + 2 + * self.gas_species_fractions[ + :, + self.snapshot_datasets.get_column_index( + "PartType0/SpeciesFractions", "H2" + ), + ] + * self.mass_gas + ) + return (self.gas_large_dust_mass_fractions * molecular_gas_mass).sum() @lazy_property def DustLargeGrainMassInColdDenseGas(self) -> unyt.unyt_quantity: @@ -2049,6 +2243,17 @@ def DustLargeGrainMassInColdDenseGas(self) -> unyt.unyt_quantity: * self.mass_gas[self.gas_is_cold_dense] ).sum() + @lazy_property + def DustLargeGrainMassSFRWeighted(self) -> unyt.unyt_quantity: + """ + Large dust grain mass, weighted by particle SFR + """ + if (self.Ngas == 0) or np.isclose(np.sum(self.gas_SFR), 0): + return None + return np.sum( + self.gas_large_dust_mass_fractions * self.mass_gas * self.gas_SFR + ) / np.sum(self.gas_SFR) + @lazy_property def DustSmallGrainMass(self) -> unyt.unyt_quantity: """ @@ -2065,7 +2270,17 @@ def DustSmallGrainMassInMolecularGas(self) -> unyt.unyt_quantity: """ if self.Ngas == 0: return None - return (self.gas_small_dust_mass_fractions * self.gas_mass_H2).sum() + molecular_gas_mass = ( + 2 + * self.gas_species_fractions[ + :, + self.snapshot_datasets.get_column_index( + "PartType0/SpeciesFractions", "H2" + ), + ] + * self.mass_gas + ) + return (self.gas_small_dust_mass_fractions * molecular_gas_mass).sum() @lazy_property def DustSmallGrainMassInColdDenseGas(self) -> unyt.unyt_quantity: @@ -2079,6 +2294,17 @@ def DustSmallGrainMassInColdDenseGas(self) -> unyt.unyt_quantity: * self.mass_gas[self.gas_is_cold_dense] ).sum() + @lazy_property + def DustSmallGrainMassSFRWeighted(self) -> unyt.unyt_quantity: + """ + Small dust grain mass, weighted by particle SFR + """ + if (self.Ngas == 0) or np.isclose(np.sum(self.gas_SFR), 0): + return None + return np.sum( + self.gas_small_dust_mass_fractions * self.mass_gas * self.gas_SFR + ) / np.sum(self.gas_SFR) + @lazy_property def GasMassInColdDenseGas(self) -> unyt.unyt_quantity: """ @@ -2112,7 +2338,7 @@ def gas_diffuse_carbon_mass(self) -> unyt.unyt_array: self.gas_diffuse_element_fractions[ :, self.snapshot_datasets.get_column_index( - "ElementMassFractions", "Carbon" + "PartType0/ElementMassFractions", "Carbon" ), ] * self.mass_gas @@ -2129,7 +2355,7 @@ def gas_diffuse_oxygen_mass(self) -> unyt.unyt_array: self.gas_diffuse_element_fractions[ :, self.snapshot_datasets.get_column_index( - "ElementMassFractions", "Oxygen" + "PartType0/ElementMassFractions", "Oxygen" ), ] * self.mass_gas @@ -2146,7 +2372,7 @@ def gas_diffuse_magnesium_mass(self) -> unyt.unyt_array: self.gas_diffuse_element_fractions[ :, self.snapshot_datasets.get_column_index( - "ElementMassFractions", "Magnesium" + "PartType0/ElementMassFractions", "Magnesium" ), ] * self.mass_gas @@ -2163,7 +2389,7 @@ def gas_diffuse_silicon_mass(self) -> unyt.unyt_array: self.gas_diffuse_element_fractions[ :, self.snapshot_datasets.get_column_index( - "ElementMassFractions", "Silicon" + "PartType0/ElementMassFractions", "Silicon" ), ] * self.mass_gas @@ -2179,7 +2405,9 @@ def gas_diffuse_iron_mass(self) -> unyt.unyt_array: return ( self.gas_diffuse_element_fractions[ :, - self.snapshot_datasets.get_column_index("ElementMassFractions", "Iron"), + self.snapshot_datasets.get_column_index( + "PartType0/ElementMassFractions", "Iron" + ), ] * self.mass_gas ) @@ -2238,10 +2466,15 @@ def gas_O_over_H_total(self) -> unyt.unyt_array: return None nH = self.gas_element_fractions[ :, - self.snapshot_datasets.get_column_index("ElementMassFractions", "Hydrogen"), + self.snapshot_datasets.get_column_index( + "PartType0/ElementMassFractions", "Hydrogen" + ), ] nO = self.gas_element_fractions[ - :, self.snapshot_datasets.get_column_index("ElementMassFractions", "Oxygen") + :, + self.snapshot_datasets.get_column_index( + "PartType0/ElementMassFractions", "Oxygen" + ), ] return nO / (16.0 * nH) @@ -2254,10 +2487,15 @@ def gas_N_over_O_total(self) -> unyt.unyt_array: return None nN = self.gas_element_fractions[ :, - self.snapshot_datasets.get_column_index("ElementMassFractions", "Nitrogen"), + self.snapshot_datasets.get_column_index( + "PartType0/ElementMassFractions", "Nitrogen" + ), ] nO = self.gas_element_fractions[ - :, self.snapshot_datasets.get_column_index("ElementMassFractions", "Oxygen") + :, + self.snapshot_datasets.get_column_index( + "PartType0/ElementMassFractions", "Oxygen" + ), ] ratio = np.zeros_like(nN) ratio[nO != 0] = (16.0 * nN[nO != 0]) / (14.0 * nO[nO != 0]) @@ -2271,10 +2509,16 @@ def gas_C_over_O_total(self) -> unyt.unyt_array: if self.Ngas == 0: return None nC = self.gas_element_fractions[ - :, self.snapshot_datasets.get_column_index("ElementMassFractions", "Carbon") + :, + self.snapshot_datasets.get_column_index( + "PartType0/ElementMassFractions", "Carbon" + ), ] nO = self.gas_element_fractions[ - :, self.snapshot_datasets.get_column_index("ElementMassFractions", "Oxygen") + :, + self.snapshot_datasets.get_column_index( + "PartType0/ElementMassFractions", "Oxygen" + ), ] ratio = np.zeros_like(nC) ratio[nO != 0] = (16.0 * nC[nO != 0]) / (12.011 * nO[nO != 0]) @@ -2290,10 +2534,15 @@ def gas_N_over_O_diffuse(self) -> unyt.unyt_array: return None nN = self.gas_diffuse_element_fractions[ :, - self.snapshot_datasets.get_column_index("ElementMassFractions", "Nitrogen"), + self.snapshot_datasets.get_column_index( + "PartType0/ElementMassFractions", "Nitrogen" + ), ] nO = self.gas_diffuse_element_fractions[ - :, self.snapshot_datasets.get_column_index("ElementMassFractions", "Oxygen") + :, + self.snapshot_datasets.get_column_index( + "PartType0/ElementMassFractions", "Oxygen" + ), ] ratio = np.zeros_like(nN) ratio[nO != 0] = (16.0 * nN[nO != 0]) / (14.0 * nO[nO != 0]) @@ -2307,10 +2556,16 @@ def gas_C_over_O_diffuse(self) -> unyt.unyt_array: if self.Ngas == 0: return None nC = self.gas_diffuse_element_fractions[ - :, self.snapshot_datasets.get_column_index("ElementMassFractions", "Carbon") + :, + self.snapshot_datasets.get_column_index( + "PartType0/ElementMassFractions", "Carbon" + ), ] nO = self.gas_diffuse_element_fractions[ - :, self.snapshot_datasets.get_column_index("ElementMassFractions", "Oxygen") + :, + self.snapshot_datasets.get_column_index( + "PartType0/ElementMassFractions", "Oxygen" + ), ] ratio = np.zeros_like(nC) ratio[nO != 0] = (16.0 * nC[nO != 0]) / (12.011 * nO[nO != 0]) @@ -2325,10 +2580,15 @@ def gas_O_over_H_diffuse(self) -> unyt.unyt_array: return None nH = self.gas_diffuse_element_fractions[ :, - self.snapshot_datasets.get_column_index("ElementMassFractions", "Hydrogen"), + self.snapshot_datasets.get_column_index( + "PartType0/ElementMassFractions", "Hydrogen" + ), ] nO = self.gas_diffuse_element_fractions[ - :, self.snapshot_datasets.get_column_index("ElementMassFractions", "Oxygen") + :, + self.snapshot_datasets.get_column_index( + "PartType0/ElementMassFractions", "Oxygen" + ), ] return nO / (16.0 * nH) @@ -2600,10 +2860,13 @@ def LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfGasLowLimit( """ if (self.Ngas == 0) or (self.GasMassInColdDenseGas == 0): return None - return 10 ** (( - self.gas_log10_O_over_H_diffuse_low_limit[self.gas_is_cold_dense] - * self.mass_gas[self.gas_is_cold_dense] - ).sum() / self.GasMassInColdDenseGas) + return 10 ** ( + ( + self.gas_log10_O_over_H_diffuse_low_limit[self.gas_is_cold_dense] + * self.mass_gas[self.gas_is_cold_dense] + ).sum() + / self.GasMassInColdDenseGas + ) @lazy_property def LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfGasHighLimit( @@ -2616,10 +2879,13 @@ def LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfGasHighLimit( """ if (self.Ngas == 0) or (self.GasMassInColdDenseGas == 0): return None - return 10 ** (( - self.gas_log10_O_over_H_diffuse_high_limit[self.gas_is_cold_dense] - * self.mass_gas[self.gas_is_cold_dense] - ).sum() / self.GasMassInColdDenseGas) + return 10 ** ( + ( + self.gas_log10_O_over_H_diffuse_high_limit[self.gas_is_cold_dense] + * self.mass_gas[self.gas_is_cold_dense] + ).sum() + / self.GasMassInColdDenseGas + ) @lazy_property def LogarithmicMassWeightedDiffuseNitrogenOverOxygenOfGasLowLimit( @@ -2632,10 +2898,13 @@ def LogarithmicMassWeightedDiffuseNitrogenOverOxygenOfGasLowLimit( """ if (self.Ngas == 0) or (self.GasMassInColdDenseGas == 0): return None - return 10 ** (( - self.gas_log10_N_over_O_diffuse_low_limit[self.gas_is_cold_dense] - * self.mass_gas[self.gas_is_cold_dense] - ).sum() / self.GasMassInColdDenseGas) + return 10 ** ( + ( + self.gas_log10_N_over_O_diffuse_low_limit[self.gas_is_cold_dense] + * self.mass_gas[self.gas_is_cold_dense] + ).sum() + / self.GasMassInColdDenseGas + ) @lazy_property def LogarithmicMassWeightedDiffuseNitrogenOverOxygenOfGasHighLimit( @@ -2648,10 +2917,13 @@ def LogarithmicMassWeightedDiffuseNitrogenOverOxygenOfGasHighLimit( """ if (self.Ngas == 0) or (self.GasMassInColdDenseGas == 0): return None - return 10 ** (( - self.gas_log10_N_over_O_diffuse_high_limit[self.gas_is_cold_dense] - * self.mass_gas[self.gas_is_cold_dense] - ).sum() / self.GasMassInColdDenseGas) + return 10 ** ( + ( + self.gas_log10_N_over_O_diffuse_high_limit[self.gas_is_cold_dense] + * self.mass_gas[self.gas_is_cold_dense] + ).sum() + / self.GasMassInColdDenseGas + ) @lazy_property def LogarithmicMassWeightedDiffuseCarbonOverOxygenOfGasLowLimit( @@ -2664,10 +2936,13 @@ def LogarithmicMassWeightedDiffuseCarbonOverOxygenOfGasLowLimit( """ if (self.Ngas == 0) or (self.GasMassInColdDenseGas == 0): return None - return 10 ** (( - self.gas_log10_C_over_O_diffuse_low_limit[self.gas_is_cold_dense] - * self.mass_gas[self.gas_is_cold_dense] - ).sum() / self.GasMassInColdDenseGas) + return 10 ** ( + ( + self.gas_log10_C_over_O_diffuse_low_limit[self.gas_is_cold_dense] + * self.mass_gas[self.gas_is_cold_dense] + ).sum() + / self.GasMassInColdDenseGas + ) @lazy_property def LogarithmicMassWeightedDiffuseCarbonOverOxygenOfGasHighLimit( @@ -2680,10 +2955,13 @@ def LogarithmicMassWeightedDiffuseCarbonOverOxygenOfGasHighLimit( """ if (self.Ngas == 0) or (self.GasMassInColdDenseGas == 0): return None - return 10 ** (( - self.gas_log10_C_over_O_diffuse_high_limit[self.gas_is_cold_dense] - * self.mass_gas[self.gas_is_cold_dense] - ).sum() / self.GasMassInColdDenseGas) + return 10 ** ( + ( + self.gas_log10_C_over_O_diffuse_high_limit[self.gas_is_cold_dense] + * self.mass_gas[self.gas_is_cold_dense] + ).sum() + / self.GasMassInColdDenseGas + ) @lazy_property def LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfAtomicGasLowLimit( @@ -2696,10 +2974,13 @@ def LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfAtomicGasLowLimit( """ if (self.Ngas == 0) or (self.AtomicHydrogenMass == 0): return None - return 10 ** (( - self.gas_log10_O_over_H_diffuse_low_limit[self.gas_is_cold_dense] - * self.gas_mass_HI[self.gas_is_cold_dense] - ).sum() / self.AtomicHydrogenMass) + return 10 ** ( + ( + self.gas_log10_O_over_H_diffuse_low_limit[self.gas_is_cold_dense] + * self.gas_mass_HI[self.gas_is_cold_dense] + ).sum() + / self.AtomicHydrogenMass + ) @lazy_property def LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfAtomicGasHighLimit( @@ -2712,10 +2993,13 @@ def LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfAtomicGasHighLimit( """ if (self.Ngas == 0) or (self.AtomicHydrogenMass == 0): return None - return 10 ** (( - self.gas_log10_O_over_H_diffuse_high_limit[self.gas_is_cold_dense] - * self.gas_mass_HI[self.gas_is_cold_dense] - ).sum() / self.AtomicHydrogenMass) + return 10 ** ( + ( + self.gas_log10_O_over_H_diffuse_high_limit[self.gas_is_cold_dense] + * self.gas_mass_HI[self.gas_is_cold_dense] + ).sum() + / self.AtomicHydrogenMass + ) @lazy_property def LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfMolecularGasLowLimit( @@ -2728,10 +3012,13 @@ def LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfMolecularGasLowLimit( """ if (self.Ngas == 0) or (self.MolecularHydrogenMass == 0): return None - return 10 ** (( - self.gas_log10_O_over_H_diffuse_low_limit[self.gas_is_cold_dense] - * self.gas_mass_H2[self.gas_is_cold_dense] - ).sum() / self.MolecularHydrogenMass) + return 10 ** ( + ( + self.gas_log10_O_over_H_diffuse_low_limit[self.gas_is_cold_dense] + * self.gas_mass_H2[self.gas_is_cold_dense] + ).sum() + / self.MolecularHydrogenMass + ) @lazy_property def LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfMolecularGasHighLimit( @@ -2744,10 +3031,13 @@ def LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfMolecularGasHighLimit( """ if (self.Ngas == 0) or (self.MolecularHydrogenMass == 0): return None - return 10 ** (( - self.gas_log10_O_over_H_diffuse_high_limit[self.gas_is_cold_dense] - * self.gas_mass_H2[self.gas_is_cold_dense] - ).sum() / self.MolecularHydrogenMass) + return 10 ** ( + ( + self.gas_log10_O_over_H_diffuse_high_limit[self.gas_is_cold_dense] + * self.gas_mass_H2[self.gas_is_cold_dense] + ).sum() + / self.MolecularHydrogenMass + ) @lazy_property def star_Fe_over_H(self) -> unyt.unyt_array: @@ -2758,10 +3048,15 @@ def star_Fe_over_H(self) -> unyt.unyt_array: return None nH = self.star_element_fractions[ :, - self.snapshot_datasets.get_column_index("ElementMassFractions", "Hydrogen"), + self.snapshot_datasets.get_column_index( + "PartType4/ElementMassFractions", "Hydrogen" + ), ] nFe = self.star_element_fractions[ - :, self.snapshot_datasets.get_column_index("ElementMassFractions", "Iron") + :, + self.snapshot_datasets.get_column_index( + "PartType4/ElementMassFractions", "Iron" + ), ] return nFe / (55.845 * nH) @@ -2775,7 +3070,9 @@ def star_Fe_from_SNIa_over_H(self) -> unyt.unyt_array: return None nH = self.star_element_fractions[ :, - self.snapshot_datasets.get_column_index("ElementMassFractions", "Hydrogen"), + self.snapshot_datasets.get_column_index( + "PartType4/ElementMassFractions", "Hydrogen" + ), ] nFe = self.get_dataset("PartType4/IronMassFractionsFromSNIa")[ self.star_mask_all @@ -2850,7 +3147,9 @@ def LogarithmicMassWeightedIronOverHydrogenOfStarsLowLimit( """ if self.Nstar == 0: return None - return 10 ** ((self.star_log10_Fe_over_H_low_limit * self.mass_star).sum() / self.Mstar) + return 10 ** ( + (self.star_log10_Fe_over_H_low_limit * self.mass_star).sum() / self.Mstar + ) @lazy_property def LogarithmicMassWeightedIronOverHydrogenOfStarsHighLimit( @@ -2862,7 +3161,9 @@ def LogarithmicMassWeightedIronOverHydrogenOfStarsHighLimit( """ if self.Nstar == 0: return None - return 10 ** ((self.star_log10_Fe_over_H_high_limit * self.mass_star).sum() / self.Mstar) + return 10 ** ( + (self.star_log10_Fe_over_H_high_limit * self.mass_star).sum() / self.Mstar + ) @lazy_property def LogarithmicMassWeightedIronFromSNIaOverHydrogenOfStarsLowLimit( @@ -2875,10 +3176,15 @@ def LogarithmicMassWeightedIronFromSNIaOverHydrogenOfStarsLowLimit( """ if self.Nstar == 0: return None - return 10 ** ((self.star_log10_Fe_from_SNIa_over_H_low_limit * self.mass_star).sum() / self.Mstar) + return 10 ** ( + (self.star_log10_Fe_from_SNIa_over_H_low_limit * self.mass_star).sum() + / self.Mstar + ) @lazy_property - def LinearMassWeightedIronFromSNIaOverHydrogenOfStars(self,) -> unyt.unyt_quantity: + def LinearMassWeightedIronFromSNIaOverHydrogenOfStars( + self, + ) -> unyt.unyt_quantity: """ Mass-weighted sum of the iron over hydrogen ratio for star particles, times the solar ratio, set in the parameter file, and @@ -2897,12 +3203,14 @@ def star_Mg_over_H(self) -> unyt.unyt_array: return None nH = self.star_element_fractions[ :, - self.snapshot_datasets.get_column_index("ElementMassFractions", "Hydrogen"), + self.snapshot_datasets.get_column_index( + "PartType4/ElementMassFractions", "Hydrogen" + ), ] nMg = self.star_element_fractions[ :, self.snapshot_datasets.get_column_index( - "ElementMassFractions", "Magnesium" + "PartType4/ElementMassFractions", "Magnesium" ), ] return nMg / (24.305 * nH) @@ -2958,7 +3266,9 @@ def LogarithmicMassWeightedMagnesiumOverHydrogenOfStarsLowLimit( """ if self.Nstar == 0: return None - return 10 ** ((self.star_log10_Mg_over_H_low_limit * self.mass_star).sum() / self.Mstar) + return 10 ** ( + (self.star_log10_Mg_over_H_low_limit * self.mass_star).sum() / self.Mstar + ) @lazy_property def LogarithmicMassWeightedMagnesiumOverHydrogenOfStarsHighLimit( @@ -2970,7 +3280,9 @@ def LogarithmicMassWeightedMagnesiumOverHydrogenOfStarsHighLimit( """ if self.Nstar == 0: return None - return 10 ** ((self.star_log10_Mg_over_H_high_limit * self.mass_star).sum() / self.Mstar) + return 10 ** ( + (self.star_log10_Mg_over_H_high_limit * self.mass_star).sum() / self.Mstar + ) @lazy_property def HalfMassRadiusGas(self) -> unyt.unyt_quantity: @@ -2981,6 +3293,39 @@ def HalfMassRadiusGas(self) -> unyt.unyt_quantity: self.radius[self.type == 0], self.mass_gas, self.Mgas ) + @lazy_property + def HalfMassRadiusDust(self) -> unyt.unyt_quantity: + """ + Half mass radius of dust. + """ + if self.Ngas == 0: + return None + return get_half_mass_radius( + self.radius[self.type == 0], self.mass_dust, self.DustMass + ) + + @lazy_property + def HalfMassRadiusAtomicHydrogen(self) -> unyt.unyt_quantity: + """ + Half mass radius of HI. + """ + if self.Ngas == 0: + return None + return get_half_mass_radius( + self.radius[self.type == 0], self.gas_mass_HI, self.AtomicHydrogenMass + ) + + @lazy_property + def HalfMassRadiusMolecularHydrogen(self) -> unyt.unyt_quantity: + """ + Half mass radius of H2. + """ + if self.Ngas == 0: + return None + return get_half_mass_radius( + self.radius[self.type == 0], self.gas_mass_H2, self.MolecularHydrogenMass + ) + @lazy_property def HalfMassRadiusDM(self) -> unyt.unyt_quantity: """ @@ -2997,6 +3342,19 @@ def HalfMassRadiusStar(self) -> unyt.unyt_quantity: self.radius[self.type == 4], self.mass_star, self.Mstar ) + @lazy_property + def HalfLightRadiusStar(self) -> unyt.unyt_array: + """ + Half light radius of stars for the 9 GAMA bands. + """ + if self.Nstar == 0: + return None + return get_half_light_radius( + self.radius[self.type == 4], + self.stellar_luminosities, + self.StellarLuminosity, + ) + @lazy_property def HalfMassRadiusBaryon(self) -> unyt.unyt_quantity: """ @@ -3008,6 +3366,157 @@ def HalfMassRadiusBaryon(self) -> unyt.unyt_quantity: self.Mbaryons, ) + @lazy_property + def R_vmax_soft(self) -> unyt.unyt_quantity: + """ + Radius at which the maximum circular velocity of the halo is reached. + Particles are set to have minimum radius equal to their softening length. + + This includes contributions from all particle types. + """ + if self.Mtot == 0: + return None + if not hasattr(self, "vmax_soft"): + soft_r = np.maximum(self.softening, self.radius) + self.r_vmax_soft, self.vmax_soft = get_vmax(self.mass, soft_r) + return self.r_vmax_soft + + @lazy_property + def Vmax_soft(self): + """ + Maximum circular velocity of the halo. + Particles are set to have minimum radius equal to their softening length. + This includes contributions from all particle types. + """ + if self.Mtot == 0: + return None + if not hasattr(self, "vmax_soft"): + soft_r = np.maximum(self.softening, self.radius) + self.r_vmax_soft, self.vmax_soft = get_vmax(self.mass, soft_r) + return self.vmax_soft + + def stellar_inertia_tensor(self, **kwargs) -> unyt.unyt_array: + """ + Helper function for calculating stellar inertia tensors. + """ + + # We mask because we want all bound stellar particles, regardless of the initial aperture. By + # iterating, we will deform the ellipsoid and we might need particles beyond the initial spherical + # aperture + mass = self.get_dataset("PartType4/Masses")[self.star_mask_all] + position = ( + self.get_dataset("PartType4/Coordinates")[self.star_mask_all] - self.centre + ) + + return get_inertia_tensor_mass_weighted( + mass, position, self.aperture_radius, **kwargs + ) + + def stellar_inertia_tensor_luminosity_weighted(self, **kwargs) -> unyt.unyt_array: + """ + Helper function for calculating luminosity-weighted stellar inertia tensors. + """ + + # We mask because we want all bound stellar particles, regardless of the initial aperture. By + # iterating, we will deform the ellipsoid and we might need particles beyond the initial spherical + # aperture + position = ( + self.get_dataset("PartType4/Coordinates")[self.star_mask_all] - self.centre + ) + luminosity = self.get_dataset("PartType4/Luminosities")[self.star_mask_all] + + return get_inertia_tensor_luminosity_weighted( + luminosity, position, self.aperture_radius, **kwargs + ) + + @lazy_property + def StellarInertiaTensor(self) -> unyt.unyt_array: + """ + Inertia tensor of the stellar mass distribution. + Computed iteratively using an ellipsoid with volume equal to that of + a sphere with radius 10 x HalfMassRadiusStar. Only considers bound particles. + """ + if self.Mstar == 0: + return None + return self.stellar_inertia_tensor() + + @lazy_property + def StellarInertiaTensorReduced(self) -> unyt.unyt_array: + """ + Reduced inertia tensor of the stellar mass distribution. + Computed iteratively using an ellipsoid with volume equal to that of + a sphere with radius 10 x HalfMassRadiusStar. Only considers bound particles. + """ + if self.Mstar == 0: + return None + return self.stellar_inertia_tensor(reduced=True) + + @lazy_property + def StellarInertiaTensorNoniterative(self) -> unyt.unyt_array: + """ + Inertia tensor of the stellar mass distribution. + Computed using all bound star particles within 10 x HalfMassRadiusStar. + """ + if self.Mstar == 0: + return None + return self.stellar_inertia_tensor(max_iterations=1) + + @lazy_property + def StellarInertiaTensorReducedNoniterative(self) -> unyt.unyt_array: + """ + Reduced inertia tensor of the stellar mass distribution. + Computed using all bound star particles within 10 x HalfMassRadiusStar. + """ + if self.Mstar == 0: + return None + return self.stellar_inertia_tensor(reduced=True, max_iterations=1) + + @lazy_property + def StellarInertiaTensorLuminosityWeighted(self) -> unyt.unyt_array: + """ + Inertia tensor of the stellar luminosity distribution for each GAMA band. + Computed iteratively using an ellipsoid with volume equal to that of + a sphere with radius 10 x HalfMassRadiusStar. Only considers bound particles. + """ + if self.Mstar == 0: + return None + return self.stellar_inertia_tensor_luminosity_weighted() + + @lazy_property + def StellarInertiaTensorReducedLuminosityWeighted(self) -> unyt.unyt_array: + """ + Reduced inertia tensor of the stellar luminosity distribution for each GAMA band. + Computed iteratively using an ellipsoid with volume equal to that of + a sphere with radius 10 x HalfMassRadiusStar. Only considers bound particles. + """ + if self.Mstar == 0: + return None + return self.stellar_inertia_tensor_luminosity_weighted(reduced=True) + + @lazy_property + def StellarInertiaTensorNoniterativeLuminosityWeighted(self) -> unyt.unyt_array: + """ + Inertia tensor of the stellar luminosity distribution for each GAMA band. + Computed using all bound star particles within 10 x HalfMassRadiusStar. + """ + if self.Mstar == 0: + return None + return self.stellar_inertia_tensor_luminosity_weighted(max_iterations=1) + + @lazy_property + def StellarInertiaTensorReducedNoniterativeLuminosityWeighted( + self, + ) -> unyt.unyt_array: + """ + Reduced inertia tensor of the stellar luminosity distribution for each GAMA band. + Computed using all bound star particles within 10 x HalfMassRadiusStar. + """ + if self.Mstar == 0: + return None + return self.stellar_inertia_tensor_luminosity_weighted( + reduced=True, max_iterations=1 + ) + class ApertureProperties(HaloProperty): """ @@ -3017,154 +3526,179 @@ class ApertureProperties(HaloProperty): are bound to the halo. """ - """ - List of properties from the table that we want to compute. - Each property should have a corresponding method/property/lazy_property in - the ApertureParticleData class above. - """ - property_list: List[Tuple] = [ - (prop, *PropertyTable.full_property_list[prop]) - for prop in [ - "Mtot", - "Mgas", - "Mdm", - "Mstar", - "Mstar_init", - "Mbh_dynamical", - "Mbh_subgrid", - "Ngas", - "Ndm", - "Nstar", - "Nbh", - "BHlasteventa", - "BHmaxM", - "BHmaxID", - "BHmaxpos", - "BHmaxvel", - "BHmaxAR", - "BHmaxlasteventa", - "BlackHolesTotalInjectedThermalEnergy", - "BlackHolesTotalInjectedJetEnergy", - "MostMassiveBlackHoleAveragedAccretionRate", - "MostMassiveBlackHoleInjectedThermalEnergy", - "MostMassiveBlackHoleNumberOfAGNEvents", - "MostMassiveBlackHoleAccretionMode", - "MostMassiveBlackHoleGWMassLoss", - "MostMassiveBlackHoleInjectedJetEnergyByMode", - "MostMassiveBlackHoleLastJetEventScalefactor", - "MostMassiveBlackHoleNumberOfAGNJetEvents", - "MostMassiveBlackHoleNumberOfMergers", - "MostMassiveBlackHoleRadiatedEnergyByMode", - "MostMassiveBlackHoleTotalAccretedMassesByMode", - "MostMassiveBlackHoleWindEnergyByMode", - "MostMassiveBlackHoleSpin", - "MostMassiveBlackHoleTotalAccretedMass", - "MostMassiveBlackHoleFormationScalefactor", - "com", - "com_star", - "vcom", - "vcom_star", - "Lgas", - "Ldm", - "Lstar", - "kappa_corot_gas", - "kappa_corot_star", - "Lbaryons", - "kappa_corot_baryons", - "veldisp_matrix_gas", - "veldisp_matrix_dm", - "veldisp_matrix_star", - "Ekin_gas", - "Ekin_star", - "Mgas_SF", - "gasmetalfrac", - "gasmetalfrac_SF", - "gasOfrac", - "gasOfrac_SF", - "gasFefrac", - "gasFefrac_SF", - "Tgas", - "Tgas_no_agn", - "SFR", - "AveragedStarFormationRate", - "StellarLuminosity", - "starmetalfrac", - "HalfMassRadiusGas", - "HalfMassRadiusDM", - "HalfMassRadiusStar", - "HalfMassRadiusBaryon", - "spin_parameter", - "DtoTgas", - "DtoTstar", - "starOfrac", - "starFefrac", - "stellar_age_mw", - "stellar_age_lw", - "TotalSNIaRate", - "HydrogenMass", - "HeliumMass", - "MolecularHydrogenMass", - "AtomicHydrogenMass", - "starMgfrac", - "DustGraphiteMass", - "DustGraphiteMassInAtomicGas", - "DustGraphiteMassInMolecularGas", - "DustGraphiteMassInColdDenseGas", - "DustLargeGrainMass", - "DustLargeGrainMassInMolecularGas", - "DustLargeGrainMassInColdDenseGas", - "DustSilicatesMass", - "DustSilicatesMassInAtomicGas", - "DustSilicatesMassInMolecularGas", - "DustSilicatesMassInColdDenseGas", - "DustSmallGrainMass", - "DustSmallGrainMassInMolecularGas", - "DustSmallGrainMassInColdDenseGas", - "GasMassInColdDenseGas", - "DiffuseCarbonMass", - "DiffuseOxygenMass", - "DiffuseMagnesiumMass", - "DiffuseSiliconMass", - "DiffuseIronMass", - "LinearMassWeightedOxygenOverHydrogenOfGas", - "LinearMassWeightedNitrogenOverOxygenOfGas", - "LinearMassWeightedCarbonOverOxygenOfGas", - "LinearMassWeightedDiffuseOxygenOverHydrogenOfGas", - "LinearMassWeightedDiffuseNitrogenOverOxygenOfGas", - "LinearMassWeightedDiffuseCarbonOverOxygenOfGas", - "LogarithmicMassWeightedDiffuseNitrogenOverOxygenOfGasLowLimit", - "LogarithmicMassWeightedDiffuseNitrogenOverOxygenOfGasHighLimit", - "LogarithmicMassWeightedDiffuseCarbonOverOxygenOfGasLowLimit", - "LogarithmicMassWeightedDiffuseCarbonOverOxygenOfGasHighLimit", - "LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfGasLowLimit", - "LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfGasHighLimit", - "LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfAtomicGasLowLimit", - "LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfAtomicGasHighLimit", - "LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfMolecularGasLowLimit", - "LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfMolecularGasHighLimit", - "LinearMassWeightedMagnesiumOverHydrogenOfStars", - "LogarithmicMassWeightedMagnesiumOverHydrogenOfStarsLowLimit", - "LogarithmicMassWeightedMagnesiumOverHydrogenOfStarsHighLimit", - "LinearMassWeightedIronOverHydrogenOfStars", - "LogarithmicMassWeightedIronOverHydrogenOfStarsLowLimit", - "LogarithmicMassWeightedIronOverHydrogenOfStarsHighLimit", - "GasMassInColdDenseDiffuseMetals", - "LogarithmicMassWeightedIronFromSNIaOverHydrogenOfStarsLowLimit", - "LinearMassWeightedIronFromSNIaOverHydrogenOfStars", - ] - ] + base_halo_type = "ApertureProperties" + # Properties to calculate for ApertureProperties. Key is the name of the property. + # The value indicates the property has a direct dependence on aperture size. + # This is needed since for larger apertures we sometimes copy across the + # values computed by the previous aperture (if the number of particles was + # the same for both apertures), but we can't do this for all properties + property_names = { + "Mtot": False, + "Mgas": False, + "Mdm": False, + "Mstar": False, + "Mstar_init": False, + "Mbh_dynamical": False, + "Mbh_subgrid": False, + "Ngas": False, + "Ndm": False, + "Nstar": False, + "Nbh": False, + "BHlasteventa": False, + "BHmaxM": False, + "BHmaxID": False, + "BHmaxpos": False, + "BHmaxvel": False, + "BHmaxAR": False, + "BHmaxlasteventa": False, + "BlackHolesTotalInjectedThermalEnergy": False, + "BlackHolesTotalInjectedJetEnergy": False, + "MostMassiveBlackHoleAveragedAccretionRate": False, + "MostMassiveBlackHoleInjectedThermalEnergy": False, + "MostMassiveBlackHoleNumberOfAGNEvents": False, + "MostMassiveBlackHoleAccretionMode": False, + "MostMassiveBlackHoleGWMassLoss": False, + "MostMassiveBlackHoleInjectedJetEnergyByMode": False, + "MostMassiveBlackHoleLastJetEventScalefactor": False, + "MostMassiveBlackHoleNumberOfAGNJetEvents": False, + "MostMassiveBlackHoleNumberOfMergers": False, + "MostMassiveBlackHoleRadiatedEnergyByMode": False, + "MostMassiveBlackHoleTotalAccretedMassesByMode": False, + "MostMassiveBlackHoleWindEnergyByMode": False, + "MostMassiveBlackHoleSpin": False, + "MostMassiveBlackHoleTotalAccretedMass": False, + "MostMassiveBlackHoleFormationScalefactor": False, + "com": False, + "com_dm": False, + "com_star": False, + "vcom": False, + "vcom_star": False, + "Lgas": False, + "Ldm": False, + "Lstar": False, + "kappa_corot_gas": False, + "kappa_corot_star": False, + "Lbaryons": False, + "kappa_corot_baryons": False, + "veldisp_matrix_gas": False, + "veldisp_matrix_dm": False, + "veldisp_matrix_star": False, + "KineticEnergyGas": False, + "KineticEnergyStars": False, + "Mgas_SF": False, + "gasmetalfrac": False, + "gasmetalfrac_SF": False, + "gasOfrac": False, + "gasOfrac_SF": False, + "gasFefrac": False, + "gasFefrac_SF": False, + "Tgas": False, + "Tgas_no_agn": False, + "SFR": False, + "AveragedStarFormationRate": False, + "StellarLuminosity": False, + "starmetalfrac": False, + "HalfMassRadiusGas": False, + "HalfMassRadiusDust": False, + "HalfMassRadiusAtomicHydrogen": False, + "HalfMassRadiusMolecularHydrogen": False, + "HalfMassRadiusDM": False, + "HalfMassRadiusStar": False, + "HalfLightRadiusStar": False, + "HalfMassRadiusBaryon": False, + "DtoTgas": False, + "DtoTstar": False, + "DtoTstar_luminosity_weighted_luminosity_ratio": False, + "DtoTstar_luminosity_weighted_mass_ratio": False, + "kappa_corot_star_luminosity_weighted": False, + "Lstar_luminosity_weighted": False, + "starOfrac": False, + "starFefrac": False, + "stellar_age_mw": False, + "stellar_age_lw": False, + "TotalSNIaRate": False, + "HydrogenMass": False, + "HeliumMass": False, + "MolecularHydrogenMass": False, + "AtomicHydrogenMass": False, + "starMgfrac": False, + "DustMass": False, + "DustGraphiteMass": False, + "DustGraphiteMassInAtomicGas": False, + "DustGraphiteMassInMolecularGas": False, + "DustGraphiteMassInColdDenseGas": False, + "DustLargeGrainMass": False, + "DustLargeGrainMassInMolecularGas": False, + "DustLargeGrainMassInColdDenseGas": False, + "DustLargeGrainMassSFRWeighted": False, + "DustSilicatesMass": False, + "DustSilicatesMassInAtomicGas": False, + "DustSilicatesMassInMolecularGas": False, + "DustSilicatesMassInColdDenseGas": False, + "DustSmallGrainMass": False, + "DustSmallGrainMassInMolecularGas": False, + "DustSmallGrainMassInColdDenseGas": False, + "DustSmallGrainMassSFRWeighted": False, + "GasMassInColdDenseGas": False, + "DiffuseCarbonMass": False, + "DiffuseOxygenMass": False, + "DiffuseMagnesiumMass": False, + "DiffuseSiliconMass": False, + "DiffuseIronMass": False, + "LinearMassWeightedOxygenOverHydrogenOfGas": False, + "LinearMassWeightedNitrogenOverOxygenOfGas": False, + "LinearMassWeightedCarbonOverOxygenOfGas": False, + "LinearMassWeightedDiffuseOxygenOverHydrogenOfGas": False, + "LinearMassWeightedDiffuseNitrogenOverOxygenOfGas": False, + "LinearMassWeightedDiffuseCarbonOverOxygenOfGas": False, + "LogarithmicMassWeightedDiffuseNitrogenOverOxygenOfGasLowLimit": False, + "LogarithmicMassWeightedDiffuseNitrogenOverOxygenOfGasHighLimit": False, + "LogarithmicMassWeightedDiffuseCarbonOverOxygenOfGasLowLimit": False, + "LogarithmicMassWeightedDiffuseCarbonOverOxygenOfGasHighLimit": False, + "LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfGasLowLimit": False, + "LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfGasHighLimit": False, + "LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfAtomicGasLowLimit": False, + "LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfAtomicGasHighLimit": False, + "LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfMolecularGasLowLimit": False, + "LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfMolecularGasHighLimit": False, + "LinearMassWeightedMagnesiumOverHydrogenOfStars": False, + "LogarithmicMassWeightedMagnesiumOverHydrogenOfStarsLowLimit": False, + "LogarithmicMassWeightedMagnesiumOverHydrogenOfStarsHighLimit": False, + "LinearMassWeightedIronOverHydrogenOfStars": False, + "LogarithmicMassWeightedIronOverHydrogenOfStarsLowLimit": False, + "LogarithmicMassWeightedIronOverHydrogenOfStarsHighLimit": False, + "GasMassInColdDenseDiffuseMetals": False, + "LogarithmicMassWeightedIronFromSNIaOverHydrogenOfStarsLowLimit": False, + "LinearMassWeightedIronFromSNIaOverHydrogenOfStars": False, + "Vmax_soft": False, + "R_vmax_soft": False, + "StellarInertiaTensor": True, + "StellarInertiaTensorReduced": True, + "StellarInertiaTensorNoniterative": False, + "StellarInertiaTensorReducedNoniterative": False, + "StellarInertiaTensorLuminosityWeighted": True, + "StellarInertiaTensorReducedLuminosityWeighted": True, + "StellarInertiaTensorNoniterativeLuminosityWeighted": False, + "StellarInertiaTensorReducedNoniterativeLuminosityWeighted": False, + } + + property_list = { + name: PropertyTable.full_property_list[name] for name in property_names + } def __init__( self, cellgrid: SWIFTCellGrid, parameters: ParameterFile, - physical_radius_kpc: float, + aperture_physical_radius_kpc: float | None, + aperture_property: tuple[str, float] | None, recently_heated_gas_filter: RecentlyHeatedGasFilter, stellar_age_calculator: StellarAgeCalculator, cold_dense_gas_filter: ColdDenseGasFilter, category_filter: CategoryFilter, halo_filter: str, - inclusive: bool = False, + inclusive: bool, + all_radii_kpc: list, ): """ Construct an ApertureProperties object with the given physical @@ -3178,9 +3712,17 @@ def __init__( - parameters: ParameterFile Parameter file object containing the parameters from the parameter file. - - physical_radius_kpc: float + - aperture_physical_radius_kpc: float | None Physical radius of the aperture. Unitless and assumed to be expressed - in units of kpc. + in units of kpc. If not None then aperture_property must be None. + If None then aperture_property must be passed. + - aperture_property: tuple[str, float] | None, + Tuple to indicate the radius of this aperture based on a previous property + calculated by SOAP. The first element should be the full name of the + property to use (e.g. BoundSubhalo/HalfMassRadius). The second element + is a multipler (e.g. if you want the aperture radius to be twice the + value of the property, then pass 2). If not None then aperture_physical_radius_kpc + must be None. If None then aperture_physical_radius_kpc must be passed. - recently_heated_gas_filter: RecentlyHeatedGasFilter Filter used to mask out gas particles that were recently heated by AGN feedback. @@ -3197,14 +3739,17 @@ def __init__( The filter to apply to this halo type. Halos which do not fulfil the filter requirements will be skipped. - inclusive: bool - Should properties include particles that are not gravitationally bound to the - subhalo? + Should properties include particles that are not gravitationally bound + to the subhalo? + - all_radii_kpc: list + A list of all the radii for which we are computing an ApertureProperties. + This can allow us to skip property calculation for larger apertures. """ super().__init__(cellgrid) - self.property_mask = parameters.get_property_mask( - "ApertureProperties", [prop[1] for prop in self.property_list] + self.property_filters = parameters.get_property_filters( + "ApertureProperties", [prop.name for prop in self.property_list.values()] ) self.recently_heated_gas_filter = recently_heated_gas_filter @@ -3213,22 +3758,45 @@ def __init__( self.category_filter = category_filter self.snapshot_datasets = cellgrid.snapshot_datasets self.halo_filter = halo_filter - - # no density criterion for these properties - self.mean_density_multiple = None - self.critical_density_multiple = None - - # Minimum physical radius to read in (pMpc) - self.physical_radius_mpc = 0.001 * physical_radius_kpc - + self.record_timings = parameters.record_property_timings + self.all_radii_kpc = all_radii_kpc + self.strict_halo_copy = parameters.strict_halo_copy() + self.boxsize = cellgrid.boxsize + + self.cosmology = {} + self.cosmology["H"] = cellgrid.cosmology[ + "H [internal units]" + ] / cellgrid.get_unit("code_time") + + self.aperture_physical_radius_kpc = aperture_physical_radius_kpc + self.aperture_property = aperture_property self.inclusive = inclusive + if self.aperture_physical_radius_kpc is not None: + self.physical_radius_mpc = 0.001 * self.aperture_physical_radius_kpc + assert self.aperture_physical_radius_kpc >= 0.001 + if self.aperture_physical_radius_kpc < 1: + aperture_name = f"{1000*self.aperture_physical_radius_kpc:.0f}pc" + else: + aperture_name = f"{self.aperture_physical_radius_kpc:.0f}kpc" + else: + prop = self.aperture_property[0].split("/")[-1] + assert self.aperture_property[0].split("/")[0] == "BoundSubhalo" + multiplier = self.aperture_property[1] + if multiplier == 1: + aperture_name = prop + else: + aperture_name = f"{int(multiplier)}x{prop}" + # This value needs to be set since it's used to guess the initial + # load region for each particle + self.physical_radius_mpc = 0 + if self.inclusive: - self.name = f"inclusive_sphere_{physical_radius_kpc:.0f}kpc" - self.group_name = f"InclusiveSphere/{self.physical_radius_mpc*1000.:.0f}kpc" + self.name = f"inclusive_sphere_{aperture_name}" + self.group_name = f"InclusiveSphere/{aperture_name}" else: - self.name = f"exclusive_sphere_{physical_radius_kpc:.0f}kpc" - self.group_name = f"ExclusiveSphere/{self.physical_radius_mpc*1000.:.0f}kpc" + self.name = f"exclusive_sphere_{aperture_name}" + self.group_name = f"ExclusiveSphere/{aperture_name}" self.mask_metadata = self.category_filter.get_filter_metadata(halo_filter) # List of particle properties we need to read in @@ -3247,14 +3815,15 @@ def __init__( } # add additional particle properties based on the selected halo # properties in the parameter file - for prop in self.property_list: - outputname = prop[1] - if not self.property_mask[outputname]: + for prop in self.property_list.values(): + outputname = prop.name + # Skip if this property is disabled in the parameter file + if not self.property_filters[outputname]: continue - is_dmo = prop[8] - if self.category_filter.dmo and not is_dmo: + # Skip non-DMO properties for DMO runs + if self.category_filter.dmo and not prop.dmo_property: continue - partprops = prop[9] + partprops = prop.particle_properties for partprop in partprops: pgroup, dset = parameters.get_particle_property(partprop) if not pgroup in self.particle_properties: @@ -3286,27 +3855,28 @@ def calculate( """ aperture_sphere = {} + timings = {} + # declare all the variables we will compute # we set them to 0 in case a particular variable cannot be computed # all variables are defined with physical units and an appropriate dtype # we need to use the custom unit registry so that everything can be converted # back to snapshot units in the end registry = input_halo["cofp"].units.registry - for prop in self.property_list: - outputname = prop[1] - # skip properties that are masked - if not self.property_mask[outputname]: + for name, prop in self.property_list.items(): + outputname = prop.name + # Skip if this property is disabled in the parameter file + filter_name = self.property_filters[outputname] + if not filter_name: continue - # skip non-DMO properties in DMO run mode - is_dmo = prop[8] - if self.category_filter.dmo and not is_dmo: + # Skip non-DMO properties when in DMO run mode + if self.category_filter.dmo and not prop.dmo_property: continue - name = prop[0] - shape = prop[2] - dtype = prop[3] - unit = unyt.Unit(prop[4], registry=registry) - physical = prop[10] - a_exponent = prop[11] + shape = prop.shape + dtype = prop.dtype + unit = unyt.Unit(prop.unit, registry=registry) + physical = prop.output_physical + a_exponent = prop.a_scale_exponent if shape > 1: val = [0] * shape else: @@ -3319,9 +3889,65 @@ def calculate( do_calculation = self.category_filter.get_do_calculation(halo_result) - # Determine whether to skip this halo because of filter - if do_calculation[self.halo_filter]: - if search_radius < self.physical_radius_mpc * unyt.Mpc: + skip_gt_enclose_radius = False + # Determine if the previous aperture already enclosed all + # the bound particles of the subhalo + # We don't do this if we are passed a property + if self.aperture_physical_radius_kpc is not None: + i_radius = self.all_radii_kpc.index(self.aperture_physical_radius_kpc) + else: + i_radius = 0 + if i_radius != 0 and ("BoundSubhalo/EncloseRadius" in halo_result): + r_enclose = halo_result["BoundSubhalo/EncloseRadius"][0] + r_previous_kpc = self.all_radii_kpc[i_radius - 1] + if r_previous_kpc * unyt.kpc > r_enclose: + # Skip if inclusive, don't copy over any values. Note this is + # never hit if skip_gt_enclose_radius=False in the parameter + # file, since in that case all_radii_kpc is not passed + if self.inclusive: + skip_gt_enclose_radius = True + else: + skip_gt_enclose_radius = True + # Skip if this halo has a filter + if do_calculation[self.halo_filter]: + if r_previous_kpc < 1: + prev_group_name = ( + f"ExclusiveSphere/{1000*r_previous_kpc:.0f}pc" + ) + else: + prev_group_name = f"ExclusiveSphere/{r_previous_kpc:.0f}kpc" + for name, prop in self.property_list.items(): + outputname = prop.name + # Skip if this property is disabled in the parameter file + if not self.property_filters[outputname]: + continue + # Skip non-DMO properties when in DMO run mode + if self.category_filter.dmo and not prop.dmo_property: + continue + # Skip if this property has a direct dependence on aperture + # size (and so would have a different value) + if self.strict_halo_copy and self.property_names[name]: + continue + aperture_sphere[name] = halo_result[ + f"{prev_group_name}/{outputname}" + ][0] + + # Determine whether to skip this halo (because of the filter or because we + # have copied over the values from the previous aperture) + if do_calculation[self.halo_filter] and (not skip_gt_enclose_radius): + if self.aperture_physical_radius_kpc is not None: + aperture_radius = self.aperture_physical_radius_kpc * unyt.kpc + else: + if self.aperture_property[0] not in halo_result: + raise RuntimeError( + f"{self.aperture_property[0]} must be enabled in the parameter file if you want to use it to define an aperture" + ) + aperture_radius = ( + self.aperture_property[1] + * halo_result[self.aperture_property[0]][0] + ) + + if search_radius < aperture_radius: raise SearchRadiusTooSmallError( "Search radius is smaller than aperture" ) @@ -3332,33 +3958,34 @@ def calculate( data, types_present, self.inclusive, - self.physical_radius_mpc * unyt.Mpc, + aperture_radius, self.stellar_ages, self.recently_heated_gas_filter, self.cold_dense_gas_filter, self.snapshot_datasets, self.softening_of_parttype, + self.boxsize, + self.cosmology, ) - for prop in self.property_list: - outputname = prop[1] - # skip properties that are masked - if not self.property_mask[outputname]: + for name, prop in self.property_list.items(): + outputname = prop.name + # Skip if this property is disabled in the parameter file + filter_name = self.property_filters[outputname] + if not filter_name: continue - # skip non-DMO properties in DMO run mode - is_dmo = prop[8] - if do_calculation["DMO"] and not is_dmo: + # Skip non-DMO properties when in DMO run mode + if self.category_filter.dmo and not prop.dmo_property: continue - name = prop[0] - shape = prop[2] - dtype = prop[3] - unit = unyt.Unit(prop[4], registry=registry) - category = prop[6] - physical = prop[10] - a_exponent = prop[11] + shape = prop.shape + dtype = prop.dtype + unit = unyt.Unit(prop.unit, registry=registry) + physical = prop.output_physical + a_exponent = prop.a_scale_exponent if not physical: unit = unit * unyt.Unit("a", registry=registry) ** a_exponent - if do_calculation[category]: + if do_calculation[filter_name]: + t0_calc = time.time() val = getattr(part_props, name) if val is not None: assert ( @@ -3376,27 +4003,26 @@ def calculate( registry=registry, ) else: - err = f'Overflow for halo {input_halo["index"]} when' + err = f'Overflow for halo {input_halo["index"]} when ' err += f"calculating {name} in aperture_properties" assert np.max(np.abs(val.to(unit).value)) < float( "inf" ), err aperture_sphere[name] += val + timings[name] = time.time() - t0_calc # add the new properties to the halo_result dictionary - for prop in self.property_list: - outputname = prop[1] - # skip properties that are masked - if not self.property_mask[outputname]: + for name, prop in self.property_list.items(): + outputname = prop.name + # Skip if this property is disabled in the parameter file + if not self.property_filters[outputname]: continue - # skip non-DMO properties in DMO run mode - is_dmo = prop[8] - if self.category_filter.dmo and not is_dmo: + # Skip non-DMO properties when in DMO run mode + if self.category_filter.dmo and not prop.dmo_property: continue - name = prop[0] - description = prop[5] - physical = prop[10] - a_exponent = prop[11] + description = prop.description + physical = prop.output_physical + a_exponent = prop.a_scale_exponent halo_result.update( { f"{self.group_name}/{outputname}": ( @@ -3407,6 +4033,23 @@ def calculate( ) } ) + if self.record_timings: + arr = unyt.unyt_array( + timings.get(name, 0), + dtype=np.float32, + units=unyt.dimensionless, + registry=registry, + ) + halo_result.update( + { + f"{self.group_name}/{outputname}_time": ( + arr, + "Time taken in seconds", + True, + None, + ) + } + ) return @@ -3422,12 +4065,14 @@ def __init__( self, cellgrid: SWIFTCellGrid, parameters: ParameterFile, - physical_radius_kpc: float, + aperture_physical_radius_kpc: float | None, + aperture_property: tuple[str, float] | None, recently_heated_gas_filter: RecentlyHeatedGasFilter, stellar_age_calculator: StellarAgeCalculator, cold_dense_gas_filter: ColdDenseGasFilter, category_filter: CategoryFilter, halo_filter: str, + all_radii_kpc: list, ): """ Construct an ExclusiveSphereProperties object with the given physical @@ -3441,9 +4086,17 @@ def __init__( - parameters: ParameterFile Parameter file object containing the parameters from the parameter file. - - physical_radius_kpc: float + - aperture_physical_radius_kpc: float | None Physical radius of the aperture. Unitless and assumed to be expressed - in units of kpc. + in units of kpc. If not None then aperture_property must be None. + If None then aperture_property must be passed. + - aperture_property: tuple[str, float] | None, + Tuple to indicate the radius of this aperture based on a previous property + calculated by SOAP. The first element should be the full name of the + property to use (e.g. BoundSubhalo/HalfMassRadius). The second element + is a multipler (e.g. if you want the aperture radius to be twice the + value of the property, then pass 2). If not None then aperture_physical_radius_kpc + must be None. If None then aperture_physical_radius_kpc must be passed. - recently_heated_gas_filter: RecentlyHeatedGasFilter Filter used to mask out gas particles that were recently heated by AGN feedback. @@ -3459,17 +4112,22 @@ def __init__( - halo_filter: str The filter to apply to this halo type. Halos which do not fulfil the filter requirements will be skipped. + - all_radii_kpc: list + A list of all the radii for which we compute an ExclusiveSphere. This + can allow us to skip property calculation for larger apertures """ super().__init__( cellgrid, parameters, - physical_radius_kpc, + aperture_physical_radius_kpc, + aperture_property, recently_heated_gas_filter, stellar_age_calculator, cold_dense_gas_filter, category_filter, halo_filter, False, + all_radii_kpc, ) @@ -3484,12 +4142,14 @@ def __init__( self, cellgrid: SWIFTCellGrid, parameters: ParameterFile, - physical_radius_kpc: float, + aperture_physical_radius_kpc: float | None, + aperture_property: tuple[str, float] | None, recently_heated_gas_filter: RecentlyHeatedGasFilter, stellar_age_calculator: StellarAgeCalculator, cold_dense_gas_filter: ColdDenseGasFilter, category_filter: CategoryFilter, halo_filter: str, + all_radii_kpc: list, ): """ Construct an InclusiveSphereProperties object with the given physical @@ -3503,9 +4163,17 @@ def __init__( - parameters: ParameterFile Parameter file object containing the parameters from the parameter file. - - physical_radius_kpc: float + - aperture_physical_radius_kpc: float | None Physical radius of the aperture. Unitless and assumed to be expressed - in units of kpc. + in units of kpc. If not None then aperture_property must be None. + If None then aperture_property must be passed. + - aperture_property: tuple[str, float] | None, + Tuple to indicate the radius of this aperture based on a previous property + calculated by SOAP. The first element should be the full name of the + property to use (e.g. BoundSubhalo/HalfMassRadius). The second element + is a multipler (e.g. if you want the aperture radius to be twice the + value of the property, then pass 2). If not None then aperture_physical_radius_kpc + must be None. If None then aperture_physical_radius_kpc must be passed. - recently_heated_gas_filter: RecentlyHeatedGasFilter Filter used to mask out gas particles that were recently heated by AGN feedback. @@ -3521,241 +4189,20 @@ def __init__( - halo_filter: str The filter to apply to this halo type. Halos which do not fulfil the filter requirements will be skipped. + - all_radii_kpc: list + A list of all the radii for which we compute an InclusiveSphere. This + can allow us to skip property calculation for larger apertures """ super().__init__( cellgrid, parameters, - physical_radius_kpc, + aperture_physical_radius_kpc, + aperture_property, recently_heated_gas_filter, stellar_age_calculator, cold_dense_gas_filter, category_filter, halo_filter, True, + all_radii_kpc, ) - - -def test_aperture_properties(): - """ - Unit test for the aperture property calculations. - - We generate 100 random "dummy" halos and feed them to - ExclusiveSphereProperties::calculate() and - InclusiveSphereProperties::calculate(). We check that the returned values - are present, and have the right units, size and dtype - """ - - import pytest - from dummy_halo_generator import DummyHaloGenerator - - # initialise the DummyHaloGenerator with a random seed - dummy_halos = DummyHaloGenerator(3256) - recently_heated_filter = dummy_halos.get_recently_heated_gas_filter() - stellar_age_calculator = StellarAgeCalculator(dummy_halos.get_cell_grid()) - cold_dense_gas_filter = dummy_halos.get_cold_dense_gas_filter() - cat_filter = CategoryFilter( - dummy_halos.get_filters( - {"general": 0, "gas": 0, "dm": 0, "star": 0, "baryon": 0} - ) - ) - parameters = ParameterFile( - parameter_dictionary={ - "aliases": { - "PartType0/ElementMassFractions": "PartType0/SmoothedElementMassFractions", - "PartType4/ElementMassFractions": "PartType4/SmoothedElementMassFractions", - } - } - ) - dummy_halos.get_cell_grid().snapshot_datasets.setup_aliases( - parameters.get_aliases() - ) - parameters.get_halo_type_variations( - "ApertureProperties", - { - "exclusive_50_kpc": {"radius_in_kpc": 50.0, "inclusive": False}, - "inclusive_50_kpc": {"radius_in_kpc": 50.0, "inclusive": True}, - }, - ) - - pc_exclusive = ExclusiveSphereProperties( - dummy_halos.get_cell_grid(), - parameters, - 50.0, - recently_heated_filter, - stellar_age_calculator, - cold_dense_gas_filter, - cat_filter, - "basic", - ) - pc_inclusive = InclusiveSphereProperties( - dummy_halos.get_cell_grid(), - parameters, - 50.0, - recently_heated_filter, - stellar_age_calculator, - cold_dense_gas_filter, - cat_filter, - "basic", - ) - - # Create a filter that no halos will satisfy - fail_filter = CategoryFilter(dummy_halos.get_filters({"general": 10000000})) - pc_filter_test = ExclusiveSphereProperties( - dummy_halos.get_cell_grid(), - parameters, - 50.0, - recently_heated_filter, - stellar_age_calculator, - cold_dense_gas_filter, - fail_filter, - "general", - ) - - # generate 100 random halos - for i in range(100): - input_halo, data, _, _, _, particle_numbers = dummy_halos.get_random_halo( - [1, 10, 100, 1000, 10000] - ) - halo_result_template = dummy_halos.get_halo_result_template(particle_numbers) - - for pc_name, pc_type, pc_calc in [ - ("ExclusiveSphere", "ExclusiveSphere", pc_exclusive), - ("InclusiveSphere", "InclusiveSphere", pc_inclusive), - ("filter_test", "ExclusiveSphere", pc_filter_test), - ]: - input_data = {} - for ptype in pc_calc.particle_properties: - if ptype in data: - input_data[ptype] = {} - for dset in pc_calc.particle_properties[ptype]: - input_data[ptype][dset] = data[ptype][dset] - input_halo_copy = input_halo.copy() - input_data_copy = input_data.copy() - - # Check halo fails if search radius is too small - halo_result = dict(halo_result_template) - if pc_name != "filter_test": - with pytest.raises(SearchRadiusTooSmallError): - pc_calc.calculate( - input_halo, 10 * unyt.kpc, input_data, halo_result - ) - # Skipped halos shouldn't ever require a larger search radius - else: - pc_calc.calculate(input_halo, 10 * unyt.kpc, input_data, halo_result) - - halo_result = dict(halo_result_template) - pc_calc.calculate(input_halo, 100 * unyt.kpc, input_data, halo_result) - assert input_halo == input_halo_copy - assert input_data == input_data_copy - - # check that the calculation returns the correct values - for prop in pc_calc.property_list: - outputname = prop[1] - size = prop[2] - dtype = prop[3] - unit_string = prop[4] - physical = prop[10] - a_exponent = prop[11] - full_name = f"{pc_type}/50kpc/{outputname}" - assert full_name in halo_result - result = halo_result[full_name][0] - assert (len(result.shape) == 0 and size == 1) or result.shape[0] == size - assert result.dtype == dtype - unit = unyt.Unit(unit_string, registry=dummy_halos.unit_registry) - if not physical: - unit = ( - unit - * unyt.Unit("a", registry=dummy_halos.unit_registry) - ** a_exponent - ) - assert result.units == unit.units - - # Check properties were not calculated for filtered halos - if pc_name == "filter_test": - for prop in pc_calc.property_list: - outputname = prop[1] - size = prop[2] - full_name = f"{pc_type}/50kpc/{outputname}" - assert np.all(halo_result[full_name][0].value == np.zeros(size)) - - # Now test the calculation for each property individually, to make sure that - # all properties read all the datasets they require - # we reuse the last random halo for this - all_parameters = parameters.get_parameters() - for property in all_parameters["ApertureProperties"]["properties"]: - print(f"Testing only {property}...") - single_property = dict(all_parameters) - for other_property in all_parameters["ApertureProperties"]["properties"]: - single_property["ApertureProperties"]["properties"][other_property] = ( - other_property == property - ) or other_property.startswith("NumberOf") - single_parameters = ParameterFile(parameter_dictionary=single_property) - pc_exclusive = ExclusiveSphereProperties( - dummy_halos.get_cell_grid(), - single_parameters, - 50.0, - recently_heated_filter, - stellar_age_calculator, - cold_dense_gas_filter, - cat_filter, - "basic", - ) - pc_inclusive = InclusiveSphereProperties( - dummy_halos.get_cell_grid(), - single_parameters, - 50.0, - recently_heated_filter, - stellar_age_calculator, - cold_dense_gas_filter, - cat_filter, - "basic", - ) - - halo_result_template = dummy_halos.get_halo_result_template(particle_numbers) - - for pc_type, pc_calc in [ - ("ExclusiveSphere", pc_exclusive), - ("InclusiveSphere", pc_inclusive), - ]: - input_data = {} - for ptype in pc_calc.particle_properties: - if ptype in data: - input_data[ptype] = {} - for dset in pc_calc.particle_properties[ptype]: - input_data[ptype][dset] = data[ptype][dset] - input_halo_copy = input_halo.copy() - input_data_copy = input_data.copy() - halo_result = dict(halo_result_template) - pc_calc.calculate(input_halo, 100 * unyt.kpc, input_data, halo_result) - assert input_halo == input_halo_copy - assert input_data == input_data_copy - - # check that the calculation returns the correct values - for prop in pc_calc.property_list: - outputname = prop[1] - if not outputname == property: - continue - size = prop[2] - dtype = prop[3] - unit_string = prop[4] - full_name = f"{pc_type}/50kpc/{outputname}" - assert full_name in halo_result - result = halo_result[full_name][0] - assert (len(result.shape) == 0 and size == 1) or result.shape[0] == size - assert result.dtype == dtype - unit = unyt.Unit(unit_string, registry=dummy_halos.unit_registry) - assert result.units.same_dimensions_as(unit.units) - - dummy_halos.get_cell_grid().snapshot_datasets.print_dataset_log() - - -if __name__ == "__main__": - """ - Standalone version of the program: just run the unit test. - - Note that this can also be achieved by running "pytest *.py" in the folder. - """ - - print("Running test_aperture_properties()...") - test_aperture_properties() - print("Test passed.") diff --git a/halo_properties.py b/SOAP/particle_selection/halo_properties.py similarity index 89% rename from halo_properties.py rename to SOAP/particle_selection/halo_properties.py index 5996c18f..3198ae4a 100644 --- a/halo_properties.py +++ b/SOAP/particle_selection/halo_properties.py @@ -29,3 +29,7 @@ def __init__(self, cellgrid): "PartType5": cellgrid.baryon_softening, "PartType6": cellgrid.nu_softening, } # Softening length of each particle type + + # No density criterion by default + self.mean_density_multiple = None + self.critical_density_multiple = None diff --git a/projected_aperture_properties.py b/SOAP/particle_selection/projected_aperture_properties.py similarity index 72% rename from projected_aperture_properties.py rename to SOAP/particle_selection/projected_aperture_properties.py index 68488c55..6b7be711 100644 --- a/projected_aperture_properties.py +++ b/SOAP/particle_selection/projected_aperture_properties.py @@ -19,22 +19,29 @@ projections. Besides this difference, the approach is very similar. """ +import time +from typing import Dict, List + import numpy as np +from numpy.typing import NDArray import unyt -from swift_cells import SWIFTCellGrid -from halo_properties import HaloProperty, SearchRadiusTooSmallError -from dataset_names import mass_dataset -from half_mass_radius import get_half_mass_radius -from property_table import PropertyTable -from kinematic_properties import get_projected_inertia_tensor -from lazy_properties import lazy_property -from category_filter import CategoryFilter -from parameter_file import ParameterFile -from snapshot_datasets import SnapshotDatasets - -from typing import Dict, List -from numpy.typing import NDArray +from .halo_properties import HaloProperty, SearchRadiusTooSmallError +from SOAP.core.swift_cells import SWIFTCellGrid +from SOAP.core.lazy_properties import lazy_property +from SOAP.core.category_filter import CategoryFilter +from SOAP.core.parameter_file import ParameterFile +from SOAP.core.snapshot_datasets import SnapshotDatasets +from SOAP.core.dataset_names import mass_dataset +from SOAP.property_calculation.half_mass_radius import ( + get_half_mass_radius, + get_half_light_radius, +) +from SOAP.property_table import PropertyTable +from SOAP.property_calculation.inertia_tensors import ( + get_projected_inertia_tensor_mass_weighted, + get_projected_inertia_tensor_luminosity_weighted, +) class ProjectedApertureParticleData: @@ -53,6 +60,7 @@ def __init__( types_present: List[str], aperture_radius: unyt.unyt_quantity, snapshot_datasets: SnapshotDatasets, + boxsize: unyt.unyt_quantity, ): """ Constructor. @@ -70,12 +78,15 @@ def __init__( - snapshot_datasets: SnapshotDatasets Object containing metadata about the datasets in the snapshot, like appropriate aliases and column names. + - boxsize: unyt.unyt_quantity + Boxsize for correcting periodic boundary conditions """ self.input_halo = input_halo self.data = data self.types_present = types_present self.aperture_radius = aperture_radius self.snapshot_datasets = snapshot_datasets + self.boxsize = boxsize self.compute_basics() def get_dataset(self, name: str) -> unyt.unyt_array: @@ -749,9 +760,21 @@ def com(self) -> unyt.unyt_array: """ if self.Mtot == 0: return None - return (self.mass_fraction[:, None] * self.proj_position).sum( - axis=0 - ) + self.centre + return ( + (self.mass_fraction[:, None] * self.proj_position).sum(axis=0) + self.centre + ) % self.part_props.boxsize + + @lazy_property + def com_star(self) -> unyt.unyt_array: + """ + Centre of mass of star particles in the subhalo. + """ + if self.Mstar == 0: + return None + return ( + (self.star_mass_fraction[:, None] * self.proj_pos_star).sum(axis=0) + + self.centre + ) % self.part_props.boxsize @lazy_property def vcom(self) -> unyt.unyt_array: @@ -771,7 +794,7 @@ def ProjectedTotalInertiaTensor(self) -> unyt.unyt_array: """ if self.Mtot == 0: return None - return get_projected_inertia_tensor( + return get_projected_inertia_tensor_mass_weighted( self.part_props.mass, self.part_props.position, self.iproj, @@ -787,7 +810,7 @@ def ProjectedTotalInertiaTensorReduced(self) -> unyt.unyt_array: """ if self.Mtot == 0: return None - return get_projected_inertia_tensor( + return get_projected_inertia_tensor_mass_weighted( self.part_props.mass, self.part_props.position, self.iproj, @@ -803,7 +826,7 @@ def ProjectedTotalInertiaTensorNoniterative(self) -> unyt.unyt_array: """ if self.Mtot == 0: return None - return get_projected_inertia_tensor( + return get_projected_inertia_tensor_mass_weighted( self.proj_mass, self.proj_position, self.iproj, @@ -819,7 +842,7 @@ def ProjectedTotalInertiaTensorReducedNoniterative(self) -> unyt.unyt_array: """ if self.Mtot == 0: return None - return get_projected_inertia_tensor( + return get_projected_inertia_tensor_mass_weighted( self.proj_mass, self.proj_position, self.iproj, @@ -857,7 +880,7 @@ def gas_inertia_tensor(self, **kwargs) -> unyt.unyt_array: """ mass = self.part_props.mass[self.part_props.types == 0] position = self.part_props.position[self.part_props.types == 0] - return get_projected_inertia_tensor( + return get_projected_inertia_tensor_mass_weighted( mass, position, self.iproj, self.aperture_radius, **kwargs ) @@ -891,7 +914,7 @@ def ProjectedGasInertiaTensorNoniterative(self) -> unyt.unyt_array: """ if self.Mgas == 0: return None - return get_projected_inertia_tensor( + return get_projected_inertia_tensor_mass_weighted( self.proj_mass_gas, self.proj_pos_gas, self.iproj, @@ -907,7 +930,7 @@ def ProjectedGasInertiaTensorReducedNoniterative(self) -> unyt.unyt_array: """ if self.Mgas == 0: return None - return get_projected_inertia_tensor( + return get_projected_inertia_tensor_mass_weighted( self.proj_mass_gas, self.proj_pos_gas, self.iproj, @@ -968,10 +991,28 @@ def stellar_inertia_tensor(self, **kwargs) -> unyt.unyt_array: """ mass = self.part_props.mass[self.part_props.types == 4] position = self.part_props.position[self.part_props.types == 4] - return get_projected_inertia_tensor( + return get_projected_inertia_tensor_mass_weighted( mass, position, self.iproj, self.aperture_radius, **kwargs ) + def stellar_inertia_tensor_luminosity_weighted(self, **kwargs) -> unyt.unyt_array: + """ + Helper function for calculating projected luminosity-weighted stellar inertia tensors + """ + mass = self.part_props.mass[self.part_props.types == 4] + position = self.part_props.position[self.part_props.types == 4] + + # self.stellar_luminosities correspond to bound particles within the + # initial aperture. In the iterative case we want all bound, regardless + # of whether they are within the initial projected aperture. Hence, we + # cannot use self.stellar_luminosities directly. + luminosity = self.part_props.get_dataset("PartType4/Luminosities")[ + self.star_mask_all + ] + return get_projected_inertia_tensor_luminosity_weighted( + luminosity, position, self.iproj, self.aperture_radius, **kwargs + ) + @lazy_property def ProjectedStellarInertiaTensor(self) -> unyt.unyt_array: """ @@ -1002,7 +1043,7 @@ def ProjectedStellarInertiaTensorNoniterative(self) -> unyt.unyt_array: """ if self.Mstar == 0: return None - return get_projected_inertia_tensor( + return get_projected_inertia_tensor_mass_weighted( self.proj_mass_star, self.proj_pos_star, self.iproj, @@ -1018,7 +1059,7 @@ def ProjectedStellarInertiaTensorReducedNoniterative(self) -> unyt.unyt_array: """ if self.Mstar == 0: return None - return get_projected_inertia_tensor( + return get_projected_inertia_tensor_mass_weighted( self.proj_mass_star, self.proj_pos_star, self.iproj, @@ -1027,6 +1068,67 @@ def ProjectedStellarInertiaTensorReducedNoniterative(self) -> unyt.unyt_array: max_iterations=1, ) + @lazy_property + def ProjectedStellarInertiaTensorLuminosityWeighted(self) -> unyt.unyt_array: + """ + Inertia tensor of the stellar luminosity distribution for each GAMA band in projection. + Computed iteratively using an ellipse with area equal to that of a circle with radius + equal to the aperture radius. Only considers bound particles within the projected aperture. + """ + if self.Mstar == 0: + return None + return self.stellar_inertia_tensor_luminosity_weighted() + + @lazy_property + def ProjectedStellarInertiaTensorReducedLuminosityWeighted( + self, + ) -> unyt.unyt_array: + """ + Reduced inertia tensor of the stellar luminosity distribution for each GAMA band in projection. + Computed iteratively using an ellipse with area equal to that of a circle with radius + equal to the aperture radius. Only considers bound particles within the projected aperture. + """ + if self.Mstar == 0: + return None + return self.stellar_inertia_tensor_luminosity_weighted(reduced=True) + + @lazy_property + def ProjectedStellarInertiaTensorNoniterativeLuminosityWeighted( + self, + ) -> unyt.unyt_array: + """ + Inertia tensor of the stellar luminosity distribution for each GAMA band in projection. + Computed using all bound star particles within the projected aperture. + """ + if self.Mstar == 0: + return None + return get_projected_inertia_tensor_luminosity_weighted( + self.stellar_luminosities, # Bound and within initial aperture. + self.proj_pos_star, + self.iproj, + self.aperture_radius, + max_iterations=1, + ) + + @lazy_property + def ProjectedStellarInertiaTensorReducedNoniterativeLuminosityWeighted( + self, + ) -> unyt.unyt_array: + """ + Reduced inertia tensor of the stellar luminosity distribution for each GAMA band in projection. + Computed using all bound star particles within the projected aperture. + """ + if self.Mstar == 0: + return None + return get_projected_inertia_tensor_luminosity_weighted( + self.stellar_luminosities, # Bound and within initial aperture. + self.proj_pos_star, + self.iproj, + self.aperture_radius, + reduced=True, + max_iterations=1, + ) + @lazy_property def gas_mask_all(self) -> NDArray[bool]: """ @@ -1037,6 +1139,33 @@ def gas_mask_all(self) -> NDArray[bool]: return None return self.part_props.get_dataset("PartType0/GroupNr_bound") == self.index + @lazy_property + def gas_total_dust_mass_fractions(self) -> unyt.unyt_array: + """ + Total dust mass fractions in gas particles in the projection. + """ + if self.Ngas == 0: + return None + return self.part_props.get_dataset("PartType0/TotalDustMassFractions")[ + self.gas_mask_all + ][self.gas_mask_ap] + + @lazy_property + def proj_mass_dust(self) -> unyt.unyt_array: + """ + Dust masses of the gas particles in the subhalo. + """ + return self.gas_total_dust_mass_fractions * self.proj_mass_gas + + @lazy_property + def DustMass(self) -> unyt.unyt_quantity: + """ + Total dust mass of the gas particles in the subhalo. + """ + if self.Ngas == 0: + return None + return self.proj_mass_dust.sum() + @lazy_property def gas_SFR(self) -> unyt.unyt_array: """ @@ -1151,7 +1280,7 @@ def gas_mass_H(self) -> unyt.unyt_array: self.gas_element_fractions[ :, self.part_props.snapshot_datasets.get_column_index( - "ElementMassFractions", "Hydrogen" + "PartType0/ElementMassFractions", "Hydrogen" ), ] * self.proj_mass_gas @@ -1168,7 +1297,7 @@ def gas_mass_He(self) -> unyt.unyt_array: self.gas_element_fractions[ :, self.part_props.snapshot_datasets.get_column_index( - "ElementMassFractions", "Helium" + "PartType0/ElementMassFractions", "Helium" ), ] * self.proj_mass_gas @@ -1197,7 +1326,7 @@ def gas_mass_HI(self) -> unyt.unyt_array: * self.gas_species_fractions[ :, self.part_props.snapshot_datasets.get_column_index( - "SpeciesFractions", "HI" + "PartType0/SpeciesFractions", "HI" ), ] ) @@ -1214,7 +1343,7 @@ def gas_mass_H2(self) -> unyt.unyt_array: * self.gas_species_fractions[ :, self.part_props.snapshot_datasets.get_column_index( - "SpeciesFractions", "H2" + "PartType0/SpeciesFractions", "H2" ), ] * 2.0 @@ -1278,7 +1407,7 @@ def star_mass_O(self) -> unyt.unyt_array: self.star_element_fractions[ :, self.part_props.snapshot_datasets.get_column_index( - "ElementMassFractions", "Oxygen" + "PartType4/ElementMassFractions", "Oxygen" ), ] * self.proj_mass_star @@ -1295,7 +1424,7 @@ def star_mass_Mg(self) -> unyt.unyt_array: self.star_element_fractions[ :, self.part_props.snapshot_datasets.get_column_index( - "ElementMassFractions", "Magnesium" + "PartType4/ElementMassFractions", "Magnesium" ), ] * self.proj_mass_star @@ -1312,7 +1441,7 @@ def star_mass_Fe(self) -> unyt.unyt_array: self.star_element_fractions[ :, self.part_props.snapshot_datasets.get_column_index( - "ElementMassFractions", "Iron" + "PartType4/ElementMassFractions", "Iron" ), ] * self.proj_mass_star @@ -1368,6 +1497,43 @@ def HalfMassRadiusGas(self) -> unyt.unyt_quantity: self.proj_radius[self.proj_type == 0], self.proj_mass_gas, self.Mgas ) + @lazy_property + def HalfMassRadiusDust(self) -> unyt.unyt_quantity: + """ + Half-mass radius of the dust. + """ + if self.Ngas == 0: + return None + return get_half_mass_radius( + self.proj_radius[self.proj_type == 0], self.proj_mass_dust, self.DustMass + ) + + @lazy_property + def HalfMassRadiusAtomicHydrogen(self) -> unyt.unyt_quantity: + """ + Half-mass radius of the atomic hydrogen. + """ + if self.Ngas == 0: + return None + return get_half_mass_radius( + self.proj_radius[self.proj_type == 0], + self.gas_mass_HI, + self.AtomicHydrogenMass, + ) + + @lazy_property + def HalfMassRadiusMolecularHydrogen(self) -> unyt.unyt_quantity: + """ + Half-mass radius of the molecular hydrogen. + """ + if self.Ngas == 0: + return None + return get_half_mass_radius( + self.proj_radius[self.proj_type == 0], + self.gas_mass_H2, + self.MolecularHydrogenMass, + ) + @lazy_property def HalfMassRadiusDM(self) -> unyt.unyt_quantity: """ @@ -1386,6 +1552,19 @@ def HalfMassRadiusStar(self) -> unyt.unyt_quantity: self.proj_radius[self.proj_type == 4], self.proj_mass_star, self.Mstar ) + @lazy_property + def HalfLightRadiusStar(self) -> unyt.unyt_array: + """ + Half light radius of stars for the 9 GAMA bands. + """ + if self.Nstar == 0: + return None + return get_half_light_radius( + self.proj_radius[self.proj_type == 4], + self.stellar_luminosities, + self.StellarLuminosity, + ) + @lazy_property def HalfMassRadiusBaryon(self) -> unyt.unyt_quantity: """ @@ -1408,93 +1587,106 @@ class ProjectedApertureProperties(HaloProperty): the halo along the projection axis. """ - """ - List of properties from the table that we want to compute. - Each property should have a corresponding method/property/lazy_property in - the SingleProjectionProjectedApertureParticleData class above. - """ - property_list = [ - (prop, *PropertyTable.full_property_list[prop]) - for prop in [ - "Mtot", - "Mgas", - "Mdm", - "Mstar", - "Mstar_init", - "Mbh_dynamical", - "Mbh_subgrid", - "Ngas", - "Ndm", - "Nstar", - "Nbh", - "com", - "vcom", - "SFR", - "AveragedStarFormationRate", - "StellarLuminosity", - "HalfMassRadiusGas", - "HalfMassRadiusDM", - "HalfMassRadiusStar", - "HalfMassRadiusBaryon", - "proj_veldisp_gas", - "proj_veldisp_dm", - "proj_veldisp_star", - "BHmaxAR", - "BHmaxM", - "BHmaxID", - "BHmaxpos", - "BHmaxvel", - "BHlasteventa", - "BHmaxlasteventa", - "BlackHolesTotalInjectedThermalEnergy", - "BlackHolesTotalInjectedJetEnergy", - "MostMassiveBlackHoleAveragedAccretionRate", - "MostMassiveBlackHoleNumberOfAGNEvents", - "MostMassiveBlackHoleAccretionMode", - "MostMassiveBlackHoleGWMassLoss", - "MostMassiveBlackHoleInjectedJetEnergyByMode", - "MostMassiveBlackHoleLastJetEventScalefactor", - "MostMassiveBlackHoleNumberOfAGNJetEvents", - "MostMassiveBlackHoleNumberOfMergers", - "MostMassiveBlackHoleRadiatedEnergyByMode", - "MostMassiveBlackHoleTotalAccretedMassesByMode", - "MostMassiveBlackHoleWindEnergyByMode", - "MostMassiveBlackHoleInjectedThermalEnergy", - "MostMassiveBlackHoleSpin", - "MostMassiveBlackHoleTotalAccretedMass", - "MostMassiveBlackHoleFormationScalefactor", - "ProjectedTotalInertiaTensor", - "ProjectedGasInertiaTensor", - "ProjectedStellarInertiaTensor", - "ProjectedTotalInertiaTensorReduced", - "ProjectedGasInertiaTensorReduced", - "ProjectedStellarInertiaTensorReduced", - "ProjectedTotalInertiaTensorNoniterative", - "ProjectedGasInertiaTensorNoniterative", - "ProjectedStellarInertiaTensorNoniterative", - "ProjectedTotalInertiaTensorReducedNoniterative", - "ProjectedGasInertiaTensorReducedNoniterative", - "ProjectedStellarInertiaTensorReducedNoniterative", - "HydrogenMass", - "HeliumMass", - "MolecularHydrogenMass", - "AtomicHydrogenMass", - "starFefrac", - "starMgfrac", - "starOfrac", - "starmetalfrac", - "gasmetalfrac", - "gasmetalfrac_SF", - ] - ] + base_halo_type = "ProjectedApertureProperties" + # Properties to calculate. The key is the name of the property, + # the value indicates the property has a direct dependence on aperture size. + # This is needed since for larger apertures we sometimes copy across the + # values computed by the previous aperture (if the number of particles was + # the same for both apertures), but we can't do this for all properties + property_names = { + "Mtot": False, + "Mgas": False, + "Mdm": False, + "Mstar": False, + "Mstar_init": False, + "Mbh_dynamical": False, + "Mbh_subgrid": False, + "DustMass": True, + "Ngas": False, + "Ndm": False, + "Nstar": False, + "Nbh": False, + "com": False, + "com_star": False, + "vcom": False, + "SFR": False, + "AveragedStarFormationRate": False, + "StellarLuminosity": False, + "HalfMassRadiusGas": False, + "HalfMassRadiusDust": False, + "HalfMassRadiusAtomicHydrogen": False, + "HalfMassRadiusMolecularHydrogen": False, + "HalfMassRadiusDM": False, + "HalfMassRadiusStar": False, + "HalfLightRadiusStar": False, + "HalfMassRadiusBaryon": False, + "proj_veldisp_gas": False, + "proj_veldisp_dm": False, + "proj_veldisp_star": False, + "BHmaxAR": False, + "BHmaxM": False, + "BHmaxID": False, + "BHmaxpos": False, + "BHmaxvel": False, + "BHlasteventa": False, + "BHmaxlasteventa": False, + "BlackHolesTotalInjectedThermalEnergy": False, + "BlackHolesTotalInjectedJetEnergy": False, + "MostMassiveBlackHoleAveragedAccretionRate": False, + "MostMassiveBlackHoleNumberOfAGNEvents": False, + "MostMassiveBlackHoleAccretionMode": False, + "MostMassiveBlackHoleGWMassLoss": False, + "MostMassiveBlackHoleInjectedJetEnergyByMode": False, + "MostMassiveBlackHoleLastJetEventScalefactor": False, + "MostMassiveBlackHoleNumberOfAGNJetEvents": False, + "MostMassiveBlackHoleNumberOfMergers": False, + "MostMassiveBlackHoleRadiatedEnergyByMode": False, + "MostMassiveBlackHoleTotalAccretedMassesByMode": False, + "MostMassiveBlackHoleWindEnergyByMode": False, + "MostMassiveBlackHoleInjectedThermalEnergy": False, + "MostMassiveBlackHoleSpin": False, + "MostMassiveBlackHoleTotalAccretedMass": False, + "MostMassiveBlackHoleFormationScalefactor": False, + "ProjectedTotalInertiaTensor": True, + "ProjectedGasInertiaTensor": True, + "ProjectedStellarInertiaTensor": True, + "ProjectedStellarInertiaTensorLuminosityWeighted": True, + "ProjectedTotalInertiaTensorReduced": True, + "ProjectedGasInertiaTensorReduced": True, + "ProjectedStellarInertiaTensorReduced": True, + "ProjectedStellarInertiaTensorReducedLuminosityWeighted": True, + "ProjectedTotalInertiaTensorNoniterative": False, + "ProjectedGasInertiaTensorNoniterative": False, + "ProjectedStellarInertiaTensorNoniterative": False, + "ProjectedStellarInertiaTensorNoniterativeLuminosityWeighted": False, + "ProjectedTotalInertiaTensorReducedNoniterative": False, + "ProjectedGasInertiaTensorReducedNoniterative": False, + "ProjectedStellarInertiaTensorReducedNoniterative": False, + "ProjectedStellarInertiaTensorReducedNoniterativeLuminosityWeighted": False, + "HydrogenMass": False, + "HeliumMass": False, + "MolecularHydrogenMass": False, + "AtomicHydrogenMass": False, + "starFefrac": False, + "starMgfrac": False, + "starOfrac": False, + "starmetalfrac": False, + "gasmetalfrac": False, + "gasmetalfrac_SF": False, + } + property_list = { + name: PropertyTable.full_property_list[name] for name in property_names + } def __init__( self, cellgrid: SWIFTCellGrid, parameters: ParameterFile, - physical_radius_kpc: float, + aperture_physical_radius_kpc: float | None, + aperture_property: tuple[str, float] | None, category_filter: CategoryFilter, halo_filter: str, + all_radii_kpc: list, ): """ Construct an ProjectedApertureProperties object with the given physical @@ -1507,6 +1699,18 @@ def __init__( - parameters: ParameterFile Parameter file object containing the parameters from the parameter file. + - aperture_physical_radius_kpc: float | None + Physical radius of the aperture. Unitless and assumed to be expressed + in units of kpc. If not None then aperture_property must be None. + If None then aperture_property must be passed. + - aperture_property: tuple[str, float] | None, + Tuple to indicate the radius of this aperture based on a previous property + calculated by SOAP. The first element should be the full name of the + property to use (e.g. BoundSubhalo/HalfMassRadius). The second element + is a multipler (e.g. if you want the aperture radius to be twice the + value of the property, then pass 2). If not None then + aperture_physical_radius_kpc must be None. If None then + aperture_physical_radius_kpc must be passed. - physical_radius_kpc: float Physical radius of the aperture. Unitless and assumed to be expressed in units of kpc. @@ -1517,26 +1721,47 @@ def __init__( - halo_filter: str The filter to apply to this halo type. Halos which do not fulfil the filter requirements will be skipped. + - all_radii_kpc: list + A list of all the radii for which we compute a ProjectedAperture. This + can allow us to skip the property calculation for larger apertures """ super().__init__(cellgrid) - self.property_mask = parameters.get_property_mask( - "ProjectedApertureProperties", [prop[1] for prop in self.property_list] + self.property_filters = parameters.get_property_filters( + "ProjectedApertureProperties", + [prop.name for prop in self.property_list.values()], ) - # No density criterion - self.mean_density_multiple = None - self.critical_density_multiple = None - - # Minimum physical radius to read in (pMpc) - self.physical_radius_mpc = 0.001 * physical_radius_kpc - self.category_filter = category_filter self.snapshot_datasets = cellgrid.snapshot_datasets + self.aperture_physical_radius_kpc = aperture_physical_radius_kpc + self.aperture_property = aperture_property self.halo_filter = halo_filter - - self.name = f"projected_aperture_{physical_radius_kpc:.0f}kpc" - self.group_name = f"ProjectedAperture/{self.physical_radius_mpc*1000.:.0f}kpc" + self.record_timings = parameters.record_property_timings + self.all_radii_kpc = all_radii_kpc + self.strict_halo_copy = parameters.strict_halo_copy() + self.boxsize = cellgrid.boxsize + + if self.aperture_physical_radius_kpc is not None: + self.physical_radius_mpc = 0.001 * self.aperture_physical_radius_kpc + assert self.aperture_physical_radius_kpc >= 0.001 + if self.aperture_physical_radius_kpc < 1: + aperture_name = f"{1000*self.aperture_physical_radius_kpc:.0f}pc" + else: + aperture_name = f"{self.aperture_physical_radius_kpc:.0f}kpc" + else: + prop = self.aperture_property[0].split("/")[-1] + multiplier = self.aperture_property[1] + if multiplier == 1: + aperture_name = prop + else: + aperture_name = f"{int(multiplier)}x{prop}" + # This value needs to be set since it's used to guess the initial + # load region for each particle + self.physical_radius_mpc = 0 + + self.name = f"projected_aperture_{aperture_name}" + self.group_name = f"ProjectedAperture/{aperture_name}" self.mask_metadata = self.category_filter.get_filter_metadata(halo_filter) # List of particle properties we need to read in @@ -1555,15 +1780,15 @@ def __init__( } # add additional particle properties based on the selected halo # properties in the parameter file - for prop in self.property_list: - outputname = prop[1] - if not self.property_mask[outputname]: + for name, prop in self.property_list.items(): + outputname = prop.name + # Skip if this property is disabled in the parameter file + if not self.property_filters[outputname]: continue - is_dmo = prop[8] - if self.category_filter.dmo and not is_dmo: + # Skip non-DMO properties when in DMO run mode + if self.category_filter.dmo and not prop.dmo_property: continue - partprops = prop[9] - for partprop in partprops: + for partprop in prop.particle_properties: pgroup, dset = parameters.get_particle_property(partprop) if not pgroup in self.particle_properties: self.particle_properties[pgroup] = [] @@ -1597,6 +1822,21 @@ def calculate( registry = input_halo["cofp"].units.registry projected_aperture = {} + timings = {} + + skip_gt_enclose_radius = False + # Determine if the previous aperture already enclosed + # all the bound particles of the subhalo + if self.aperture_physical_radius_kpc is not None: + i_radius = self.all_radii_kpc.index(1000 * self.physical_radius_mpc) + else: + i_radius = 0 + if i_radius != 0 and ("BoundSubhalo/EncloseRadius" in halo_result): + r_enclose = halo_result["BoundSubhalo/EncloseRadius"][0] + r_previous_kpc = self.all_radii_kpc[i_radius - 1] + if r_previous_kpc * unyt.kpc > r_enclose: + skip_gt_enclose_radius = True + # loop over the different projections for projname in ["projx", "projy", "projz"]: projected_aperture[projname] = {} @@ -1605,21 +1845,20 @@ def calculate( # all variables are defined with physical units and an appropriate dtype # we need to use the custom unit registry so that everything can be converted # back to snapshot units in the end - for prop in self.property_list: - outputname = prop[1] - # skip properties that are masked - if not self.property_mask[outputname]: + for name, prop in self.property_list.items(): + outputname = prop.name + # Skip if this property is disabled in the parameter file + if not self.property_filters[outputname]: continue - # skip non-DMO properties in DMO run mode - is_dmo = prop[8] - if self.category_filter.dmo and not is_dmo: + # Skip non-DMO properties when in DMO run mode + if self.category_filter.dmo and not prop.dmo_property: continue - name = prop[0] - shape = prop[2] - dtype = prop[3] - unit = unyt.Unit(prop[4], registry=registry) - physical = prop[10] - a_exponent = prop[11] + + shape = prop.shape + dtype = prop.dtype + unit = unyt.Unit(prop.unit, registry=registry) + physical = prop.output_physical + a_exponent = prop.a_scale_exponent if shape > 1: val = [0] * shape else: @@ -1630,43 +1869,69 @@ def calculate( val, dtype=dtype, units=unit, registry=registry ) - # Determine whether to skip halo - if do_calculation[self.halo_filter]: + if skip_gt_enclose_radius: + if r_previous_kpc < 1: + prev_group_name = ( + f"ProjectedAperture/{1000*r_previous_kpc:.0f}pc" + ) + else: + prev_group_name = f"ProjectedAperture/{r_previous_kpc:.0f}kpc" + prev_prop = f"{prev_group_name}/{projname}/{outputname}" + # Skip if this property has a direct dependence on + # aperture size (and so would have a different value) + if self.strict_halo_copy and self.property_names[name]: + continue + projected_aperture[projname][name] = halo_result[prev_prop][0] + + # Determine whether to skip this halo (because of the filter or because we + # have copied over the values from the previous aperture) + if do_calculation[self.halo_filter] and (not skip_gt_enclose_radius): # For projected apertures we are only using bound particles - # Therefore we don't need to check if the serach_radius is large enough, + # Therefore we don't need to check if the search_radius is large enough, # because all particles will have been loaded + if self.aperture_physical_radius_kpc is not None: + aperture_radius = self.aperture_physical_radius_kpc * unyt.kpc + else: + if self.aperture_property[0] not in halo_result: + raise RuntimeError( + f"{self.aperture_property[0]} must be enabled in the parameter file if you want to use it to define an aperture" + ) + aperture_radius = ( + self.aperture_property[1] + * halo_result[self.aperture_property[0]][0] + ) types_present = [type for type in self.particle_properties if type in data] part_props = ProjectedApertureParticleData( input_halo, data, types_present, - self.physical_radius_mpc * unyt.Mpc, + aperture_radius, self.snapshot_datasets, + self.boxsize, ) for projname in ["projx", "projy", "projz"]: proj_part_props = SingleProjectionProjectedApertureParticleData( part_props, projname ) - for prop in self.property_list: - outputname = prop[1] - # skip properties that are masked - if not self.property_mask[outputname]: + for name, prop in self.property_list.items(): + outputname = prop.name + # Skip if this property is disabled in the parameter file + filter_name = self.property_filters[outputname] + if not filter_name: continue - # skip non-DMO properties in DMO run mode - is_dmo = prop[8] - if do_calculation["DMO"] and not is_dmo: + # Skip non-DMO properties when in DMO run mode + if self.category_filter.dmo and not prop.dmo_property: continue - name = prop[0] - shape = prop[2] - dtype = prop[3] - unit = unyt.Unit(prop[4], registry=registry) - category = prop[6] - physical = prop[10] - a_exponent = prop[11] + shape = prop.shape + dtype = prop.dtype + unit = unyt.Unit(prop.unit, registry=registry) + physical = prop.output_physical + a_exponent = prop.a_scale_exponent if not physical: unit = unit * unyt.Unit("a", registry=registry) ** a_exponent - if do_calculation[category]: + if do_calculation[filter_name]: + t0_calc = time.time() val = getattr(proj_part_props, name) if val is not None: assert ( @@ -1684,209 +1949,54 @@ def calculate( registry=registry, ) else: - err = f'Overflow for halo {input_halo["index"]} when' + err = f'Overflow for halo {input_halo["index"]} when ' err += f"calculating {name} in projected_properties" assert np.max(np.abs(val.to(unit).value)) < float( "inf" ), err projected_aperture[projname][name] += val + # Include the time from previous projection calculations + timings[name] = timings.get(name, 0) + time.time() - t0_calc - for projname in ["projx", "projy", "projz"]: - # add the new properties to the halo_result dictionary - prefix = ( - f"ProjectedAperture/{self.physical_radius_mpc*1000.:.0f}kpc/{projname}" - ) - for prop in self.property_list: - outputname = prop[1] - # skip properties that are masked - if not self.property_mask[outputname]: - continue - # skip non-DMO properties in DMO run mode - is_dmo = prop[8] - if self.category_filter.dmo and not is_dmo: - continue - name = prop[0] - description = prop[5] - physical = prop[10] - a_exponent = prop[11] + for name, prop in self.property_list.items(): + outputname = prop.name + # Skip if this property is disabled in the parameter file + if not self.property_filters[outputname]: + continue + # Skip non-DMO properties when in DMO run mode + if self.category_filter.dmo and not prop.dmo_property: + continue + + for projname in ["projx", "projy", "projz"]: + # add the new properties to the halo_result dictionary halo_result.update( { - f"{prefix}/{outputname}": ( + f"{self.group_name}/{projname}/{outputname}": ( projected_aperture[projname][name], - description, - physical, - a_exponent, + prop.description, + prop.output_physical, + prop.a_scale_exponent, ) } ) - return - - -def test_projected_aperture_properties(): - """ - Unit test for the projected aperture calculation. - - Generates 100 random halos and passes them on to - ProjectedApertureProperties::calculate(). - Tests that all expected return values are computed and have the right size, - dtype and units. - """ - - import pytest - from dummy_halo_generator import DummyHaloGenerator - - dummy_halos = DummyHaloGenerator(127) - category_filter = CategoryFilter(dummy_halos.get_filters({"general": 100})) - parameters = ParameterFile( - parameter_dictionary={ - "aliases": { - "PartType0/ElementMassFractions": "PartType0/SmoothedElementMassFractions", - "PartType4/ElementMassFractions": "PartType4/SmoothedElementMassFractions", - } - } - ) - dummy_halos.get_cell_grid().snapshot_datasets.setup_aliases( - parameters.get_aliases() - ) - parameters.get_halo_type_variations( - "ProjectedApertureProperties", {"30_kpc": {"radius_in_kpc": 30.0}} - ) - - pc_projected = ProjectedApertureProperties( - dummy_halos.get_cell_grid(), parameters, 30.0, category_filter, "basic" - ) - - # Create a filter that no halos will satisfy - fail_filter = CategoryFilter(dummy_halos.get_filters({"general": 10000000})) - pc_filter_test = ProjectedApertureProperties( - dummy_halos.get_cell_grid(), parameters, 30.0, fail_filter, "general" - ) - - for i in range(100): - input_halo, data, _, _, _, particle_numbers = dummy_halos.get_random_halo( - [1, 10, 100, 1000, 10000] - ) - halo_result_template = dummy_halos.get_halo_result_template(particle_numbers) - - for pc_name, pc_calc in [ - ("ProjectedAperture", pc_projected), - ("filter_test", pc_filter_test), - ]: - input_data = {} - for ptype in pc_calc.particle_properties: - if ptype in data: - input_data[ptype] = {} - for dset in pc_calc.particle_properties[ptype]: - input_data[ptype][dset] = data[ptype][dset] - input_halo_copy = input_halo.copy() - input_data_copy = input_data.copy() - - halo_result = dict(halo_result_template) - pc_calc.calculate(input_halo, 50 * unyt.kpc, input_data, halo_result) - assert input_halo == input_halo_copy - assert input_data == input_data_copy - - for proj in ["projx", "projy", "projz"]: - for prop in pc_calc.property_list: - outputname = prop[1] - size = prop[2] - dtype = prop[3] - unit_string = prop[4] - full_name = f"ProjectedAperture/30kpc/{proj}/{outputname}" - assert full_name in halo_result - result = halo_result[full_name][0] - assert (len(result.shape) == 0 and size == 1) or result.shape[ - 0 - ] == size - assert result.dtype == dtype - unit = unyt.Unit(unit_string, registry=dummy_halos.unit_registry) - assert result.units.same_dimensions_as(unit.units) - - # Check properties were not calculated for filtered halos - if pc_name == "filter_test": - for proj in ["projx", "projy", "projz"]: - for prop in pc_calc.property_list: - outputname = prop[1] - size = prop[2] - full_name = f"ProjectedAperture/30kpc/{proj}/{outputname}" - assert np.all(halo_result[full_name][0].value == np.zeros(size)) - - # Now test the calculation for each property individually, to make sure that - # all properties read all the datasets they require - all_parameters = parameters.get_parameters() - for property in all_parameters["ProjectedApertureProperties"]["properties"]: - print(f"Testing only {property}...") - single_property = dict(all_parameters) - for other_property in all_parameters["ProjectedApertureProperties"][ - "properties" - ]: - single_property["ProjectedApertureProperties"]["properties"][ - other_property - ] = (other_property == property) or other_property.startswith("NumberOf") - single_parameters = ParameterFile(parameter_dictionary=single_property) - - property_calculator = ProjectedApertureProperties( - dummy_halos.get_cell_grid(), - single_parameters, - 30.0, - category_filter, - "basic", - ) - - halo_result_template = dummy_halos.get_halo_result_template(particle_numbers) - - input_data = {} - for ptype in property_calculator.particle_properties: - if ptype in data: - input_data[ptype] = {} - for dset in property_calculator.particle_properties[ptype]: - input_data[ptype][dset] = data[ptype][dset] - input_halo_copy = input_halo.copy() - input_data_copy = input_data.copy() - halo_result = dict(halo_result_template) - property_calculator.calculate( - input_halo, 50 * unyt.kpc, input_data, halo_result - ) - assert input_halo == input_halo_copy - assert input_data == input_data_copy - - for proj in ["projx", "projy", "projz"]: - for prop in property_calculator.property_list: - outputname = prop[1] - if not outputname == property: - continue - size = prop[2] - dtype = prop[3] - unit_string = prop[4] - physical = prop[10] - a_exponent = prop[11] - full_name = f"ProjectedAperture/30kpc/{proj}/{outputname}" - assert full_name in halo_result - result = halo_result[full_name][0] - assert (len(result.shape) == 0 and size == 1) or result.shape[0] == size - assert result.dtype == dtype - unit = unyt.Unit(unit_string, registry=dummy_halos.unit_registry) - if not physical: - unit = ( - unit - * unyt.Unit("a", registry=dummy_halos.unit_registry) - ** a_exponent - ) - assert result.units == unit.units - - dummy_halos.get_cell_grid().snapshot_datasets.print_dataset_log() - - -if __name__ == "__main__": - """ - Standalone mode: simply run the unit test. - - Note that this can also be achieved by running - python3 -m pytest *.py - in the main folder. - """ + # Storing total time for each calculation over the three projections + if self.record_timings: + arr = unyt.unyt_array( + timings.get(name, 0), + dtype=np.float32, + units=unyt.dimensionless, + registry=registry, + ) + halo_result.update( + { + f"{self.group_name}/{outputname}_time": ( + arr, + "Time taken in seconds", + True, + None, + ) + } + ) - print("Calling test_projected_aperture_properties()...") - test_projected_aperture_properties() - print("Test passed.") + return diff --git a/subhalo_properties.py b/SOAP/particle_selection/subhalo_properties.py similarity index 70% rename from subhalo_properties.py rename to SOAP/particle_selection/subhalo_properties.py index 746c2132..a11231f5 100644 --- a/subhalo_properties.py +++ b/SOAP/particle_selection/subhalo_properties.py @@ -3,50 +3,53 @@ """ subhalo_properties.py -Halo properties for VR subhalo groups. - -Subhalos are identified by VR as substructures of 3D FOF groups. They are found -by first running a 3D FOF algorithm and then subdividing the resulting 3D FOF -groups using a 6D phase space FOF algorithm. The 6D FOF subhalo groups come in -two flavours: a version that includes all the particles within the 6D FOF group -(simply called the FOFSubhalo), and a version that only includes the particles -that are also gravitationally bound to the subhalo (the BoundSubhalo). The union -of all the FOFSubhalo groups is the original 3D FOF group (which can be useful -to recover e.g. the FOF mass or CoM of the 3D FOF group). +Calculate subhalo properties using all the particles deemed to be +a member of that subhalo. Note that all the membership information used to determine which particles are -a member of the FOFSubhalo and the BoundSubhalo comes directly from VR, i.e. we -do not perform any FOF algorithm or boundedness calculation in SOAP. +a member of a subhalo comes directly from the halo finder, SOAP does not +perform any FOF algorithm or boundedness calculations. Just like the other HaloProperty implementations, the calculation of the properties is done lazily: only calculations that are actually needed are performed. See aperture_properties.py for a fully documented example. """ +import time +from typing import Dict, List + import numpy as np +from numpy.typing import NDArray import unyt -from halo_properties import HaloProperty, SearchRadiusTooSmallError -from dataset_names import mass_dataset -from half_mass_radius import get_half_mass_radius -from kinematic_properties import ( +from .halo_properties import HaloProperty, SearchRadiusTooSmallError +from SOAP.property_calculation.half_mass_radius import ( + get_half_mass_radius, + get_half_light_radius, +) +from SOAP.property_calculation.kinematic_properties import ( get_angular_momentum, - get_angular_momentum_and_kappa_corot, + get_angular_momentum_and_kappa_corot_mass_weighted, + get_angular_momentum_and_kappa_corot_luminosity_weighted, get_vmax, - get_inertia_tensor, get_velocity_dispersion_matrix, ) -from recently_heated_gas_filter import RecentlyHeatedGasFilter -from stellar_age_calculator import StellarAgeCalculator -from property_table import PropertyTable -from lazy_properties import lazy_property -from category_filter import CategoryFilter -from parameter_file import ParameterFile -from snapshot_datasets import SnapshotDatasets -from swift_cells import SWIFTCellGrid - -from typing import Dict, List -from numpy.typing import NDArray +from SOAP.property_calculation.inertia_tensors import ( + get_inertia_tensor_mass_weighted, + get_inertia_tensor_luminosity_weighted, +) +from SOAP.property_calculation.cylindrical_coordinates import ( + calculate_cylindrical_velocities, +) +from SOAP.particle_filter.recently_heated_gas_filter import RecentlyHeatedGasFilter +from SOAP.property_calculation.stellar_age_calculator import StellarAgeCalculator +from SOAP.property_table import PropertyTable +from SOAP.core.dataset_names import mass_dataset +from SOAP.core.lazy_properties import lazy_property +from SOAP.core.category_filter import CategoryFilter +from SOAP.core.parameter_file import ParameterFile +from SOAP.core.snapshot_datasets import SnapshotDatasets +from SOAP.core.swift_cells import SWIFTCellGrid class SubhaloParticleData: @@ -69,11 +72,12 @@ def __init__( input_halo: Dict, data: Dict, types_present: List[str], - grnr: int, stellar_age_calculator: StellarAgeCalculator, recently_heated_gas_filter: RecentlyHeatedGasFilter, snapshot_datasets: SnapshotDatasets, softening_of_parttype: unyt.unyt_array, + boxsize: unyt.unyt_quantity, + cosmology: dict, ): """ Constructor. @@ -86,9 +90,6 @@ def __init__( - types_present: List List of all particle types (e.g. 'PartType0') that are present in the data dictionary. - - grnr: int - VR index of this particular subhalo. Used to match particles to this - subhalo. - stellar_age_calculator: StellarAgeCalculator Object used to compute stellar ages from the current cosmological scale factor and the birth scale factors of star particles. @@ -98,15 +99,20 @@ def __init__( - snapshot_datasets: SnapshotDatasets Object containing metadata about the datasets in the snapshot, like appropriate aliases and column names. + - boxsize: unyt.unyt_quantity + Boxsize for correcting periodic boundary conditions + - cosmology: dict + Cosmological parameters required for SO calculation """ self.input_halo = input_halo self.data = data self.types_present = types_present - self.grnr = grnr self.stellar_age_calculator = stellar_age_calculator self.recently_heated_gas_filter = recently_heated_gas_filter self.snapshot_datasets = snapshot_datasets self.softening_of_parttype = softening_of_parttype + self.boxsize = boxsize + self.cosmology = cosmology self.compute_basics() def get_dataset(self, name: str) -> unyt.unyt_array: @@ -130,7 +136,7 @@ def compute_basics(self): types = [] softening = [] for ptype in self.types_present: - grnr = self.get_dataset(f"{ptype}/{self.grnr}") + grnr = self.get_dataset(f"{ptype}/GroupNr_bound") in_halo = grnr == self.index mass.append(self.get_dataset(f"{ptype}/{mass_dataset(ptype)}")[in_halo]) pos = ( @@ -228,6 +234,13 @@ def mass_gas(self) -> unyt.unyt_array: """ return self.mass[self.gas_mask_sh] + @lazy_property + def mass_dust(self) -> unyt.unyt_array: + """ + Masses of the dust particles in the subhalo. + """ + return self.gas_total_dust_mass_fractions * self.mass_gas + @lazy_property def mass_dm(self) -> unyt.unyt_array: """ @@ -319,6 +332,28 @@ def Mgas(self) -> unyt.unyt_quantity: """ return self.mass_gas.sum() + @lazy_property + def DustMass(self) -> unyt.unyt_quantity: + """ + Total dust mass of the gas particles in the subhalo. + """ + if self.Ngas == 0: + return None + return self.mass_dust.sum() + + @lazy_property + def gas_total_dust_mass_fractions(self) -> unyt.unyt_array: + """ + Total dust mass fractions in gas particles. + """ + if self.Ngas == 0: + return None + mass_frac = self.get_dataset("PartType0/TotalDustMassFractions")[ + self.gas_mask_all + ] + mass_frac[mass_frac < 10 ** (-10)] = 0 * unyt.dimensionless + return mass_frac + @lazy_property def Mdm(self) -> unyt.unyt_quantity: """ @@ -348,7 +383,7 @@ def star_mask_all(self) -> NDArray[bool]: """ if self.Nstar == 0: return None - return self.get_dataset(f"PartType4/{self.grnr}") == self.index + return self.get_dataset(f"PartType4/GroupNr_bound") == self.index @lazy_property def mass_star_init(self) -> unyt.unyt_array: @@ -436,7 +471,8 @@ def stellar_age_lw(self) -> unyt.unyt_array: if self.Nstar == 0: return None Lr = self.stellar_luminosities[ - :, self.snapshot_datasets.get_column_index("Luminosities", "GAMA_r") + :, + self.snapshot_datasets.get_column_index("PartType4/Luminosities", "GAMA_r"), ] Lrtot = Lr.sum() return ((Lr / Lrtot) * self.stellar_ages).sum() @@ -449,7 +485,7 @@ def bh_mask_all(self) -> NDArray[bool]: """ if self.Nbh == 0: return None - return self.get_dataset(f"PartType5/{self.grnr}") == self.index + return self.get_dataset(f"PartType5/GroupNr_bound") == self.index @lazy_property def Mbh_subgrid(self) -> unyt.unyt_quantity: @@ -764,9 +800,21 @@ def com(self) -> unyt.unyt_array: """ if self.Mtot == 0: return None - return (self.total_mass_fraction[:, None] * self.position).sum( - axis=0 - ) + self.centre + return ( + (self.total_mass_fraction[:, None] * self.position).sum(axis=0) + + self.centre + ) % self.boxsize + + @lazy_property + def com_star(self) -> unyt.unyt_array: + """ + Centre of mass of star particles in the subhalo. + """ + if self.Mstar == 0: + return None + return ( + (self.star_mass_fraction[:, None] * self.pos_star).sum(axis=0) + self.centre + ) % self.boxsize @lazy_property def vcom(self) -> unyt.unyt_array: @@ -777,11 +825,145 @@ def vcom(self) -> unyt.unyt_array: return None return (self.total_mass_fraction[:, None] * self.velocity).sum(axis=0) + @lazy_property + def KineticEnergyTotal(self) -> unyt.unyt_quantity: + """ + Total kinetic energy of all particles. + """ + if self.Mtot == 0: + return None + v_tot = self.velocity - self.vcom[None, :] + v_tot += self.position * self.cosmology["H"] + ekin_tot = self.mass * (v_tot**2).sum(axis=1) + return 0.5 * ekin_tot.sum() + + @lazy_property + def pressure_gas(self) -> unyt.unyt_array: + """ + Pressure of the gas particles in the subhalo. + """ + if self.Ngas == 0: + return None + return self.get_dataset("PartType0/Pressures")[self.gas_mask_all] + + @lazy_property + def density_gas(self) -> unyt.unyt_array: + """ + Density of the gas particles in the subhalo. + """ + if self.Ngas == 0: + return None + return self.get_dataset("PartType0/Densities")[self.gas_mask_all] + + @lazy_property + def ThermalEnergyGas(self) -> unyt.unyt_array: + """ + Total thermal energy of gas particles. + + While this could be computed from PartType0/InternalEnergies, we use + the equation of state + P = (gamma-1) * rho * u + (with gamma=5/3) because some simulations (read: FLAMINGO) do not output + the internal energies. + """ + if self.Ngas == 0: + return None + etherm_gas = 1.5 * self.mass_gas * self.pressure_gas / self.density_gas + return etherm_gas.sum() + + @lazy_property + def potential_energy_gas(self) -> unyt.unyt_array: + """ + Potential energy of the gas particles in the subhalo. + """ + if self.Ngas == 0: + return None + return ( + self.mass_gas + * self.get_dataset("PartType0/SpecificPotentialEnergies")[self.gas_mask_all] + ) + + @lazy_property + def dm_mask_all(self) -> NDArray[bool]: + """ + Mask that can be used to filter out DM particles that belong to this + subhalo in raw particle arrays, like PartType0/Masses. + """ + return self.get_dataset(f"PartType1/GroupNr_bound") == self.index + + @lazy_property + def potential_energy_dm(self) -> unyt.unyt_array: + """ + Potential energy of the DM particles in the subhalo. + """ + if self.Ndm == 0: + return None + return ( + self.mass_dm + * self.get_dataset("PartType1/SpecificPotentialEnergies")[self.dm_mask_all] + ) + + @lazy_property + def potential_energy_star(self) -> unyt.unyt_array: + """ + Potential energy of the star particles in the subhalo. + """ + if self.Nstar == 0: + return None + return ( + self.mass_star + * self.get_dataset("PartType4/SpecificPotentialEnergies")[ + self.star_mask_all + ] + ) + + @lazy_property + def potential_energy_bh(self) -> unyt.unyt_array: + """ + Potential energy of the BH particles in the subhalo. + """ + if self.Nbh == 0: + return None + return ( + self.mass[self.bh_mask_sh] + * self.get_dataset("PartType5/SpecificPotentialEnergies")[self.bh_mask_all] + ) + + @lazy_property + def PotentialEnergyTotal(self) -> unyt.unyt_quantity: + """ + Total potential energy of the subhalo. + """ + if self.Mtot == 0: + return None + # Create unyt array with correct units + epot_tot = unyt.unyt_array( + 0, + dtype=np.float32, + units=unyt.Unit("snap_mass", registry=self.mass.units.registry) + * ( + unyt.Unit("snap_length", registry=self.mass.units.registry) + / unyt.Unit("snap_time", registry=self.mass.units.registry) + ) + ** 2, + ) + # Add contribution from each particle type + if self.Ngas != 0: + epot_tot += self.potential_energy_gas.sum() + if self.Ndm != 0: + epot_tot += self.potential_energy_dm.sum() + if self.Nstar != 0: + epot_tot += self.potential_energy_star.sum() + if self.Nbh != 0: + epot_tot += self.potential_energy_bh.sum() + # Factor of 2 since we should only be summing over each pair of particles + return epot_tot / 2 + @lazy_property def R_vmax_unsoft(self) -> unyt.unyt_quantity: """ Radius at which the maximum circular velocity of the halo is reached. - Particles are not constrained to be at least one softening length away + Particles are not constrained to be at least one softening length away from the centre. This includes contributions from all particle types. @@ -798,7 +980,7 @@ def R_vmax_unsoft(self) -> unyt.unyt_quantity: def Vmax_unsoft(self) -> unyt.unyt_quantity: """ Maximum circular velocity of the halo. - Particles are not constrained to be at least one softening length away + Particles are not constrained to be at least one softening length away from the centre. This includes contributions from all particle types. @@ -875,49 +1057,51 @@ def TotalInertiaTensor(self) -> unyt.unyt_array: """ Inertia tensor of the total mass distribution. Computed iteratively using an ellipsoid with volume equal to that of - a sphere with radius HalfMassRadiusTot. Only considers bound particles. + a sphere with radius 10 x HalfMassRadiusTot. Only considers bound particles. """ if self.Mtot == 0: return None - return get_inertia_tensor(self.mass, self.position, self.HalfMassRadiusTot) + return get_inertia_tensor_mass_weighted( + self.mass, self.position, 10 * self.HalfMassRadiusTot + ) @lazy_property def TotalInertiaTensorReduced(self) -> unyt.unyt_array: """ Reduced inertia tensor of the total mass distribution. Computed iteratively using an ellipsoid with volume equal to that of - a sphere with radius HalfMassRadiusTot. Only considers bound particles. + a sphere with radius 10 x HalfMassRadiusTot. Only considers bound particles. """ if self.Mtot == 0: return None - return get_inertia_tensor( - self.mass, self.position, self.HalfMassRadiusTot, reduced=True + return get_inertia_tensor_mass_weighted( + self.mass, self.position, 10 * self.HalfMassRadiusTot, reduced=True ) @lazy_property def TotalInertiaTensorNoniterative(self) -> unyt.unyt_array: """ Inertia tensor of the total mass distribution. - Computed using all bound particles within HalfMassRadiusTot. + Computed using all bound particles within 10 * HalfMassRadiusTot. """ if self.Mtot == 0: return None - return get_inertia_tensor( - self.mass, self.position, self.HalfMassRadiusTot, max_iterations=1 + return get_inertia_tensor_mass_weighted( + self.mass, self.position, 10 * self.HalfMassRadiusTot, max_iterations=1 ) @lazy_property def TotalInertiaTensorReducedNoniterative(self) -> unyt.unyt_array: """ Reduced inertia tensor of the total mass distribution. - Computed using all bound particles within HalfMassRadiusTot. + Computed using all bound particles within 10 * HalfMassRadiusTot. """ if self.Mtot == 0: return None - return get_inertia_tensor( + return get_inertia_tensor_mass_weighted( self.mass, self.position, - self.HalfMassRadiusTot, + 10 * self.HalfMassRadiusTot, reduced=True, max_iterations=1, ) @@ -951,11 +1135,11 @@ def compute_Lgas_props(self): self.internal_Lgas, self.internal_kappa_gas, self.internal_Mcountrot_gas, - ) = get_angular_momentum_and_kappa_corot( + ) = get_angular_momentum_and_kappa_corot_mass_weighted( self.mass_gas, self.pos_gas, self.vel_gas, - ref_velocity=self.vcom_gas, + reference_velocity=self.vcom_gas, do_counterrot_mass=True, ) @@ -1004,49 +1188,51 @@ def GasInertiaTensor(self) -> unyt.unyt_array: """ Inertia tensor of the gas mass distribution. Computed iteratively using an ellipsoid with volume equal to that of - a sphere with radius HalfMassRadiusGas. Only considers bound particles. + a sphere with radius 10 x HalfMassRadiusGas. Only considers bound particles. """ if self.Mgas == 0: return None - return get_inertia_tensor(self.mass_gas, self.pos_gas, self.HalfMassRadiusGas) + return get_inertia_tensor_mass_weighted( + self.mass_gas, self.pos_gas, 10 * self.HalfMassRadiusGas + ) @lazy_property def GasInertiaTensorReduced(self) -> unyt.unyt_array: """ Reduced inertia tensor of the gas mass distribution. Computed iteratively using an ellipsoid with volume equal to that of - a sphere with radius HalfMassRadiusGas. Only considers bound particles. + a sphere with radius 10 x HalfMassRadiusGas. Only considers bound particles. """ if self.Mgas == 0: return None - return get_inertia_tensor( - self.mass_gas, self.pos_gas, self.HalfMassRadiusGas, reduced=True + return get_inertia_tensor_mass_weighted( + self.mass_gas, self.pos_gas, 10 * self.HalfMassRadiusGas, reduced=True ) @lazy_property def GasInertiaTensorNoniterative(self) -> unyt.unyt_array: """ Inertia tensor of the gas mass distribution. - Computed using all bound gas particles within HalfMassRadiusGas. + Computed using all bound gas particles within 10 x HalfMassRadiusGas. """ if self.Mgas == 0: return None - return get_inertia_tensor( - self.mass_gas, self.pos_gas, self.HalfMassRadiusGas, max_iterations=1 + return get_inertia_tensor_mass_weighted( + self.mass_gas, self.pos_gas, 10 * self.HalfMassRadiusGas, max_iterations=1 ) @lazy_property def GasInertiaTensorReducedNoniterative(self) -> unyt.unyt_array: """ Reduced inertia tensor of the gas mass distribution. - Computed using all bound gas particles within HalfMassRadiusGas. + Computed using all bound gas particles within 10 x HalfMassRadiusGas. """ if self.Mgas == 0: return None - return get_inertia_tensor( + return get_inertia_tensor_mass_weighted( self.mass_gas, self.pos_gas, - self.HalfMassRadiusGas, + 10 * self.HalfMassRadiusGas, reduced=True, max_iterations=1, ) @@ -1098,49 +1284,51 @@ def DarkMatterInertiaTensor(self) -> unyt.unyt_array: """ Inertia tensor of the dark matter mass distribution. Computed iteratively using an ellipsoid with volume equal to that of - a sphere with radius HalfMassRadiusDM. Only considers bound particles. + a sphere with radius 10 x HalfMassRadiusDM. Only considers bound particles. """ if self.Mdm == 0: return None - return get_inertia_tensor(self.mass_dm, self.pos_dm, self.HalfMassRadiusDM) + return get_inertia_tensor_mass_weighted( + self.mass_dm, self.pos_dm, 10 * self.HalfMassRadiusDM + ) @lazy_property def DarkMatterInertiaTensorReduced(self) -> unyt.unyt_array: """ Reduced inertia tensor of the dark matter mass distribution. Computed iteratively using an ellipsoid with volume equal to that of - a sphere with radius HalfMassRadiusDM. Only considers bound particles. + a sphere with radius 10 x HalfMassRadiusDM. Only considers bound particles. """ if self.Mdm == 0: return None - return get_inertia_tensor( - self.mass_dm, self.pos_dm, self.HalfMassRadiusDM, reduced=True + return get_inertia_tensor_mass_weighted( + self.mass_dm, self.pos_dm, 10 * self.HalfMassRadiusDM, reduced=True ) @lazy_property def DarkMatterInertiaTensorNoniterative(self) -> unyt.unyt_array: """ Inertia tensor of the dark matter mass distribution. - Computed using all bound DM particles within HalfMassRadiusDM. + Computed using all bound DM particles within 10 x HalfMassRadiusDM. """ if self.Mdm == 0: return None - return get_inertia_tensor( - self.mass_dm, self.pos_dm, self.HalfMassRadiusDM, max_iterations=1 + return get_inertia_tensor_mass_weighted( + self.mass_dm, self.pos_dm, 10 * self.HalfMassRadiusDM, max_iterations=1 ) @lazy_property def DarkMatterInertiaTensorReducedNoniterative(self) -> unyt.unyt_array: """ Reduced inertia tensor of the dark matter mass distribution. - Computed using all bound DM particles within HalfMassRadiusDM. + Computed using all bound DM particles within 10 x HalfMassRadiusDM. """ if self.Mdm == 0: return None - return get_inertia_tensor( + return get_inertia_tensor_mass_weighted( self.mass_dm, self.pos_dm, - self.HalfMassRadiusDM, + 10 * self.HalfMassRadiusDM, reduced=True, max_iterations=1, ) @@ -1198,6 +1386,82 @@ def star_mass_fraction(self) -> unyt.unyt_array: return None return self.mass_star / self.Mstar + @lazy_property + def star_cylindrical_velocities(self) -> unyt.unyt_array: + """ + Calculate the velocities of the star particles in cyclindrical + coordinates, where the axes are centred on the stellar CoM, + and the z axis is aligned with the stellar angular momentum. + """ + + # We need at least 2 particles to have an angular momentum vector + if self.Nstar < 2: + return None + + # This can happen if we have particles on top of each other + # or with the same velocity + if np.sum(self.Lstar) == 0: + return None + + # Calculate the position of the stars relative to their CoM + pos = self.pos_star - (self.star_mass_fraction[:, None] * self.pos_star).sum( + axis=0 + ) + + # Calculate relative velocity of stars + vrel = self.vel_star - self.vcom[None, :] + + # Get velocities in cylindrical coordinates + return calculate_cylindrical_velocities( + pos, + vrel, + self.Lstar, + ) + + @lazy_property + def StellarRotationalVelocity(self) -> unyt.unyt_array: + if (self.Nstar < 2) or (np.sum(self.Lstar) == 0): + return None + v_cylindrical = self.star_cylindrical_velocities + v_phi = v_cylindrical[:, 1] + return (self.star_mass_fraction * v_phi).sum() + + @lazy_property + def stellar_cylindrical_squared_velocity_dispersion_vector(self) -> unyt.unyt_array: + if (self.Nstar < 2) or (np.sum(self.Lstar) == 0): + return None + v_cylindrical = self.star_cylindrical_velocities + + # This implementation of standard deviation is more numerically stable than using - ^2 + mean_velocity = (self.star_mass_fraction[:, None] * v_cylindrical).sum(axis=0) + squared_velocity_dispersion = ( + self.star_mass_fraction[:, None] * (v_cylindrical - mean_velocity) ** 2 + ).sum(axis=0) + + return squared_velocity_dispersion + + @lazy_property + def StellarCylindricalVelocityDispersion(self) -> unyt.unyt_array: + if self.stellar_cylindrical_squared_velocity_dispersion_vector is None: + return None + return np.sqrt( + self.stellar_cylindrical_squared_velocity_dispersion_vector.sum() / 3 + ) + + @lazy_property + def StellarCylindricalVelocityDispersionVertical(self) -> unyt.unyt_array: + if self.stellar_cylindrical_squared_velocity_dispersion_vector is None: + return None + return np.sqrt(self.stellar_cylindrical_squared_velocity_dispersion_vector[2]) + + @lazy_property + def StellarCylindricalVelocityDispersionDiscPlane(self) -> unyt.unyt_array: + if self.stellar_cylindrical_squared_velocity_dispersion_vector is None: + return None + return np.sqrt( + self.stellar_cylindrical_squared_velocity_dispersion_vector[:2].sum() + ) + @lazy_property def vcom_star(self) -> unyt.unyt_array: """ @@ -1216,14 +1480,40 @@ def compute_Lstar_props(self): self.internal_Lstar, self.internal_kappa_star, self.internal_Mcountrot_star, - ) = get_angular_momentum_and_kappa_corot( + ) = get_angular_momentum_and_kappa_corot_mass_weighted( self.mass_star, self.pos_star, self.vel_star, - ref_velocity=self.vcom_star, + reference_velocity=self.vcom_star, do_counterrot_mass=True, ) + def compute_Lstar_luminosity_weighted_props(self): + """ + Compute the angular momentum and related properties for star particles, + weighted by their luminosity in a given GAMA band. + + We need this method because Lstar, kappa_star and Mcountrot_star are + computed together. + """ + + # Contrary to compute_Lstar_props, each of the output arrays contains a + # value for each GAMA filter, hence they will have shape (9,) + ( + self.internal_Lstar_luminosity_weighted, + self.internal_kappa_star_luminosity_weighted, + self.internal_Mcountrot_star_luminosity_weighted, + self.internal_Lcountrot_star_luminosity_weighted, + ) = get_angular_momentum_and_kappa_corot_luminosity_weighted( + self.mass_star, + self.pos_star, + self.vel_star, + self.stellar_luminosities, + reference_velocity=self.vcom_star, + do_counterrot_mass=True, + do_counterrot_luminosity=True, + ) + @lazy_property def Lstar(self) -> unyt.unyt_array: """ @@ -1237,6 +1527,23 @@ def Lstar(self) -> unyt.unyt_array: self.compute_Lstar_props() return self.internal_Lstar + @lazy_property + def Lstar_luminosity_weighted(self) -> unyt.unyt_array: + """ + Luminosity-weighted angular momentum of star particles for different + luminosity bands. NOTE: we reshape the 2D array of shape + (number_luminosity_bans, 3) to a 1D array of shape (number_luminosity_bans * 3,) + + This is computed together with Lstar_luminosity_weighted, kappa_star_luminosity_weighted, + Mcountrot_star_luminosity_weighted and Lcountrot_star_luminosity_weighted + by compute_Lstar_luminosity_weighted_props(). + """ + if self.Nstar == 0: + return None + if not hasattr(self, "internal_Lstar_luminosity_weighted"): + self.compute_Lstar_luminosity_weighted_props() + return self.internal_Lstar_luminosity_weighted.flatten() + @lazy_property def kappa_corot_star(self) -> unyt.unyt_quantity: """ @@ -1251,6 +1558,23 @@ def kappa_corot_star(self) -> unyt.unyt_quantity: self.compute_Lstar_props() return self.internal_kappa_star + @lazy_property + def kappa_corot_star_luminosity_weighted(self) -> unyt.unyt_array: + """ + Kinetic energy fraction of co-rotating star particles, measured for + different luminosity-weighted angular momentum vectors. + + This is computed together with Lstar_luminosity_weighted, kappa_star_luminosity_weighted, + Mcountrot_star_luminosity_weighted and Lcountrot_star_luminosity_weighted + by compute_Lstar_luminosity_weighted_props(). + """ + if self.Nstar == 0: + return None + if not hasattr(self, "internal_kappa_star_luminosity_weighted"): + self.compute_Lstar_luminosity_weighted_props() + + return self.internal_kappa_star_luminosity_weighted + @lazy_property def DtoTstar(self) -> unyt.unyt_quantity: """ @@ -1264,17 +1588,58 @@ def DtoTstar(self) -> unyt.unyt_quantity: self.compute_Lstar_props() return 1.0 - 2.0 * self.internal_Mcountrot_star / self.Mstar + @lazy_property + def DtoTstar_luminosity_weighted_luminosity_ratio(self) -> unyt.unyt_array: + """ + Disk to total luminosity ratio for all provided stellar luminosity bands. + Each band uses the luminosity-weighted angular momentum as defined in that + band. + + This is computed together with Lstar_luminosity_weighted, kappa_star_luminosity_weighted, + Mcountrot_star_luminosity_weighted and Lcountrot_star_luminosity_weighted + by compute_Lstar_luminosity_weighted_props(). + """ + if self.Nstar == 0: + return None + if not hasattr(self, "internal_Lcountrot_star_luminosity_weighted"): + self.compute_Lstar_luminosity_weighted_props() + + return ( + 1.0 + - 2.0 + * self.internal_Lcountrot_star_luminosity_weighted + / self.StellarLuminosity + ) + + @lazy_property + def DtoTstar_luminosity_weighted_mass_ratio(self) -> unyt.unyt_array: + """ + Disk to total mass ratio for all provided stellar luminosity bands. + Each band uses the luminosity-weighted angular momentum as defined in that + band. + + This is computed together with Lstar_luminosity_weighted, kappa_star_luminosity_weighted, + Mcountrot_star_luminosity_weighted and Lcountrot_star_luminosity_weighted + by compute_Lstar_luminosity_weighted_props(). + """ + if self.Nstar == 0: + return None + if not hasattr(self, "internal_Mcountrot_star_luminosity_weighted"): + self.compute_Lstar_luminosity_weighted_props() + + return 1.0 - 2.0 * self.internal_Mcountrot_star_luminosity_weighted / self.Mstar + @lazy_property def StellarInertiaTensor(self) -> unyt.unyt_array: """ Inertia tensor of the stellar mass distribution. Computed iteratively using an ellipsoid with volume equal to that of - a sphere with radius HalfMassRadiusStar. Only considers bound particles. + a sphere with radius 10 x HalfMassRadiusStar. Only considers bound particles. """ if self.Mstar == 0: return None - return get_inertia_tensor( - self.mass_star, self.pos_star, self.HalfMassRadiusStar + return get_inertia_tensor_mass_weighted( + self.mass_star, self.pos_star, 10 * self.HalfMassRadiusStar ) @lazy_property @@ -1282,38 +1647,103 @@ def StellarInertiaTensorReduced(self) -> unyt.unyt_array: """ Reduced inertia tensor of the stellar mass distribution. Computed iteratively using an ellipsoid with volume equal to that of - a sphere with radius HalfMassRadiusStar. Only considers bound particles. + a sphere with radius 10 x HalfMassRadiusStar. Only considers bound particles. """ if self.Mstar == 0: return None - return get_inertia_tensor( - self.mass_star, self.pos_star, self.HalfMassRadiusStar, reduced=True + return get_inertia_tensor_mass_weighted( + self.mass_star, self.pos_star, 10 * self.HalfMassRadiusStar, reduced=True ) @lazy_property def StellarInertiaTensorNoniterative(self) -> unyt.unyt_array: """ Inertia tensor of the stellar mass distribution. - Computed using all bound star particles within HalfMassRadiusStar. + Computed using all bound star particles within 10 x HalfMassRadiusStar. """ if self.Mstar == 0: return None - return get_inertia_tensor( - self.mass_star, self.pos_star, self.HalfMassRadiusStar, max_iterations=1 + return get_inertia_tensor_mass_weighted( + self.mass_star, + self.pos_star, + 10 * self.HalfMassRadiusStar, + max_iterations=1, ) @lazy_property def StellarInertiaTensorReducedNoniterative(self) -> unyt.unyt_array: """ Reduced inertia tensor of the stellar mass distribution. - Computed using all bound star particles within HalfMassRadiusStar. + Computed using all bound star particles within 10 x HalfMassRadiusStar. """ if self.Mstar == 0: return None - return get_inertia_tensor( + return get_inertia_tensor_mass_weighted( self.mass_star, self.pos_star, - self.HalfMassRadiusStar, + 10 * self.HalfMassRadiusStar, + reduced=True, + max_iterations=1, + ) + + @lazy_property + def StellarInertiaTensorLuminosityWeighted(self) -> unyt.unyt_array: + """ + Inertia tensor of the stellar luminosity distribution for each GAMA band. + Computed iteratively using an ellipsoid with volume equal to that of + a sphere with radius 10 x HalfMassRadiusStar. Only considers bound particles. + """ + if self.Mstar == 0: + return None + return get_inertia_tensor_luminosity_weighted( + self.stellar_luminosities, self.pos_star, 10 * self.HalfMassRadiusStar + ) + + @lazy_property + def StellarInertiaTensorReducedLuminosityWeighted(self) -> unyt.unyt_array: + """ + Reduced inertia tensor of the stellar luminosity distribution for each GAMA band. + Computed iteratively using an ellipsoid with volume equal to that of + a sphere with radius 10 x HalfMassRadiusStar. Only considers bound particles. + """ + if self.Mstar == 0: + return None + return get_inertia_tensor_luminosity_weighted( + self.stellar_luminosities, + self.pos_star, + 10 * self.HalfMassRadiusStar, + reduced=True, + ) + + @lazy_property + def StellarInertiaTensorNoniterativeLuminosityWeighted(self) -> unyt.unyt_array: + """ + Inertia tensor of the stellar luminosity distribution for each GAMA band. + Computed using all bound star particles within 10 x HalfMassRadiusStar. + """ + if self.Mstar == 0: + return None + return get_inertia_tensor_luminosity_weighted( + self.stellar_luminosities, + self.pos_star, + 10 * self.HalfMassRadiusStar, + max_iterations=1, + ) + + @lazy_property + def StellarInertiaTensorReducedNoniterativeLuminosityWeighted( + self, + ) -> unyt.unyt_array: + """ + Reduced inertia tensor of the stellar luminosity distribution for each GAMA band. + Computed using all bound star particles within 10 x HalfMassRadiusStar. + """ + if self.Mstar == 0: + return None + return get_inertia_tensor_luminosity_weighted( + self.stellar_luminosities, + self.pos_star, + 10 * self.HalfMassRadiusStar, reduced=True, max_iterations=1, ) @@ -1364,11 +1794,11 @@ def compute_Lbar_props(self): ( self.internal_Lbar, self.internal_kappa_bar, - ) = get_angular_momentum_and_kappa_corot( + ) = get_angular_momentum_and_kappa_corot_mass_weighted( self.mass_baryons, self.pos_baryons, self.vel_baryons, - ref_velocity=self.vcom_bar, + reference_velocity=self.vcom_bar, ) @lazy_property @@ -1404,7 +1834,7 @@ def gas_mask_all(self) -> NDArray[bool]: Mask that can be used to filter out gas particles that belong to this subhalo in raw particle arrays, like PartType0/Masses. """ - return self.get_dataset(f"PartType0/{self.grnr}") == self.index + return self.get_dataset(f"PartType0/GroupNr_bound") == self.index @lazy_property def gas_SFR(self) -> unyt.unyt_array: @@ -1741,6 +2171,17 @@ def HalfMassRadiusTot(self) -> unyt.unyt_quantity: """ return get_half_mass_radius(self.radius, self.mass, self.Mtot) + @lazy_property + def HalfMassRadiusDust(self) -> unyt.unyt_quantity: + """ + Half-mass radius of the dust particle distribution in the subhalo. + """ + if self.Ngas == 0: + return None + return get_half_mass_radius( + self.radius[self.gas_mask_sh], self.mass_dust, self.DustMass + ) + @lazy_property def HalfMassRadiusGas(self) -> unyt.unyt_quantity: """ @@ -1768,6 +2209,20 @@ def HalfMassRadiusStar(self) -> unyt.unyt_quantity: self.radius[self.star_mask_sh], self.mass_star, self.Mstar ) + @lazy_property + def HalfLightRadiusStar(self) -> unyt.unyt_array: + """ + Half-light radius of the star particle distribution in the subhalo, for + the 9 GAMA bands. + """ + if self.Nstar == 0: + return None + return get_half_light_radius( + self.radius[self.star_mask_sh], + self.stellar_luminosities, + self.StellarLuminosity, + ) + @lazy_property def HalfMassRadiusBaryon(self) -> unyt.unyt_quantity: """ @@ -1804,8 +2259,9 @@ class SubhaloProperties(HaloProperty): Each property should have a corresponding method/property/lazy_property in the SubhaloParticleData class above. """ - property_list = [ - (prop, *PropertyTable.full_property_list[prop]) + base_halo_type = "SubhaloProperties" + property_list = { + prop: PropertyTable.full_property_list[prop] for prop in [ "Mtot", "Mgas", @@ -1843,10 +2299,18 @@ class SubhaloProperties(HaloProperty): "MostMassiveBlackHoleTotalAccretedMass", "MostMassiveBlackHoleFormationScalefactor", "com", + "com_star", "vcom", + "KineticEnergyTotal", + "ThermalEnergyGas", + "PotentialEnergyTotal", "Lgas", "Ldm", "Lstar", + "StellarCylindricalVelocityDispersion", + "StellarCylindricalVelocityDispersionVertical", + "StellarCylindricalVelocityDispersionDiscPlane", + "StellarRotationalVelocity", "kappa_corot_gas", "kappa_corot_star", "Lbaryons", @@ -1866,32 +2330,43 @@ class SubhaloProperties(HaloProperty): "DM_Vmax_soft", "DM_R_vmax_soft", "spin_parameter", + "DustMass", "HalfMassRadiusTot", + "HalfMassRadiusDust", "HalfMassRadiusGas", "HalfMassRadiusDM", "HalfMassRadiusStar", + "HalfLightRadiusStar", "HalfMassRadiusBaryon", "GasInertiaTensor", "DarkMatterInertiaTensor", "StellarInertiaTensor", + "StellarInertiaTensorLuminosityWeighted", "TotalInertiaTensor", "GasInertiaTensorReduced", "DarkMatterInertiaTensorReduced", "StellarInertiaTensorReduced", + "StellarInertiaTensorReducedLuminosityWeighted", "TotalInertiaTensorReduced", "GasInertiaTensorNoniterative", "DarkMatterInertiaTensorNoniterative", "StellarInertiaTensorNoniterative", + "StellarInertiaTensorNoniterativeLuminosityWeighted", "TotalInertiaTensorNoniterative", "GasInertiaTensorReducedNoniterative", "DarkMatterInertiaTensorReducedNoniterative", "StellarInertiaTensorReducedNoniterative", + "StellarInertiaTensorReducedNoniterativeLuminosityWeighted", "TotalInertiaTensorReducedNoniterative", "veldisp_matrix_gas", "veldisp_matrix_dm", "veldisp_matrix_star", "DtoTgas", "DtoTstar", + "DtoTstar_luminosity_weighted_luminosity_ratio", + "DtoTstar_luminosity_weighted_mass_ratio", + "kappa_corot_star_luminosity_weighted", + "Lstar_luminosity_weighted", "stellar_age_mw", "stellar_age_lw", "Mgas_SF", @@ -1908,7 +2383,7 @@ class SubhaloProperties(HaloProperty): "LastSupernovaEventMaximumGasDensity", "EncloseRadius", ] - ] + } def __init__( self, @@ -1917,7 +2392,6 @@ def __init__( recently_heated_gas_filter: RecentlyHeatedGasFilter, stellar_age_calculator: StellarAgeCalculator, category_filter: CategoryFilter, - bound_only: bool = True, ): """ Construct a SubhaloProperties object. @@ -1939,40 +2413,33 @@ def __init__( Filter used to determine which properties can be calculated for this halo. This depends on the number of particles in the subhalo and the category of each property. - - bound_only: bool - Should properties include all particles in the 6DFOF group, or only - gravitationally bound particles? """ super().__init__(cellgrid) - self.property_mask = parameters.get_property_mask( - "SubhaloProperties", [prop[1] for prop in self.property_list] + self.property_filters = parameters.get_property_filters( + "SubhaloProperties", [prop.name for prop in self.property_list.values()] ) - self.bound_only = bound_only self.filter = recently_heated_gas_filter self.stellar_ages = stellar_age_calculator self.category_filter = category_filter self.snapshot_datasets = cellgrid.snapshot_datasets + self.record_timings = parameters.record_property_timings + self.boxsize = cellgrid.boxsize - # This specifies how large a sphere is read in: - self.mean_density_multiple = None - self.critical_density_multiple = None + self.cosmology = {} + self.cosmology["H"] = cellgrid.cosmology[ + "H [internal units]" + ] / cellgrid.get_unit("code_time") # Minimum physical radius to read in (pMpc) self.physical_radius_mpc = 0.0 # Give this calculation a name so we can select it on the command line # Save mask metadata and name of group in the final output file - if bound_only: - self.grnr = "GroupNr_bound" - self.name = "bound_subhalo_properties" - self.group_name = "BoundSubhalo" - else: - self.grnr = "GroupNr_all" - self.name = "fof_subhalo_properties" - self.group_name = "FOFSubhalo" + self.name = "bound_subhalo" + self.group_name = "BoundSubhalo" self.mask_metadata = {"Masked": False} self.halo_filter = "basic" @@ -1985,20 +2452,28 @@ def __init__( # read if that particular property is actually requested # Some basic properties are always required; these are added below self.particle_properties = { - "PartType0": ["Coordinates", "Masses", "Velocities", self.grnr], - "PartType1": ["Coordinates", "Masses", "Velocities", self.grnr], - "PartType4": ["Coordinates", "Masses", "Velocities", self.grnr], - "PartType5": ["Coordinates", "DynamicalMasses", "Velocities", self.grnr], + "PartType0": ["Coordinates", "Masses", "Velocities", "GroupNr_bound"], + "PartType1": ["Coordinates", "Masses", "Velocities", "GroupNr_bound"], + "PartType4": ["Coordinates", "Masses", "Velocities", "GroupNr_bound"], + "PartType5": [ + "Coordinates", + "DynamicalMasses", + "Velocities", + "GroupNr_bound", + ], } - for prop in self.property_list: - outputname = prop[1] - if not self.property_mask[outputname]: + # add additional particle properties based on the selected halo + # properties in the parameter file + for name, prop in self.property_list.items(): + outputname = prop.name + # Skip if this property is disabled in the parameter file + if not self.property_filters[outputname]: continue - is_dmo = prop[8] - if self.category_filter.dmo and not is_dmo: + # Skip non-DMO properties when in DMO run mode + if self.category_filter.dmo and not prop.dmo_property: continue - partprops = prop[9] + partprops = prop.particle_properties for partprop in partprops: pgroup, dset = parameters.get_particle_property(partprop) if not pgroup in self.particle_properties: @@ -2027,51 +2502,68 @@ def calculate(self, input_halo, search_radius, data, halo_result): input_halo, data, types_present, - self.grnr, self.stellar_ages, self.filter, self.snapshot_datasets, self.softening_of_parttype, + self.boxsize, + self.cosmology, ) - if self.bound_only: - # this is the halo type that we use for the filter particle numbers, - # so we have to pass the numbers for the category filters manually - do_calculation = self.category_filter.get_do_calculation( - halo_result, - { - "BoundSubhalo/NumberOfDarkMatterParticles": part_props.Ndm, - "BoundSubhalo/NumberOfGasParticles": part_props.Ngas, - "BoundSubhalo/NumberOfStarParticles": part_props.Nstar, - "BoundSubhalo/NumberOfBlackHoleParticles": part_props.Nbh, - }, + # this is the halo type that we use for the filter particle numbers, + # so we have to pass the numbers for the category filters manually + do_calculation = self.category_filter.get_do_calculation( + halo_result, + { + "BoundSubhalo/NumberOfDarkMatterParticles": part_props.Ndm, + "BoundSubhalo/NumberOfGasParticles": part_props.Ngas, + "BoundSubhalo/NumberOfStarParticles": part_props.Nstar, + "BoundSubhalo/NumberOfBlackHoleParticles": part_props.Nbh, + }, + ) + + # Check that we found the expected number of halo member particles: + # If not, we need to try again with a larger search radius. + # For HBT this should not happen since we use the radius of the most distant + # bound particle. + Ntot = part_props.Ngas + part_props.Ndm + part_props.Nstar + part_props.Nbh + Nexpected = input_halo["nr_bound_part"] + if Ntot < Nexpected: + # Try again with a larger search radius + # print( + # f"Ntot = {Ntot}, Nexpected = {Nexpected}, search_radius = {search_radius}" + # ) + raise SearchRadiusTooSmallError( + "Search radius does not contain expected number of particles!" + ) + elif Ntot > Nexpected: + # This would indicate a bug somewhere + raise RuntimeError( + f'Found more particles than expected for halo {input_halo["index"]}' ) - else: - do_calculation = self.category_filter.get_do_calculation(halo_result) subhalo = {} + timings = {} # declare all the variables we will compute # we set them to 0 in case a particular variable cannot be computed # all variables are defined with physical units and an appropriate dtype # we need to use the custom unit registry so that everything can be converted # back to snapshot units in the end registry = part_props.mass.units.registry - for prop in self.property_list: - outputname = prop[1] - # skip properties that are masked - if not self.property_mask[outputname]: + for name, prop in self.property_list.items(): + outputname = prop.name + # Skip if this property is disabled in the parameter file + filter_name = self.property_filters[outputname] + if not filter_name: continue - # skip non-DMO properties in DMO run mode - is_dmo = prop[8] - if do_calculation["DMO"] and not is_dmo: + # Skip non-DMO properties when in DMO run mode + if self.category_filter.dmo and not prop.dmo_property: continue - name = prop[0] - shape = prop[2] - dtype = prop[3] - unit = unyt.Unit(prop[4], registry=registry) - category = prop[6] - physical = prop[10] - a_exponent = prop[11] + shape = prop.shape + dtype = prop.dtype + unit = unyt.Unit(prop.unit, registry=registry) + physical = prop.output_physical + a_exponent = prop.a_scale_exponent if shape > 1: val = [0] * shape else: @@ -2081,7 +2573,8 @@ def calculate(self, input_halo, search_radius, data, halo_result): subhalo[name] = unyt.unyt_array( val, dtype=dtype, units=unit, registry=registry ) - if do_calculation[category]: + if do_calculation[filter_name]: + t0_calc = time.time() val = getattr(part_props, name) if val is not None: assert ( @@ -2099,218 +2592,46 @@ def calculate(self, input_halo, search_radius, data, halo_result): registry=registry, ) else: - err = f'Overflow for halo {input_halo["index"]} when' + err = f'Overflow for halo {input_halo["index"]} when ' err += f"calculating {name} in subhalo_properties" assert np.max(np.abs(val.to(unit).value)) < float("inf"), err subhalo[name] += val - - # Check that we found the expected number of halo member particles: - # If not, we need to try again with a larger search radius. - Ntot = part_props.Ngas + part_props.Ndm + part_props.Nstar + part_props.Nbh - if self.bound_only: - Nexpected = input_halo["nr_bound_part"] - else: - Nexpected = input_halo["nr_bound_part"] + input_halo["nr_unbound_part"] - if Ntot < Nexpected: - # Try again with a larger search radius - # print(f"Ntot = {Ntot}, Nexpected = {Nexpected}, search_radius = {search_radius}") - raise SearchRadiusTooSmallError( - "Search radius does not contain expected number of particles!" - ) - elif Ntot > Nexpected: - # This would indicate a bug somewhere - raise RuntimeError(f'Found more particles than expected for halo {input_halo["index"]}') + timings[name] = time.time() - t0_calc # Add these properties to the output - for prop in self.property_list: - outputname = prop[1] - # skip properties that are masked - if not self.property_mask[outputname]: + for name, prop in self.property_list.items(): + outputname = prop.name + # Skip if this property is disabled in the parameter file + if not self.property_filters[outputname]: continue - is_dmo = prop[8] - if do_calculation["DMO"] and not is_dmo: + # Skip non-DMO properties when in DMO run mode + if self.category_filter.dmo and not prop.dmo_property: continue - name = prop[0] - description = prop[5] - physical = prop[10] - a_exponent = prop[11] + # Add data array and metadata to halo_result halo_result.update( { f"{self.group_name}/{outputname}": ( subhalo[name], - description, - physical, - a_exponent, + prop.description, + prop.output_physical, + prop.a_scale_exponent, ) } ) - - -def test_subhalo_properties(): - """ - Unit test for the subhalo property calculations. - - We generate 100 random "dummy" halos and feed them to - SubhaloProperties::calculate(). We check that the returned values - are present, and have the right units, size and dtype - """ - - from dummy_halo_generator import DummyHaloGenerator - - # initialise the DummyHaloGenerator with a random seed - dummy_halos = DummyHaloGenerator(16902) - cat_filter = CategoryFilter( - dummy_halos.get_filters( - {"general": 100, "gas": 100, "dm": 100, "star": 100, "baryon": 100} - ) - ) - parameters = ParameterFile( - parameter_dictionary={ - "aliases": { - "PartType0/ElementMassFractions": "PartType0/SmoothedElementMassFractions", - "PartType4/ElementMassFractions": "PartType4/SmoothedElementMassFractions", - } - } - ) - dummy_halos.get_cell_grid().snapshot_datasets.setup_aliases( - parameters.get_aliases() - ) - parameters.get_halo_type_variations( - "SubhaloProperties", - {"FOF": {"bound_only": False}, "Bound": {"bound_only": True}}, - ) - - recently_heated_gas_filter = dummy_halos.get_recently_heated_gas_filter() - stellar_age_calculator = StellarAgeCalculator(dummy_halos.get_cell_grid()) - - property_calculator_bound = SubhaloProperties( - dummy_halos.get_cell_grid(), - parameters, - recently_heated_gas_filter, - stellar_age_calculator, - cat_filter, - ) - property_calculator_both = SubhaloProperties( - dummy_halos.get_cell_grid(), - parameters, - recently_heated_gas_filter, - stellar_age_calculator, - cat_filter, - False, - ) - # generate 100 random halos - for i in range(100): - input_halo, data, _, _, _, _ = dummy_halos.get_random_halo( - [1, 10, 100, 1000, 10000] - ) - - halo_result = {} - for subhalo_name, prop_calc in [ - ("BoundSubhalo", property_calculator_bound), - # ("FOFSubhaloProperties", property_calculator_both), - ]: - input_data = {} - for ptype in prop_calc.particle_properties: - if ptype in data: - input_data[ptype] = {} - for dset in prop_calc.particle_properties[ptype]: - input_data[ptype][dset] = data[ptype][dset] - input_halo_copy = input_halo.copy() - input_data_copy = input_data.copy() - prop_calc.calculate(input_halo, 0.0 * unyt.kpc, input_data, halo_result) - assert input_halo == input_halo_copy - assert input_data == input_data_copy - - # check that the calculation returns the correct values - for prop in prop_calc.property_list: - outputname = prop[1] - size = prop[2] - dtype = prop[3] - unit_string = prop[4] - physical = prop[10] - a_exponent = prop[11] - full_name = f"{subhalo_name}/{outputname}" - assert full_name in halo_result - result = halo_result[full_name][0] - assert (len(result.shape) == 0 and size == 1) or result.shape[0] == size - assert result.dtype == dtype - unit = unyt.Unit(unit_string, registry=dummy_halos.unit_registry) - if not physical: - unit = ( - unit - * unyt.Unit("a", registry=dummy_halos.unit_registry) - ** a_exponent - ) - assert result.units == unit.units - - # Now test the calculation for each property individually, to make sure that - # all properties read all the datasets they require - all_parameters = parameters.get_parameters() - for property in all_parameters["SubhaloProperties"]["properties"]: - print(f"Testing only {property}...") - single_property = dict(all_parameters) - for other_property in all_parameters["SubhaloProperties"]["properties"]: - single_property["SubhaloProperties"]["properties"][other_property] = ( - other_property == property - ) or other_property.startswith("NumberOf") - single_parameters = ParameterFile(parameter_dictionary=single_property) - property_calculator_bound = SubhaloProperties( - dummy_halos.get_cell_grid(), - single_parameters, - recently_heated_gas_filter, - stellar_age_calculator, - cat_filter, - ) - property_calculator_both = SubhaloProperties( - dummy_halos.get_cell_grid(), - single_parameters, - recently_heated_gas_filter, - stellar_age_calculator, - cat_filter, - False, - ) - halo_result = {} - for subhalo_name, prop_calc in [ - # ("FOFSubhaloProperties", property_calculator_both), - ("BoundSubhalo", property_calculator_bound) - ]: - input_data = {} - for ptype in prop_calc.particle_properties: - if ptype in data: - input_data[ptype] = {} - for dset in prop_calc.particle_properties[ptype]: - input_data[ptype][dset] = data[ptype][dset] - input_halo_copy = input_halo.copy() - input_data_copy = input_data.copy() - prop_calc.calculate(input_halo, 0.0 * unyt.kpc, input_data, halo_result) - assert input_halo == input_halo_copy - assert input_data == input_data_copy - - # check that the calculation returns the correct values - for prop in prop_calc.property_list: - outputname = prop[1] - if not outputname == property: - continue - size = prop[2] - dtype = prop[3] - unit_string = prop[4] - full_name = f"{subhalo_name}/{outputname}" - assert full_name in halo_result - result = halo_result[full_name][0] - assert (len(result.shape) == 0 and size == 1) or result.shape[0] == size - assert result.dtype == dtype - unit = unyt.Unit(unit_string, registry=dummy_halos.unit_registry) - assert result.units.same_dimensions_as(unit.units) - - dummy_halos.get_cell_grid().snapshot_datasets.print_dataset_log() - - -if __name__ == "__main__": - """ - Standalone version of the program: just run the unit test. - - Note that this can also be achieved by running "pytest *.py" in the folder. - """ - print("Running test_subhalo_properties()...") - test_subhalo_properties() - print("Test passed.") + if self.record_timings: + arr = unyt.unyt_array( + timings.get(name, 0), + dtype=np.float32, + units=unyt.dimensionless, + registry=registry, + ) + halo_result.update( + { + f"{self.group_name}/{outputname}_time": ( + arr, + "Time taken in seconds", + True, + None, + ) + } + ) diff --git a/SOAP/property_calculation/cylindrical_coordinates.py b/SOAP/property_calculation/cylindrical_coordinates.py new file mode 100644 index 00000000..223ac266 --- /dev/null +++ b/SOAP/property_calculation/cylindrical_coordinates.py @@ -0,0 +1,77 @@ +#! /usr/bin/env python + +""" +cylindrical_coordinates.py + +Utility function for converting particles to a cylindrical coordinate system + +""" + +import numpy as np + + +def build_rotation_matrix(z_target): + """ + Build a rotation matrix that aligns the new z-axis + with the given `z_target` vector. + + Parameters: + z_target: A 3-element array representing + the target direction to align with the new z-axis. + + Returns: + R: A (3, 3) rotation matrix. Applying R to a + set of vectors rotates them into new frame: + v_new = v_original @ R.T + where v_original has shape (N, 3) + """ + z_axis = z_target / np.linalg.norm(z_target) + + # Pick a helper vector that is not nearly parallel to z_axis + helper = np.array([1, 0, 0]) + if np.allclose(z_axis, helper / np.linalg.norm(helper), rtol=0.1): + helper = np.array([0, 1, 0]) + + # Construct orthonormal basis + x_axis = np.cross(helper, z_axis) + x_axis /= np.linalg.norm(x_axis) + + y_axis = np.cross(z_axis, x_axis) + + R = np.vstack([x_axis, y_axis, z_axis]) + return R + + +def calculate_cylindrical_velocities(positions, velocities, z_target): + """ + Convert 3D Cartesian velocities to cylindrical coordinates (v_r, v_phi, v_z), + after rotating the system such that the z-axis aligns with `z_target`. + + Parameters: + positions: (N, 3) array of particle positions in the original Cartesian frame. + velocities: (N, 3) array of particle velocities in the original Cartesian frame. + z_target: A 3-element vector indicating the new z-axis direction. + + Returns: + cyl_velocities: (N, 3) array of velocities in cylindrical coordinates: + [v_r, v_phi, v_z] for each particle. + """ + R = build_rotation_matrix(z_target) + + # Rotate positions and velocities into new frame + positions_rot = positions @ R.T + velocities_rot = velocities @ R.T + + x = positions_rot[:, 0] + y = positions_rot[:, 1] + vx = velocities_rot[:, 0] + vy = velocities_rot[:, 1] + vz = velocities_rot[:, 2] + + phi = np.arctan2(y, x) + + v_r = vx * np.cos(phi) + vy * np.sin(phi) + v_phi = -vx * np.sin(phi) + vy * np.cos(phi) + v_z = vz + + return np.stack([v_r, v_phi, v_z], axis=1) diff --git a/SOAP/property_calculation/half_mass_radius.py b/SOAP/property_calculation/half_mass_radius.py new file mode 100644 index 00000000..1daecf1d --- /dev/null +++ b/SOAP/property_calculation/half_mass_radius.py @@ -0,0 +1,161 @@ +#! /usr/bin/env python + +""" +half_mass_radius.py + +Utility functions to compute the half mass or half light radius of a particle +distribution. + +We put this in a separate file to facilitate unit testing. +""" + +import numpy as np +import unyt + + +def get_half_weight_radius( + radius: unyt.unyt_array, weights: unyt.unyt_array, total_weight: unyt.unyt_quantity +) -> unyt.unyt_quantity: + """ + Get the radius that encloses half of the total weight of the given particle distribution. + + We obtain the half weight radius by sorting the particles on radius and then computing + the cumulative weight profile from this. We then determine in which "bin" the cumulative + weight profile intersects the target half weight value and obtain the corresponding + radius from linear interpolation. + + Parameters: + - radius: unyt.unyt_array + Radii of the particles. + - weight: unyt.unyt_array + Weight of individual particles. + - total_weight: unyt.unyt_quantity + Total weight of the particles. Should be weights.sum(). We pass this on as an argument + because this value might already have been computed before. If it was not, then + computing it in the function call is still an efficient way to do this. + + Returns the radius that encloses half of the summed weight, defined as the radius + at which the cumulative weight profile reaches 0.5 * total_weight. + """ + if total_weight == 0.0 * total_weight.units or len(weights) < 1: + return 0.0 * radius.units + + target_weight = 0.5 * total_weight + + isort = np.argsort(radius) + sorted_radius = radius[isort] + + # Compute sum in double precision to avoid numerical overflow due to + # weird unit conversions in unyt + cumulative_weights = weights[isort].cumsum(dtype=np.float64) + + # Consistency check. + # np.sum() and np.cumsum() use different orders, so we have to allow for + # some small difference. + if cumulative_weights[-1] < 0.999 * total_weight: + raise RuntimeError( + "Weights sum up to less than the given total weight value:" + f" cumulative_weights[-1] = {cumulative_weights[-1]}," + f" total_weights = {total_weight}!" + ) + + # Find the intersection point, abd if that is the first bin, set the lower limits to 0. + ihalf = np.argmax(cumulative_weights >= target_weight) + if ihalf == 0: + rmin = 0.0 * radius.units + WeightMin = 0.0 * weights.units + else: + rmin = sorted_radius[ihalf - 1] + WeightMin = cumulative_weights[ihalf - 1] + rmax = sorted_radius[ihalf] + WeightMax = cumulative_weights[ihalf] + + # Now get the radius by linearly interpolating. If the bin edges coincide + # (two particles at exactly the same radius) then we simply take that radius + if WeightMin == WeightMax: + half_weight_radius = 0.5 * (rmin + rmax) + else: + half_weight_radius = rmin + (target_weight - WeightMin) / ( + WeightMax - WeightMin + ) * (rmax - rmin) + + # Consistency check. + # We cannot use '>=', since equality would happen if half_mass_radius == 0. + if half_weight_radius > sorted_radius[-1]: + raise RuntimeError( + "Half weight radius larger than input radii:" + f" half_mass_radius = {half_weight_radius}," + f" sorted_radius[-1] = {sorted_radius[-1]}!" + f" ihalf = {ihalf}, Npart = {len(radius)}," + f" target_weight = {target_weight}," + f" rmin = {rmin}, rmax = {rmax}," + f" WeightMin = {WeightMin}, WeightMax = {WeightMax}," + f" sorted_radius = {sorted_radius}," + f" cumulative_weights = {cumulative_weights}" + ) + + return half_weight_radius + + +def get_half_mass_radius( + radius: unyt.unyt_array, mass: unyt.unyt_array, total_mass: unyt.unyt_quantity +) -> unyt.unyt_quantity: + """ + Get the half mass radius of the given particle distribution. + + We obtain the half mass radius by sorting the particles on radius and then computing + the cumulative mass profile from this. We then determine in which "bin" the cumulative + mass profile intersects the target half mass value and obtain the corresponding + radius from linear interpolation. + + Parameters: + - radius: unyt.unyt_array + Radii of the particles. + - mass: unyt.unyt_array + Mass of the particles. + - total_mass: unyt.unyt_quantity + Total mass of the particles. Should be mass.sum(). We pass this on as an argument + because this value might already have been computed before. If it was not, then + computing it in the function call is still an efficient way to do this. + + Returns the half mass radius, defined as the radius at which the cumulative mass profile + reaches 0.5 * total_mass. + """ + return get_half_weight_radius(radius, mass, total_mass) + + +def get_half_light_radius( + radius: unyt.unyt_array, + band_luminosity: unyt.unyt_array, + total_band_luminosites: unyt.unyt_array, +) -> unyt.unyt_quantity: + """ + Get the half light radius of the given particle distribution for the 9 GAMA + bands. + + We obtain the half light radius by sorting the particles on radius and then computing + the cumulative light profile from this. We then determine in which "bin" the cumulative + light profile intersects the target half light value and obtain the corresponding + radius from linear interpolation. + + Parameters: + - radius: unyt.unyt_array + Radii of the particles. + - band_luminosity: unyt.unyt_array + Luminosity of the particles in each GAMA band. + - total_band_luminosites: unyt.unyt_array + Total luminosisty of the particles in each GAMA band. Should be luminosity.sum(axis=0). + We pass this on as an argument because this value might already have been computed before. + If it was not, then computing it in the function call is still an efficient way to do this. + + Returns the half light radius, defined as the radius at which the cumulative mass profile + reaches 0.5 * total_luminosity, for each GAMA band. + """ + half_light_radii = np.zeros(total_band_luminosites.shape[0]) * radius.units + for i_band, (luminosity, total_luminosity) in enumerate( + zip(band_luminosity.T, total_band_luminosites) + ): + half_light_radii[i_band] = get_half_weight_radius( + radius, luminosity, total_luminosity + ) + return half_light_radii diff --git a/SOAP/property_calculation/inertia_tensors.py b/SOAP/property_calculation/inertia_tensors.py new file mode 100644 index 00000000..3c11a5fd --- /dev/null +++ b/SOAP/property_calculation/inertia_tensors.py @@ -0,0 +1,439 @@ +#! /usr/bin/env python + +""" +inertia_tensors.py + +Some utility functions to compute inertia tensors for particle spatial +distributions. + +We put them in a separate file to facilitate unit testing. +""" + +import numpy as np +from typing import Union, Tuple +import unyt + +from SOAP.particle_selection.halo_properties import SearchRadiusTooSmallError + + +def get_weighted_inertia_tensor( + particle_weights, + particle_positions, + sphere_radius, + search_radius=None, + reduced=False, + max_iterations=20, + min_particles=20, +): + """ + Get the inertia tensor of the given particle distribution weighted by a given + quantity. Computed as: + I_{ij} = w*x_i*x_j / Wtot. + + Parameters: + - particle_weights: unyt.unyt_array + Weight given to each particle. + - particle_positions: unyt.unyt_array + Positions of the particles. + - sphere_radius: unyt.unyt_quantity + Use all particles within a sphere of this size for the calculation + - search_radius: unyt.unyt_quantity + Radius of the region of the simulation for which we have particle data + This function throws a SearchRadiusTooSmallError if we need particles outside + of this region. + - reduced: bool + Whether to calculate the reduced inertia tensor + - max_iterations: int + The maximum number of iterations to repeat the inertia tensor calculation + - min_particles: int + The number of particles required within the initial sphere. The inertia tensor + is not computed if this threshold is not met. + + Returns a flattened representation of the weighted inertia tensor, with the + first 3 entries corresponding to the diagonal terms and the rest to the + off-diagonal terms. + """ + + # Check we have at least "min_particles" particles + if particle_weights.shape[0] < min_particles: + return None + + # Remove particles at centre if calculating reduced tensor + if reduced: + norm = np.linalg.norm(particle_positions, axis=1) ** 2 + mask = np.logical_not(np.isclose(norm, 0)) + + norm = norm[mask] + particle_weights = particle_weights[mask] + particle_positions = particle_positions[mask] + + # Set stopping criteria + tol = 0.0001 + q = 1000 + + # Ensure we have consistent units + R = sphere_radius.to("kpc") + particle_positions = particle_positions.to("kpc") + + # Start with a sphere of size equal to the initial aperture + eig_val = [1, 1, 1] + eig_vec = np.array([[1, 0, 0], [0, 1, 0], [0, 0, 1]]) + + for i_iter in range(max_iterations): + # Calculate shape + old_q = q + q = np.sqrt(eig_val[1] / eig_val[2]) + s = np.sqrt(eig_val[0] / eig_val[2]) + p = np.sqrt(eig_val[0] / eig_val[1]) + + # Break if converged + if abs((old_q - q) / q) < tol: + break + + # Calculate ellipsoid, determine which particles are inside + axis = R * np.array( + [1 * np.cbrt(s * p), 1 * np.cbrt(q / p), 1 / np.cbrt(q * s)] + ) + p = np.dot(particle_positions, eig_vec) / axis + r = np.linalg.norm(p, axis=1) + + # We want to skip the calculation if we have less than "min_particles" + # inside the initial sphere. We do the check here since this is the first + # time we calculate how many particles are within the sphere. + if (i_iter == 0) and (np.sum(r <= 1) < min_particles): + return None + weight = particle_weights / np.sum(particle_weights[r <= 1]) + weight[r > 1] = 0 + + # Check if we have exceeded the search radius. For subhalo_properties we + # have all the bound particles, and so the search radius doesn't matter + if (search_radius is not None) and (np.max(R) > search_radius): + raise SearchRadiusTooSmallError("Inertia tensor required more particles") + + # Calculate inertia tensor + tensor = ( + weight[:, None, None] + * particle_positions[:, :, None] + * particle_positions[:, None, :] + ) + if reduced: + tensor /= norm[:, None, None] + tensor = tensor.sum(axis=0) + eig_val, eig_vec = np.linalg.eigh(tensor.value) + + # Handle overflows into negative values of very small eigenvalues + eig_val = np.abs(eig_val) + + # Handle cases where there is only one particle after iterating. + if q == 0: + tensor.fill(0) + break + + return np.concatenate([np.diag(tensor), tensor[np.triu_indices(3, 1)]]) + + +def get_inertia_tensor_mass_weighted( + particle_masses, + particle_positions, + sphere_radius, + search_radius=None, + reduced=False, + max_iterations=20, + min_particles=20, +): + """ + Get the mass-weighted inertia tensors of the given particle distribution + in each of the available luminosity bands. Computed as: + I_{ij} = M * x_i * x_j / Mtot. + + This function calls get_weighted_inertia_tensor and weights particles by + their mass. See get_weighted_inertia_tensor for + input parameters. + + Returns a flattened representation of the mass-weighted inertia tensor, with the + first 3 entries corresponding to the diagonal terms and the rest to the + off-diagonal terms. + """ + return get_weighted_inertia_tensor( + particle_masses, + particle_positions, + sphere_radius, + search_radius, + reduced, + max_iterations, + min_particles, + ) + + +def get_inertia_tensor_luminosity_weighted( + particle_luminosities, + particle_positions, + sphere_radius, + search_radius=None, + reduced=False, + max_iterations=20, + min_particles=20, +): + """ + Get the luminosity-weighted inertia tensors of the given particle distribution + in each of the available luminosity bands. Computed as: + I_{ij} = Li * x_i * x_j / Ltot. + + This function calls get_weighted_inertia_tensor and weights particles by + their luminosity in a given band. See get_weighted_inertia_tensor for + input parameters. + + Returns an array of concatenated flattened luminosity-weighted inertia tensors, + with each 6 consecutive entries corresponding to 3 diagonal and 3 off-diagonal terms + in a given band. + """ + + number_luminosity_bands = particle_luminosities.shape[1] + + for i_band, particle_luminosities_i_band in enumerate(particle_luminosities.T): + flattened_inertia_tensor_i_band = get_weighted_inertia_tensor( + particle_luminosities_i_band, + particle_positions, + sphere_radius, + search_radius, + reduced, + max_iterations, + min_particles, + ) + + # Not enough particles in the first band, which means not enough particles + # in the other bands. + if flattened_inertia_tensor_i_band is None: + return None + + # Create the array to output here, once we know the units of the inertia tensor. + # 6 elements per luminosity band (3 diagonal + 3 off-diagonal terms). + if i_band == 0: + flattened_inertia_tensors = unyt.unyt_array( + np.zeros(6 * number_luminosity_bands), + dtype=np.float32, + units=flattened_inertia_tensor_i_band.units, + registry=flattened_inertia_tensor_i_band.units.registry, + ) + + flattened_inertia_tensors[6 * i_band : 6 * (i_band + 1)] = ( + flattened_inertia_tensor_i_band + ) + + return flattened_inertia_tensors + + +def get_weighted_projected_inertia_tensor( + particle_weights, + particle_positions, + axis, + radius, + reduced=False, + max_iterations=20, + min_particles=20, +): + """ + Takes in the particle distribution, projects it along a given axis, and + calculates the 2D inertia tensor of the projected particle distribution. + + Unlike get_inertia_tensor, we don't need to check if we have exceeded the search radius. This + is because all the bound particles are passed to this function. + + Parameters: + - particle_weights: unyt.unyt_array + Weight given to each particle. + - particle_positions: unyt.unyt_array + Positions of the particles. + - axis: 0, 1, 2 + Projection axis. Only the coordinates perpendicular to this axis are + taken into account. + - radius: unyt.unyt_quantity + Exclude particles outside this radius for the inertia tensor calculation + - reduced: bool + Whether to calculate the reduced inertia tensor + - max_iterations: int + The maximum number of iterations to repeat the inertia tensor calculation + - min_particles: int + The number of particles required within the initial circle. The inertia tensor + is not computed if this threshold is not met. + + Returns the inertia tensor. + """ + + # Check we have at least "min_particles" particles + if particle_weights.shape[0] < min_particles: + return None + + projected_position = unyt.unyt_array( + np.zeros((particle_positions.shape[0], 2)), + units=particle_positions.units, + dtype=particle_positions.dtype, + ) + if axis == 0: + projected_position[:, 0] = particle_positions[:, 1] + projected_position[:, 1] = particle_positions[:, 2] + elif axis == 1: + projected_position[:, 0] = particle_positions[:, 2] + projected_position[:, 1] = particle_positions[:, 0] + elif axis == 2: + projected_position[:, 0] = particle_positions[:, 0] + projected_position[:, 1] = particle_positions[:, 1] + else: + raise AttributeError(f"Invalid axis: {axis}!") + + # Remove particles at centre if calculating reduced tensor + if reduced: + norm = np.linalg.norm(projected_position, axis=1) ** 2 + mask = np.logical_not(np.isclose(norm, 0)) + + norm = norm[mask] + particle_weights = particle_weights[mask] + projected_position = projected_position[mask] + + # Set stopping criteria + tol = 0.0001 + q = 1000 + + # Ensure we have consistent units + R = radius.to("kpc") + projected_position = projected_position.to("kpc") + + # Start with a circle of size equal to the initial aperture + eig_val = [1, 1] + eig_vec = np.array([[1, 0], [0, 1]]) + + for i_iter in range(max_iterations): + # Calculate shape + old_q = q + q = np.sqrt(eig_val[0] / eig_val[1]) + + # Break if converged + if abs((old_q - q) / q) < tol: + break + + # Calculate ellipse, determine which particles are inside + axis = R * np.array([1 * np.sqrt(q), 1 / np.sqrt(q)]) + p = np.dot(projected_position, eig_vec) / axis + r = np.linalg.norm(p, axis=1) + + # We want to skip the calculation if we have less than "min_particles" + # inside the initial circle. We do the check here since this is the first + # time we calculate how many particles are within the circle. + if (i_iter == 0) and (np.sum(r <= 1) < min_particles): + return None + weight = particle_weights / np.sum(particle_weights[r <= 1]) + weight[r > 1] = 0 + + # Calculate inertia tensor + tensor = ( + weight[:, None, None] + * projected_position[:, :, None] + * projected_position[:, None, :] + ) + if reduced: + tensor /= norm[:, None, None] + tensor = tensor.sum(axis=0) + eig_val, eig_vec = np.linalg.eigh(tensor.value) + + # Handle cases where there is only one particle after iterating. + if q == 0: + tensor.fill(0) + break + + return np.concatenate([np.diag(tensor), [tensor[(0, 1)]]]) + + +def get_projected_inertia_tensor_mass_weighted( + particle_masses, + particle_positions, + axis, + radius, + reduced=False, + max_iterations=20, + min_particles=20, +): + """ + Takes in the particle distribution projected along a given axis, and + calculates the inertia tensor using the projected values. + + This function calls get_weighted_projected_inertia_tensor and weights + particles by their mass. See get_weighted_projected_inertia_tensor for + input parameters. + + Returns a flattened representation of the mass-weighted inertia tensor, with the + first 2 entries corresponding to the diagonal terms and the rest to the + off-diagonal terms. + """ + return get_weighted_projected_inertia_tensor( + particle_masses, + particle_positions, + axis, + radius, + reduced, + max_iterations, + min_particles, + ) + + +def get_projected_inertia_tensor_luminosity_weighted( + particle_luminosities, + particle_positions, + axis, + radius, + reduced=False, + max_iterations=20, + min_particles=20, +): + """ + Takes in the particle distribution projected along a given axis, and + calculates the luminosity-weighted inertia tensor using the projected values. + + This function calls get_weighted_projected_inertia_tensor and weights + particles by their luminosity in a given band. See + get_weighted_projected_inertia_tensor for input parameters. + + Returns an array of concatenated flattened luminosity-weighted inertia tensors, + with each 3 consecutive entries corresponding to 2 diagonal and 1 off-diagonal terms + in a given band. + """ + + number_luminosity_bands = particle_luminosities.shape[1] + + for i_band, particle_luminosities_i_band in enumerate(particle_luminosities.T): + flattened_inertia_tensor_i_band = get_weighted_projected_inertia_tensor( + particle_luminosities_i_band, + particle_positions, + axis, + radius, + reduced, + max_iterations, + min_particles, + ) + + # Not enough particles in the first band, which means not enough particles + # in the other bands. + if flattened_inertia_tensor_i_band is None: + return None + + # Create the array to output here, once we know the units of the inertia tensor. + # 3 elements per luminosity band (2 diagonal + 1 off-diagonal terms) + if i_band == 0: + flattened_inertia_tensors = unyt.unyt_array( + np.zeros(3 * number_luminosity_bands), + dtype=np.float32, + units=flattened_inertia_tensor_i_band.units, + registry=flattened_inertia_tensor_i_band.units.registry, + ) + + flattened_inertia_tensors[3 * i_band : 3 * (i_band + 1)] = ( + flattened_inertia_tensor_i_band + ) + + return flattened_inertia_tensors + + +if __name__ == "__main__": + """ + Standalone version. TODO: add test to check if inertia tensor computation works. + """ + pass diff --git a/SOAP/property_calculation/kinematic_properties.py b/SOAP/property_calculation/kinematic_properties.py new file mode 100644 index 00000000..14aeab68 --- /dev/null +++ b/SOAP/property_calculation/kinematic_properties.py @@ -0,0 +1,434 @@ +#! /usr/bin/env python + +""" +kinematic_properties.py + +Some utility functions to compute kinematic properies for particle +distributions. + +We put them in a separate file to facilitate unit testing. +""" + +import numpy as np +from typing import Union, Tuple +import unyt + + +def get_velocity_dispersion_matrix( + mass_fraction: unyt.unyt_array, + velocity: unyt.unyt_array, + ref_velocity: unyt.unyt_array, +) -> unyt.unyt_array: + """ + Compute the velocity dispersion matrix for the particles with the given + fractional mass (particle mass divided by total mass) and velocity, using + the given reference velocity as the centre of mass velocity. + + The result is a 6 element vector containing the unique components XX, YY, + ZZ, XY, XZ and YZ of the velocity dispersion matrix. + + Parameters: + - mass_fraction: unyt.unyt_array + Fractional mass of the particles (mass/mass.sum()). + - velocity: unyt.unyt_array + Velocity of the particles. + - ref_velocity: unyt.unyt_array + Reference point in velocity space. velocity and ref_velocity are assumed + to use the same reference point upon entry into this function. + + Returns an array with 6 elements: the XX, YY, ZZ, XY, XZ and YZ components + of the velocity dispersion matrix. + """ + + result = unyt.unyt_array(np.zeros(6), dtype=np.float32, units=velocity.units**2) + + vrel = velocity - ref_velocity[None, :] + result[0] += (mass_fraction * vrel[:, 0] * vrel[:, 0]).sum() + result[1] += (mass_fraction * vrel[:, 1] * vrel[:, 1]).sum() + result[2] += (mass_fraction * vrel[:, 2] * vrel[:, 2]).sum() + result[3] += (mass_fraction * vrel[:, 0] * vrel[:, 1]).sum() + result[4] += (mass_fraction * vrel[:, 0] * vrel[:, 2]).sum() + result[5] += (mass_fraction * vrel[:, 1] * vrel[:, 2]).sum() + + return result + + +def get_angular_momentum( + mass: unyt.unyt_array, + position: unyt.unyt_array, + velocity: unyt.unyt_array, + ref_position: Union[None, unyt.unyt_array] = None, + ref_velocity: Union[None, unyt.unyt_array] = None, +) -> unyt.unyt_array: + """ + Compute the total angular momentum vector for the particles with the given + masses, positions and velocities, and using the given reference position + and velocity as the centre of mass (velocity). + + Parameters: + - mass: unyt.unyt_array + Masses of the particles. + - position: unyt.unyt_array + Position of the particles. + - velocity: unyt.unyt_array + Velocities of the particles. + - ref_position: unyt.unyt_array or None + Reference position used as centre for the angular momentum calculation. + position and ref_position are assumed to use the same reference point upon + entry into this function. If None, position is assumed to be already using + the desired referece point. + - ref_velocity: unyt.unyt_array or None + Reference point in velocity space for the angular momentum calculation. + velocity and ref_velocity are assumed to use the same reference point upon + entry into this function. If None, velocity is assumed to be already using + the desired reference point. + + Returns the total angular momentum vector. + """ + + if ref_position is None: + prel = position + else: + prel = position - ref_position[None, :] + if ref_velocity is None: + vrel = velocity + else: + vrel = velocity - ref_velocity[None, :] + return (mass[:, None] * np.cross(prel, vrel)).sum(axis=0) + + +def get_angular_momentum_and_kappa_corot_weighted( + particle_masses: unyt.unyt_array, + particle_positions: unyt.unyt_array, + particle_velocities: unyt.unyt_array, + particle_weights: Union[None, unyt.unyt_array] = None, + reference_position: Union[None, unyt.unyt_array] = None, + reference_velocity: Union[None, unyt.unyt_array] = None, + do_counterrot_mass: bool = False, + do_counterrot_weight: bool = False, +) -> Union[ + Tuple[unyt.unyt_array, unyt.unyt_array], + Tuple[unyt.unyt_array, unyt.unyt_array, unyt.unyt_array], + Tuple[unyt.unyt_array, unyt.unyt_array, unyt.unyt_array, unyt.unyt_array], +]: + """ + Get the total angular momentum vector and kappa_corot (Correa et al., 2017) + for the particles with the given masses, positions, velocities and weights. + It uses the given reference position and velocity as the spatial and velocity + centres. It optionally returns the total mass and total weight value of + counterrotating particles. + + We use this function if both kappa_corot and the angular momentum vector are + requested, as it is more efficient than get_angular_momentum() and + get_kappa_corot() (if that would even exist). + + Parameters: + - particle_masses: unyt.unyt_array + Masses of the particles. + - particle_positions: unyt.unyt_array + Position of the particles. + - particle_velocities: unyt.unyt_array + Velocities of the particles. + - particle_weights: unyt.unyt_array or None + Weights given to each particle when computing the total angular momentum. + If not provided, this function returns the mass-weighted inertia tensor. + - reference_position: unyt.unyt_array or None + Reference position used as centre for the angular momentum calculation. + particle_positions and reference_position are assumed to use the same reference point upon + entry into this function. If None, particle_positions is assumed to be already using + the desired referece point. + - reference_velocity: unyt.unyt_array or None + Reference point in velocity space for the angular momentum calculation. + particle_velocities and reference_velocity are assumed to use the same reference point upon + entry into this function. If None, particle_velocities is assumed to be already using + the desired reference point. + - do_counterrot_mass: bool + Also compute the counterrotating mass? + - do_counterrot_weight: bool + Also compute the counterrotating weight quantity? + + Returns: + - The weighted total angular momentum vector, or if no weights are provided, + the usual definition of angular momentum. + - The ratio of the kinetic energy in counterrotating movement and the total + kinetic energy, kappa_corot. + - The total mass of counterrotating particles if do_counterrot_mass == True. + - The total weight of counterrotating particles if do_counterrot_weight == True. + """ + + if reference_position is None: + prel = particle_positions + else: + prel = particle_positions - reference_position[None, :] + if reference_velocity is None: + vrel = particle_velocities + else: + vrel = particle_velocities - reference_velocity[None, :] + + # We compute the normal angular momentum because we require it for the + # kinetic energy calculation in kappa_corot. + Lpart = particle_masses[:, None] * np.cross(prel, vrel) + + # If we weight the angular momentum, divide by the particle mass so that + # it L_i = w_i * (r_i x v_i). We also estimate a total weighted mass in order + # to return an angular momentum with the correct units. + if particle_weights is not None: + weights = particle_weights / particle_weights.sum() / particle_masses + weighted_total_mass = ( + particle_weights / particle_weights.sum() * particle_masses + ).sum() * len(particle_masses) + + # Normal (mass-weighted) definition of the angular momentum. + else: + weights = np.ones(1) + weighted_total_mass = 1 + + # Weighted average of the (specific) angular momentum. We multiply by the + # weighted total mass to have the correct units for angular momentum, but we + # are only really interested in its direction. + Ltot = weighted_total_mass * (weights[:, None] * Lpart).sum(axis=0) + Lnrm = np.linalg.norm(Ltot) + + # Define the output variables that we will use. Unit registry is the same for + # all fields, hence why we use particle_masses.units.registry + kappa_corot = unyt.unyt_array( + 0.0, + dtype=np.float32, + units="dimensionless", + registry=particle_masses.units.registry, + ) + + if do_counterrot_mass: + M_counterrot = unyt.unyt_array( + 0.0, + dtype=np.float32, + units=particle_masses.units, + registry=particle_masses.units.registry, + ) + + if do_counterrot_weight: + W_counterrot = unyt.unyt_array( + 0.0, + dtype=np.float32, + units=particle_weights.units, + registry=particle_masses.units.registry, + ) + + if Lnrm > 0.0 * Lnrm.units: + + # Total kinetic energy + K = 0.5 * (particle_masses[:, None] * vrel**2).sum() + + # Angular momentum of individual particles projected along the weighted + # angular momentum direction. + if K > 0.0 * K.units or do_counterrot_mass: + Ldir = Ltot / Lnrm + Li = (Lpart * Ldir[None, :]).sum(axis=1) + + if K > 0.0 * K.units: + + # Distance to origin of the coordinate system. + r2 = prel[:, 0] ** 2 + prel[:, 1] ** 2 + prel[:, 2] ** 2 + + # Get distance to axis of rotation for each particle. + rdotL = (prel * Ldir[None, :]).sum(axis=1) + Ri2 = r2 - rdotL**2 + + # Deal with division by zero (the first particle may be in the centre) + mask = Ri2 == 0.0 + Ri2[mask] = 1.0 * Ri2.units + + # Get kinetic energy in rotation & co-rotation, and hence kappa_corot. + Krot = 0.5 * (Li**2 / (particle_masses * Ri2)) + Kcorot = Krot[(~mask) & (Li > 0.0 * Li.units)].sum() + kappa_corot += Kcorot / K + + if do_counterrot_mass: + M_counterrot += particle_masses[Li < 0.0 * Li.units].sum() + + if do_counterrot_weight: + W_counterrot += particle_weights[Li < 0.0 * Li.units].sum() + + if do_counterrot_mass & do_counterrot_weight: + return Ltot, kappa_corot, M_counterrot, W_counterrot + elif do_counterrot_weight: + return Ltot, kappa_corot, W_counterrot + elif do_counterrot_mass: + return Ltot, kappa_corot, M_counterrot + else: + return Ltot, kappa_corot + + +def get_angular_momentum_and_kappa_corot_mass_weighted( + particle_masses: unyt.unyt_array, + particle_positions: unyt.unyt_array, + particle_velocities: unyt.unyt_array, + reference_position: Union[None, unyt.unyt_array] = None, + reference_velocity: Union[None, unyt.unyt_array] = None, + do_counterrot_mass: bool = False, +) -> Union[ + Tuple[unyt.unyt_array, unyt.unyt_quantity], + Tuple[unyt.unyt_array, unyt.unyt_quantity, unyt.unyt_quantity], +]: + """ + Get the total angular momentum vector and kappa_corot (Correa et al., 2017) + for the particles with the given masses, positions and velocities, and using + the given reference position and velocity as centre of mass (velocity). It + optionally returns the total mass of counterrotating particles. + + This function calls get_angular_momentum_and_kappa_corot_weighted without + weighting particles. See get_angular_momentum_and_kappa_corot_weighted for + input parameters and outputs. + """ + return get_angular_momentum_and_kappa_corot_weighted( + particle_masses=particle_masses, + particle_positions=particle_positions, + particle_velocities=particle_velocities, + reference_position=reference_position, + reference_velocity=reference_velocity, + do_counterrot_mass=do_counterrot_mass, + ) + + +def get_angular_momentum_and_kappa_corot_luminosity_weighted( + particle_masses: unyt.unyt_array, + particle_positions: unyt.unyt_array, + particle_velocities: unyt.unyt_array, + particle_luminosities: unyt.unyt_array, + reference_position: Union[None, unyt.unyt_array] = None, + reference_velocity: Union[None, unyt.unyt_array] = None, + do_counterrot_mass: bool = False, + do_counterrot_luminosity: bool = False, +) -> Union[ + Tuple[unyt.unyt_array, unyt.unyt_array], + Tuple[unyt.unyt_array, unyt.unyt_array, unyt.unyt_array], + Tuple[unyt.unyt_array, unyt.unyt_array, unyt.unyt_array, unyt.unyt_array], +]: + """ + Get the total angular momentum vector and kappa_corot (Correa et al., 2017) + for the particles with the given masses, positions, velocities and luminosities, + and using the given reference position and velocity as the spatial and velocity + centres. It optionally returns the total mass and total luminosity of counterrotating + particles. + + This function calls get_angular_momentum_and_kappa_corot_weighted and weights + particles by their luminosity in a given band. See + get_angular_momentum_and_kappa_corot_weighted for input parameters and outputs. + """ + + number_luminosity_bands = particle_luminosities.shape[1] + + # Create output arrays depending on what we have requested. + Ltot = unyt.unyt_array( + np.zeros(3 * number_luminosity_bands), + dtype=np.float32, + units=particle_masses.units + * particle_positions.units + * particle_velocities.units, + registry=particle_masses.units.registry, + ) + kappa_corot = unyt.unyt_array( + np.zeros(number_luminosity_bands), + dtype=np.float32, + units="dimensionless", + registry=particle_masses.units.registry, + ) + if do_counterrot_mass: + M_counterrot = unyt.unyt_array( + np.zeros(number_luminosity_bands), + dtype=np.float32, + units=particle_masses.units, + registry=particle_masses.units.registry, + ) + if do_counterrot_luminosity: + L_counterrot = unyt.unyt_array( + np.zeros(number_luminosity_bands), + dtype=np.float32, + units=particle_luminosities.units, + registry=particle_luminosities.units.registry, + ) + + for i_band, particle_luminosities_i_band in enumerate(particle_luminosities.T): + output = get_angular_momentum_and_kappa_corot_weighted( + particle_masses=particle_masses, + particle_positions=particle_positions, + particle_velocities=particle_velocities, + particle_weights=particle_luminosities_i_band, + reference_position=reference_position, + reference_velocity=reference_velocity, + do_counterrot_mass=do_counterrot_mass, + do_counterrot_weight=do_counterrot_luminosity, + ) + + # These entries are always in the same order. + Ltot[3 * i_band : 3 * (i_band + 1)] = output[0] + kappa_corot[i_band] = output[1] + + # Need to handle different combinations of output requests. + if do_counterrot_mass & do_counterrot_luminosity: + M_counterrot[i_band] = output[2] + L_counterrot[i_band] = output[3] + continue + elif do_counterrot_mass: + M_counterrot[i_band] = output[2] + continue + elif do_counterrot_luminosity: + L_counterrot[i_band] = output[2] + continue + + if do_counterrot_mass & do_counterrot_luminosity: + return Ltot, kappa_corot, M_counterrot, L_counterrot + elif do_counterrot_luminosity: + return Ltot, kappa_corot, L_counterrot + elif do_counterrot_mass: + return Ltot, kappa_corot, M_counterrot + else: + return Ltot, kappa_corot + + +def get_vmax( + mass: unyt.unyt_array, radius: unyt.unyt_array, nskip: int = 0 +) -> Tuple[unyt.unyt_quantity, unyt.unyt_quantity]: + """ + Get the maximum circular velocity of a particle distribution. + + The value is computed from the cumulative mass profile after + sorting the particles by radius, as + vmax = sqrt(G*M/r) + + Parameters: + - mass: unyt.unyt_array + Mass of the particles. + - radius: unyt.unyt_array + Radius of the particles. + - nskip: int + Number of particles to skip + + Returns: + - Radius at which the maximum circular velocity is reached. + - Maximum circular velocity. + """ + # obtain the gravitational constant in the right units + # (this is read from the snapshot metadata, and is hence + # guaranteed to be consistent with the value used by SWIFT) + G = unyt.Unit("newton_G", registry=mass.units.registry) + isort = np.argsort(radius) + ordered_radius = radius[isort] + cumulative_mass = mass[isort].cumsum() + nskip = max( + nskip, np.argmin(np.isclose(ordered_radius, 0.0 * ordered_radius.units)) + ) + ordered_radius = ordered_radius[nskip:] + if len(ordered_radius) == 0 or ordered_radius[0] == 0: + return 0.0 * radius.units, np.sqrt(0.0 * G * mass.units / radius.units) + cumulative_mass = cumulative_mass[nskip:] + v_over_G = cumulative_mass / ordered_radius + imax = np.argmax(v_over_G) + return ordered_radius[imax], np.sqrt(v_over_G[imax] * G) + + +if __name__ == "__main__": + """ + Standalone version. + """ + pass diff --git a/stellar_age_calculator.py b/SOAP/property_calculation/stellar_age_calculator.py similarity index 80% rename from stellar_age_calculator.py rename to SOAP/property_calculation/stellar_age_calculator.py index 0e7ef2c4..a8f952d9 100644 --- a/stellar_age_calculator.py +++ b/SOAP/property_calculation/stellar_age_calculator.py @@ -11,14 +11,11 @@ current scale factor. """ -import numpy as np -import unyt - from astropy.cosmology import w0waCDM, Cosmology import astropy.constants as const import astropy.units as astropy_units - -from swift_cells import SWIFTCellGrid +import numpy as np +import unyt class StellarAgeCalculator: @@ -38,7 +35,7 @@ class StellarAgeCalculator: # current simulation time t_now: unyt.unyt_quantity - def __init__(self, cellgrid: SWIFTCellGrid): + def __init__(self, cellgrid): """ Constructor. @@ -46,6 +43,7 @@ def __init__(self, cellgrid: SWIFTCellGrid): scale factor and time. Precomputes the current simulation time as set by the current scale factor. + Creates a lookup table rather than calling astropy for every particle Parameters: - cellgrid: SWIFTCellGrid @@ -69,18 +67,18 @@ def __init__(self, cellgrid: SWIFTCellGrid): # expressions taken directly from astropy, since they do no longer # allow access to these attributes (since version 5.1+) critdens_const = (3.0 / (8.0 * np.pi * const.G)).cgs.value - a_B_c2 = (4.0 * const.sigma_sb / const.c ** 3).cgs.value + a_B_c2 = (4.0 * const.sigma_sb / const.c**3).cgs.value # SWIFT provides Omega_r, but we need a consistent Tcmb0 for astropy. # This is an exact inversion of the procedure performed in astropy. critical_density_0 = astropy_units.Quantity( critdens_const * H0.to("1/s").value ** 2, - astropy_units.g / astropy_units.cm ** 3, + astropy_units.g / astropy_units.cm**3, ) Tcmb0 = (Omega_r * critical_density_0.value / a_B_c2) ** (1.0 / 4.0) - self.cosmology = w0waCDM( + cosmology = w0waCDM( H0=H0.to_astropy(), Om0=Omega_m, Ode0=Omega_lambda, @@ -90,9 +88,17 @@ def __init__(self, cellgrid: SWIFTCellGrid): Ob0=Omega_b, ) - self.t_now = unyt.unyt_quantity.from_astropy( - self.cosmology.lookback_time(z_now) - ).to("Myr") + t_now = unyt.unyt_quantity.from_astropy(cosmology.lookback_time(z_now)).to( + "Myr" + ) + + # Create a lookup table for z=49 to z=0 + self.a_lookup = np.linspace(1 / 50, 1, 1000) + z = (1.0 / self.a_lookup) - 1.0 + # Remember that lookback time runs backwards + self.age_lookup = ( + unyt.unyt_array.from_astropy(cosmology.lookback_time(z)).to("Myr") - t_now + ) def stellar_age(self, birth_a: unyt.unyt_array) -> unyt.unyt_array: """ @@ -104,9 +110,4 @@ def stellar_age(self, birth_a: unyt.unyt_array) -> unyt.unyt_array: Returns the corresponding stellar ages in physical time. """ - birth_z = 1.0 / birth_a - 1.0 - t_birth = unyt.unyt_array.from_astropy( - self.cosmology.lookback_time(birth_z.value) - ).to("Myr") - # remember: we use lookback time, which runs backwards! - return t_birth - self.t_now + return np.interp(birth_a, self.a_lookup, self.age_lookup) diff --git a/subhalo_rank.py b/SOAP/property_calculation/subhalo_rank.py similarity index 60% rename from subhalo_rank.py rename to SOAP/property_calculation/subhalo_rank.py index 390400e9..177b5df0 100644 --- a/subhalo_rank.py +++ b/SOAP/property_calculation/subhalo_rank.py @@ -2,7 +2,6 @@ import numpy as np import h5py -import pytest import virgo.mpi.parallel_sort as psort import virgo.mpi.parallel_hdf5 as phdf5 @@ -53,7 +52,7 @@ def compute_subhalo_rank(host_id, subhalo_mass, comm): host_id, return_counts=True, return_index=True ) del host_id - for (i, n) in zip(offset, count): + for i, n in zip(offset, count): subhalo_rank[i : i + n] = np.arange(n, dtype=np.int32) assert np.all(subhalo_rank >= 0) @@ -84,54 +83,3 @@ def compute_subhalo_rank(host_id, subhalo_mass, comm): subhalo_rank = psort.fetch_elements(subhalo_rank, order, comm=comm) return subhalo_rank - - -@pytest.mark.mpi -def test_subhalo_rank(): - - from mpi4py import MPI - - comm = MPI.COMM_WORLD - comm_rank = comm.Get_rank() - - # Read VR halos from a small FLAMINGO run (assumes single file catalogue) - vr_file = "/cosma8/data/dp004/flamingo/Runs/L0100N0180/HYDRO_FIDUCIAL/VR/halos_0006.properties.0" - with h5py.File(vr_file, "r", driver="mpio", comm=comm) as vr: - host_id = phdf5.collective_read(vr["hostHaloID"], comm=comm) - subhalo_id = phdf5.collective_read(vr["ID"], comm=comm) - subhalo_mass = phdf5.collective_read(vr["Mass_tot"], comm=comm) - if comm_rank == 0: - print("Read subhalos") - - field = host_id < 0 - host_id[field] = subhalo_id[field] - - # Compute ranking of subhalos - subhalo_rank = compute_subhalo_rank(host_id, subhalo_mass, comm) - if comm_rank == 0: - print("Computed ranks") - - # Find fraction of VR 'field' halos with rank=0 - nr_field_halos = comm.allreduce(np.sum(field)) - nr_field_rank_nonzero = comm.allreduce(np.sum((field) & (subhalo_rank > 0))) - fraction = nr_field_rank_nonzero / nr_field_halos - if comm_rank == 0: - print(f"Fraction of field halos (hostHaloID<0) with rank>0 is {fraction:.3e}") - - # Sanity check: there should be one instance of each hostHaloID with rank=0 - all_ranks = comm.gather(subhalo_rank) - all_host_ids = comm.gather(host_id) - all_ids = comm.gather(subhalo_id) - if comm_rank == 0: - all_ranks = np.concatenate(all_ranks) - all_host_ids = np.concatenate(all_host_ids) - all_ids = np.concatenate(all_ids) - all_host_ids[all_host_ids < 0] = all_ids[all_host_ids < 0] - rank0 = all_ranks == 0 - rank0_hosts = all_host_ids[rank0] - assert len(rank0_hosts) == len(np.unique(all_host_ids)) - - -if __name__ == "__main__": - # Run test with "mpirun -np 8 python3 -m mpi4py ./subhalo_rank.py" - test_subhalo_rank() diff --git a/SOAP/property_table.py b/SOAP/property_table.py new file mode 100644 index 00000000..9666b4ac --- /dev/null +++ b/SOAP/property_table.py @@ -0,0 +1,5389 @@ +#!/bin/env python + +""" +property_table.py + +This file contains all the properties that can be calculated by SOAP, and some +functionality to automatically generate the documentation (PDF) containing these +properties. + +The rationale for having all of this in one file (and what is essentially one +big dictionary) is consistency: every property is defined exactly once, with +one data type, one unit, one description... Every type of halo still +implements its own calculation of each property, but everything that is exposed +to the user is guaranteed to be consistent for all halo types. To change the +documentation, you need to change the dictionary, so you will automatically +change the code as well. If you remember to regenerate the documentation, the +code will hence always be consistent with its documentation. The documentation +includes a version string to help identify it. +""" + +import numpy as np +import unyt +import subprocess +import datetime +import os +from dataclasses import dataclass +from typing import Dict, List + +from SOAP.particle_selection.halo_properties import HaloProperty + + +def get_version_string() -> str: + """ + Generate a version string that uniquely identifies the documentation file. + + The version string will have the format + SOAP version a7baa6e -- Compiled by user ``vandenbroucke'' on winkel + on Tuesday 15 November 2022, 10:49:10 + or + Unknown SOAP version -- Compiled by user ``vandenbroucke'' on winkel + on Tuesday 15 November 2022, 10:49:10 + if no git version string can be obtained. + """ + + handle = subprocess.run("git describe --always", shell=True, stdout=subprocess.PIPE) + if handle.returncode != 0: + git_version = "Unknown SOAP version" + else: + git_version = handle.stdout.decode("utf-8").strip() + git_version = f"SOAP version ``{git_version}''" + timestamp = datetime.datetime.now().strftime("%A %-d %B %Y, %H:%M:%S") + username = os.getlogin() + hostname = os.uname().nodename + return ( + f"Generated by user ``{username}'' on {hostname} on {timestamp}. {git_version}" + ) + + +def word_wrap_name(name): + """ + Put a line break in if a name gets too long + """ + maxlen = 20 + count = 0 + output = [] + last_was_lower = False + for i in range(len(name)): + next_char = name[i] + count += 1 + if count > maxlen and next_char.isupper() and last_was_lower: + output.append(r"\-") + output.append(next_char) + last_was_lower = next_char.isupper() == False + return "".join(output) + + +@dataclass +class Property: + """ + Dataclass which holds the definition of a property + """ + + name: str + shape: int + dtype: np.dtype + unit: str + description: str + lossy_compression_filter: str + dmo_property: bool + particle_properties: list + output_physical: bool + a_scale_exponent: int + + +class PropertyTable: + """ + Auxiliary object to manipulate the property table. + + The PropertyTable object is only required to generate the documentation. + If you just want to grab the information for a particular property from + the table, you should directly access the static table, e.g. + Mstar_info = PropertyTable.full_property_list["Mstar"] + """ + + # some properties require an additional explanation in the form of a + # footnote. These footnotes are .tex files in the 'documentation' folder + # (that should exist). The name of the file acts as a key in the dictionary + # below; the corresponding value is a list of all properties that should + # include a footnote link to this particular explanation. + explanation = { + "footnote_MBH.tex": ["BHmaxM"], + "footnote_com.tex": ["com", "vcom"], + "footnote_AngMom.tex": ["Lgas", "Ldm", "Lstar", "Lbaryons"], + "footnote_kappa.tex": [ + "kappa_corot_gas", + "kappa_corot_star", + "kappa_corot_baryons", + ], + "footnote_disc_fraction.tex": ["DtoTstar", "DtoTgas"], + "footnote_SF.tex": [ + "SFR", + "gasFefrac_SF", + "gasOfrac_SF", + "Mgas_SF", + "gasmetalfrac_SF", + ], + "footnote_Tgas.tex": [ + "Tgas", + "Tgas_no_agn", + "Tgas_no_cool", + "Tgas_no_cool_no_agn", + ], + "footnote_lum.tex": ["StellarLuminosity"], + "footnote_circvel.tex": ["R_vmax_unsoft", "Vmax_unsoft", "Vmax_soft"], + "footnote_spin.tex": ["spin_parameter"], + "footnote_veldisp_matrix.tex": [ + "veldisp_matrix_gas", + "veldisp_matrix_dm", + "veldisp_matrix_star", + ], + "footnote_proj_veldisp.tex": [ + "proj_veldisp_gas", + "proj_veldisp_dm", + "proj_veldisp_star", + ], + "footnote_elements.tex": [ + "gasOfrac", + "gasOfrac_SF", + "gasFefrac", + "gasFefrac_SF", + "gasmetalfrac", + "gasmetalfrac_SF", + ], + "footnote_halfmass.tex": [ + "HalfMassRadiusTot", + "HalfMassRadiusGas", + "HalfMassRadiusDM", + "HalfMassRadiusStar", + ], + "footnote_satfrac.tex": ["Mfrac_satellites", "Mfrac_external"], + "footnote_Ekin.tex": [ + "KineticEnergyGas", + "KineticEnergyStars", + "KineticEnergyTotal", + ], + "footnote_Etherm.tex": ["ThermalEnergyGas"], + "footnote_Mnu.tex": ["Mnu", "MnuNS"], + "footnote_Xray.tex": [ + "Xraylum", + "Xraylum_restframe", + "Xrayphlum", + "Xrayphlum_restframe", + ], + "footnote_compY.tex": ["compY", "compY_no_agn"], + "footnote_dopplerB.tex": ["DopplerB"], + "footnote_coreexcision.tex": [ + "Tgas_cy_weighted_core_excision", + "Tgas_cy_weighted_core_excision_no_agn", + "Tgas_core_excision", + "Tgas_no_cool_core_excision", + "Tgas_no_agn_core_excision", + "Tgas_no_cool_no_agn_core_excision", + "Xraylum_core_excision", + "Xraylum_no_agn_core_excision", + "Xrayphlum_core_excision", + "Xrayphlum_no_agn_core_excision", + "SpectroscopicLikeTemperature_core_excision", + "SpectroscopicLikeTemperature_no_agn_core_excision", + ], + "footnote_cytemp.tex": [ + "Tgas_cy_weighted", + "Tgas_cy_weighted_no_agn", + "Tgas_cy_weighted_core_excision", + "Tgas_cy_weighted_core_excision_no_agn", + ], + "footnote_spectroscopicliketemperature.tex": [ + "SpectroscopicLikeTemperature", + "SpectroscopicLikeTemperature_core_excision", + "SpectroscopicLikeTemperature_no_agn", + "SpectroscopicLikeTemperature_no_agn_core_excision", + ], + "footnote_dust.tex": [ + "DustGraphiteMass", + "DustGraphiteMassInMolecularGas", + "DustGraphiteMassInAtomicGas", + "DustSilicatesMass", + "DustSilicatesMassInMolecularGas", + "DustSilicatesMassInAtomicGas", + "DustLargeGrainMass", + "DustLargeGrainMassInMolecularGas", + "DustSmallGrainMass", + "DustSmallGrainMassInMolecularGas", + ], + "footnote_diffuse.tex": [ + "DiffuseCarbonMass", + "DiffuseOxygenMass", + "DiffuseMagnesiumMass", + "DiffuseSiliconMass", + "DiffuseIronMass", + ], + "footnote_concentration.tex": [ + "concentration_unsoft", + "concentration_soft", + "concentration_dmo_unsoft", + "concentration_dmo_soft", + ], + "footnote_flow_rates.tex": [ + "DarkMatterMassFlowRate", + "ColdGasMassFlowRate", + "CoolGasMassFlowRate", + "WarmGasMassFlowRate", + "HotGasMassFlowRate", + "HIMassFlowRate", + "H2MassFlowRate", + "MetalMassFlowRate", + "StellarMassFlowRate", + "ColdGasEnergyFlowRate", + "CoolGasEnergyFlowRate", + "WarmGasEnergyFlowRate", + "HotGasEnergyFlowRate", + "ColdGasMomentumFlowRate", + "CoolGasMomentumFlowRate", + "WarmGasMomentumFlowRate", + "HotGasMomentumFlowRate", + ], + "footnote_tensor.tex": [ + "TotalInertiaTensor", + "TotalInertiaTensorReduced", + "TotalInertiaTensorNoniterative", + "TotalInertiaTensorReducedNoniterative", + "ProjectedTotalInertiaTensor", + "ProjectedTotalInertiaTensorReduced", + "ProjectedTotalInertiaTensorNoniterative", + "ProjectedTotalInertiaTensorReducedNoniterative", + ], + "footnote_metallicity.tex": [ + "LogarithmicMassWeightedDiffuseNitrogenOverOxygenOfGasLowLimit", + "LogarithmicMassWeightedDiffuseNitrogenOverOxygenOfGasHighLimit", + "LogarithmicMassWeightedDiffuseCarbonOverOxygenOfGasLowLimit", + "LogarithmicMassWeightedDiffuseCarbonOverOxygenOfGasHighLimit", + "LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfGasLowLimit", + "LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfGasHighLimit", + "LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfAtomicGasLowLimit", + "LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfAtomicGasHighLimit", + "LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfMolecularGasLowLimit", + "LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfMolecularGasHighLimit", + "LogarithmicMassWeightedIronOverHydrogenOfStarsLowLimit", + "LogarithmicMassWeightedIronOverHydrogenOfStarsHighLimit", + "LogarithmicMassWeightedMagnesiumOverHydrogenOfStarsLowLimit", + "LogarithmicMassWeightedMagnesiumOverHydrogenOfStarsHighLimit", + "LogarithmicMassWeightedIronFromSNIaOverHydrogenOfStarsLowLimit", + "LinearMassWeightedOxygenOverHydrogenOfGas", + "LinearMassWeightedNitrogenOverOxygenOfGas", + "LinearMassWeightedCarbonOverOxygenOfGas", + "LinearMassWeightedDiffuseNitrogenOverOxygenOfGas", + "LinearMassWeightedDiffuseCarbonOverOxygenOfGas", + "LinearMassWeightedDiffuseOxygenOverHydrogenOfGas", + "LinearMassWeightedIronOverHydrogenOfStars", + "LinearMassWeightedMagnesiumOverHydrogenOfStars", + "LinearMassWeightedIronFromSNIaOverHydrogenOfStars", + ], + "footnote_progenitor_descendant.tex": [ + "SOAP/DescendantIndex", + "SOAP/ProgenitorIndex", + ], + "footnote_cold_dense.tex": [ + "DustGraphiteMassInColdDenseGas", + "DustLargeGrainMassInColdDenseGas", + "DustSilicatesMassInColdDenseGas", + "DustSmallGrainMassInColdDenseGas", + "GasMassInColdDenseGas", + "GasMassInColdDenseDiffuseMetals", + ], + "footnote_averaged.tex": [ + "MostMassiveBlackHoleAveragedAccretionRate", + "AveragedStarFormationRate", + ], + } + + # dictionary with human-friendly descriptions of the various lossy + # compression filters that can be applied to data. + # The key is the name of a lossy compression filter (same names as used + # by SWIFT), the value is the corresponding description, which can be either + # an actual description or a representative example. + compression_description = { + "FMantissa9": "$1.36693{\\rm{}e}10 \\rightarrow{} 1.367{\\rm{}e}10$", + "FMantissa13": "$1.36693{\\rm{}e}10 \\rightarrow{} 1.3669{\\rm{}e}10$", + "DMantissa9": "$1.36693{\\rm{}e}10 \\rightarrow{} 1.367{\\rm{}e}10$", + "DScale6": "1 pc accurate", + "DScale5": "10 pc accurate", + "DScale1": "0.1 km/s accurate", + "Nbit40": "Store less bits", + "None": "no compression", + } + + # List of all properties that can be computed + # The key for each property is the name that is used internally in SOAP + # For each property, we have the following attributes: + # - name: Name of the property within the output file + # - shape: Shape of this property for a single halo (1: scalar, + # 3: vector...) + # - dtype: Data type that will be used. Should have enough precision to + # avoid over/underflow + # - unit: Units that will be used internally and for the output. + # - description: Description string that will be used to describe the + # property in the output. + # - lossy compression filter: Lossy compression filter used in the output + # to reduce the file size. Note that SOAP does not actually compress + # the output; this is done by a separate script. We support all lossy + # compression filters available in SWIFT. + # - dmo_property: Should this property be calculated for a DMO run? + # - particle_properties: Particle fields that are required to compute this + # property. Used to determine which particle fields to read in + # - output_physical: Whether to output this value as physical or co-moving. + # - a_scale_exponent: What a-scale exponent to set for this property. If set + # to None this marks that the property can not be converted to comoving + # + # Note that there is no good reason to have a diffent internal name and + # output name; this was mostly done for historical reasons. It does mean that + # you can easily change the name in the output without having to change all + # of the other .py files that use this property. + full_property_list = { + "AtomicHydrogenMass": Property( + name="AtomicHydrogenMass", + shape=1, + dtype=np.float32, + unit="snap_mass", + description="Total gas mass in atomic hydrogen.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Masses", + "PartType0/SpeciesFractions", + "PartType0/ElementMassFractions", + ], + output_physical=True, + a_scale_exponent=0, + ), + "BHlasteventa": Property( + name="BlackHolesLastEventScalefactor", + shape=1, + dtype=np.float32, + unit="dimensionless", + description="Scale-factor of last AGN event.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType5/LastAGNFeedbackScaleFactors"], + output_physical=True, + a_scale_exponent=None, + ), + "BlackHolesTotalInjectedThermalEnergy": Property( + name="BlackHolesTotalInjectedThermalEnergy", + shape=1, + dtype=np.float32, + unit="snap_mass*snap_length**2/snap_time**2", + description="Total thermal energy injected into gas particles by all black holes.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType5/AGNTotalInjectedEnergies"], + output_physical=True, + a_scale_exponent=None, + ), + "BlackHolesTotalInjectedJetEnergy": Property( + name="BlackHolesTotalInjectedJetEnergy", + shape=1, + dtype=np.float32, + unit="snap_mass*snap_length**2/snap_time**2", + description="Total jet energy injected into gas particles by all black holes.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType5/InjectedJetEnergies"], + output_physical=True, + a_scale_exponent=None, + ), + "BHmaxAR": Property( + name="MostMassiveBlackHoleAccretionRate", + shape=1, + dtype=np.float32, + unit="snap_mass/snap_time", + description="Gas accretion rate of most massive black hole.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType5/SubgridMasses", "PartType5/AccretionRates"], + output_physical=True, + a_scale_exponent=None, + ), + "MostMassiveBlackHoleAveragedAccretionRate": Property( + name="MostMassiveBlackHoleAveragedAccretionRate", + shape=2, + dtype=np.float32, + unit="snap_mass/snap_time", + description="Gas accretion rate of the most massive black hole, averaged over past 100Myr and past 10Myr. If the time between this snapshot and the previous one was less than the averaging time, then the value is averaged over the time between the snapshots.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType5/SubgridMasses", + "PartType5/AveragedAccretionRates", + ], + output_physical=True, + a_scale_exponent=None, + ), + "MostMassiveBlackHoleInjectedThermalEnergy": Property( + name="MostMassiveBlackHoleInjectedThermalEnergy", + shape=1, + dtype=np.float32, + unit="snap_mass*snap_length**2/snap_time**2", + description="Total thermal energy injected into gas particles by the most massive black hole.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType5/SubgridMasses", + "PartType5/AGNTotalInjectedEnergies", + ], + output_physical=True, + a_scale_exponent=None, + ), + "MostMassiveBlackHoleAccretionMode": Property( + name="MostMassiveBlackHoleAccretionMode", + shape=1, + dtype=np.int32, + unit="dimensionless", + description="Accretion flow regime of the most massive black hole. 0 - Thick disk, 1 - Thin disk, 2 - Slim disk", + lossy_compression_filter="None", + dmo_property=False, + particle_properties=["PartType5/SubgridMasses", "PartType5/AccretionModes"], + output_physical=True, + a_scale_exponent=0, + ), + "MostMassiveBlackHoleGWMassLoss": Property( + name="MostMassiveBlackHoleGWMassLoss", + shape=1, + dtype=np.float32, + unit="snap_mass", + description="Cumulative mass lost to GW via BH-BH mergers over the history of the most massive black holes. This includes the mass loss from all the progenitors.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType5/SubgridMasses", "PartType5/GWMassLosses"], + output_physical=True, + a_scale_exponent=0, + ), + "MostMassiveBlackHoleInjectedJetEnergyByMode": Property( + name="MostMassiveBlackHoleInjectedJetEnergyByMode", + shape=3, + dtype=np.float32, + unit="snap_mass*snap_length**2/snap_time**2", + description="The total energy injected in the kinetic jet AGN feedback mode by the mostmassive black hole, split by accretion mode. The components correspond to the jet energy dumped in the thick, thin and slim disc modes, respectively.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType5/SubgridMasses", + "PartType5/InjectedJetEnergiesByMode", + ], + output_physical=True, + a_scale_exponent=None, + ), + "MostMassiveBlackHoleFormationScalefactor": Property( + name="MostMassiveBlackHoleFormationScalefactor", + shape=1, + dtype=np.float32, + unit="dimensionless", + description="Scale-factor when most massive black hole was formed.", + lossy_compression_filter="FMantissa13", + dmo_property=False, + particle_properties=[ + "PartType5/SubgridMasses", + "PartType5/FormationScaleFactors", + ], + output_physical=True, + a_scale_exponent=None, + ), + "MostMassiveBlackHoleLastJetEventScalefactor": Property( + name="MostMassiveBlackHoleLastJetEventScalefactor", + shape=1, + dtype=np.float32, + unit="dimensionless", + description="Scale-factor of last jet event for most massive black hole.", + lossy_compression_filter="FMantissa13", + dmo_property=False, + particle_properties=[ + "PartType5/SubgridMasses", + "PartType5/LastAGNJetScaleFactors", + ], + output_physical=True, + a_scale_exponent=None, + ), + "MostMassiveBlackHoleNumberOfAGNEvents": Property( + name="MostMassiveBlackHoleNumberOfAGNEvents", + shape=1, + dtype=np.int32, + unit="dimensionless", + description="Number of thermal AGN events the most massive black hole has had so far", + lossy_compression_filter="None", + dmo_property=False, + particle_properties=[ + "PartType5/SubgridMasses", + "PartType5/NumberOfAGNEvents", + ], + output_physical=True, + a_scale_exponent=0, + ), + "MostMassiveBlackHoleNumberOfAGNJetEvents": Property( + name="MostMassiveBlackHoleNumberOfAGNJetEvents", + shape=1, + dtype=np.int32, + unit="dimensionless", + description="Number of jet events the most massive black hole has had so far", + lossy_compression_filter="None", + dmo_property=False, + particle_properties=[ + "PartType5/SubgridMasses", + "PartType5/NumberOfAGNJetEvents", + ], + output_physical=True, + a_scale_exponent=0, + ), + "MostMassiveBlackHoleNumberOfMergers": Property( + name="MostMassiveBlackHoleNumberOfMergers", + shape=1, + dtype=np.int32, + unit="dimensionless", + description="Number of mergers experienced by the most massive black hole.", + lossy_compression_filter="None", + dmo_property=False, + particle_properties=[ + "PartType5/SubgridMasses", + "PartType5/NumberOfMergers", + ], + output_physical=True, + a_scale_exponent=0, + ), + "MostMassiveBlackHoleRadiatedEnergyByMode": Property( + name="MostMassiveBlackHoleRadiatedEnergyByMode", + shape=3, + dtype=np.float32, + unit="snap_mass*snap_length**2/snap_time**2", + description="The total energy launched into radiation by the most massive black hole, split by accretion mode. The components correspond to the radiative energy dumped in the thick, thin and slim disc modes, respectively.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType5/SubgridMasses", + "PartType5/RadiatedEnergiesByMode", + ], + output_physical=True, + a_scale_exponent=None, + ), + "MostMassiveBlackHoleTotalAccretedMass": Property( + name="MostMassiveBlackHoleTotalAccretedMass", + shape=1, + dtype=np.float32, + unit="snap_mass", + description="The total mass accreted by the most massive black hole.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType5/SubgridMasses", + "PartType5/TotalAccretedMasses", + ], + output_physical=True, + a_scale_exponent=0, + ), + "MostMassiveBlackHoleTotalAccretedMassesByMode": Property( + name="MostMassiveBlackHoleTotalAccretedMassesByMode", + shape=3, + dtype=np.float32, + unit="snap_mass", + description="The total mass accreted by the most massive black hole in each accretion mode. The components correspond to the mass accreted in the thick, thin and slim disc modes, respectively.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType5/SubgridMasses", + "PartType5/TotalAccretedMassesByMode", + ], + output_physical=True, + a_scale_exponent=0, + ), + "MostMassiveBlackHoleSpin": Property( + name="MostMassiveBlackHoleSpin", + shape=1, + dtype=np.float32, + unit="dimensionless", + description="Dimensionless spin of the most massive black hole. Negative values indicate retrograde accretion.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType5/SubgridMasses", "PartType5/Spins"], + output_physical=True, + a_scale_exponent=0, + ), + "MostMassiveBlackHoleWindEnergyByMode": Property( + name="MostMassiveBlackHoleWindEnergyByMode", + shape=3, + dtype=np.float32, + unit="snap_mass*snap_length**2/snap_time**2", + description="The total energy launched into accretion disc winds by the most massive black hole, split by accretion mode. The components correspond to the radiative energy dumped in the thick, thin and slim disc modes, respectively.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType5/SubgridMasses", + "PartType5/WindEnergiesByMode", + ], + output_physical=True, + a_scale_exponent=None, + ), + "BHmaxID": Property( + name="MostMassiveBlackHoleID", + shape=1, + dtype=np.uint64, + unit="dimensionless", + description="ID of most massive black hole.", + lossy_compression_filter="None", + dmo_property=False, + particle_properties=["PartType5/SubgridMasses", "PartType5/ParticleIDs"], + output_physical=True, + a_scale_exponent=None, + ), + "BHmaxM": Property( + name="MostMassiveBlackHoleMass", + shape=1, + dtype=np.float32, + unit="snap_mass", + description="Mass of most massive black hole.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType5/SubgridMasses"], + output_physical=True, + a_scale_exponent=0, + ), + "BHmaxlasteventa": Property( + name="MostMassiveBlackHoleLastEventScalefactor", + shape=1, + dtype=np.float32, + unit="dimensionless", + description="Scale-factor of last thermal AGN event for most massive black hole.", + lossy_compression_filter="FMantissa13", + dmo_property=False, + particle_properties=[ + "PartType5/SubgridMasses", + "PartType5/LastAGNFeedbackScaleFactors", + ], + output_physical=True, + a_scale_exponent=None, + ), + "BHmaxpos": Property( + name="MostMassiveBlackHolePosition", + shape=3, + dtype=np.float64, + unit="snap_length", + description="Position of most massive black hole.", + lossy_compression_filter="DScale6", + dmo_property=False, + particle_properties=["PartType5/Coordinates", "PartType5/SubgridMasses"], + output_physical=False, + a_scale_exponent=1, + ), + "BHmaxvel": Property( + name="MostMassiveBlackHoleVelocity", + shape=3, + dtype=np.float32, + unit="snap_length/snap_time", + description="Velocity of most massive black hole relative to the simulation volume.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType5/SubgridMasses", "PartType5/Velocities"], + output_physical=True, + a_scale_exponent=0, + ), + "DarkMatterInertiaTensor": Property( + name="DarkMatterInertiaTensor", + shape=6, + dtype=np.float32, + unit="snap_length**2", + description="3D inertia tensor computed iteratively from the DM mass distribution, relative to the halo centre. Diagonal components and one off-diagonal triangle as (1,1), (2,2), (3,3), (1,2), (1,3), (2,3). Only calculated when we have more than 20 particles.", + lossy_compression_filter="FMantissa9", + dmo_property=True, + particle_properties=["PartType1/Coordinates", "PartType1/Masses"], + output_physical=True, + a_scale_exponent=2, + ), + "DarkMatterInertiaTensorReduced": Property( + name="DarkMatterInertiaTensorReduced", + shape=6, + dtype=np.float32, + unit="dimensionless", + description="Reduced 3D inertia tensor computed iteratively from the DM mass distribution, relative to the halo centre. Diagonal components and one off-diagonal triangle as (1,1), (2,2), (3,3), (1,2), (1,3), (2,3). Only calculated when we have more than 20 particles.", + lossy_compression_filter="FMantissa9", + dmo_property=True, + particle_properties=["PartType1/Coordinates", "PartType1/Masses"], + output_physical=True, + a_scale_exponent=0, + ), + "DarkMatterInertiaTensorNoniterative": Property( + name="DarkMatterInertiaTensorNoniterative", + shape=6, + dtype=np.float32, + unit="snap_length**2", + description="3D inertia tensor computed in a single interation from the DM mass distribution, relative to the halo centre. Diagonal components and one off-diagonal triangle as (1,1), (2,2), (3,3), (1,2), (1,3), (2,3). Only calculated when we have more than 20 particles.", + lossy_compression_filter="FMantissa9", + dmo_property=True, + particle_properties=["PartType1/Coordinates", "PartType1/Masses"], + output_physical=True, + a_scale_exponent=2, + ), + "DarkMatterInertiaTensorReducedNoniterative": Property( + name="DarkMatterInertiaTensorReducedNoniterative", + shape=6, + dtype=np.float32, + unit="dimensionless", + description="Reduced 3D inertia tensor computed in a single interation from the DM mass distribution, relative to the halo centre. Diagonal components and one off-diagonal triangle as (1,1), (2,2), (3,3), (1,2), (1,3), (2,3). Only calculated when we have more than 20 particles.", + lossy_compression_filter="FMantissa9", + dmo_property=True, + particle_properties=["PartType1/Coordinates", "PartType1/Masses"], + output_physical=True, + a_scale_exponent=0, + ), + "DiffuseCarbonMass": Property( + name="DiffuseCarbonMass", + shape=1, + dtype=np.float32, + unit="snap_mass", + description="Total gas mass in carbon that is not contained in dust.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Masses", + "PartType0/ElementMassFractionsDiffuse", + ], + output_physical=True, + a_scale_exponent=0, + ), + "DiffuseIronMass": Property( + name="DiffuseIronMass", + shape=1, + dtype=np.float32, + unit="snap_mass", + description="Total gas mass in iron that is not contained in dust.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Masses", + "PartType0/ElementMassFractionsDiffuse", + ], + output_physical=True, + a_scale_exponent=0, + ), + "DiffuseMagnesiumMass": Property( + name="DiffuseMagnesiumMass", + shape=1, + dtype=np.float32, + unit="snap_mass", + description="Total gas mass in magnesium that is not contained in dust.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Masses", + "PartType0/ElementMassFractionsDiffuse", + ], + output_physical=True, + a_scale_exponent=0, + ), + "DiffuseOxygenMass": Property( + name="DiffuseOxygenMass", + shape=1, + dtype=np.float32, + unit="snap_mass", + description="Total gas mass in oxygen that is not contained in dust.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Masses", + "PartType0/ElementMassFractionsDiffuse", + ], + output_physical=True, + a_scale_exponent=0, + ), + "DiffuseSiliconMass": Property( + name="DiffuseSiliconMass", + shape=1, + dtype=np.float32, + unit="snap_mass", + description="Total gas mass in silicon that is not contained in dust.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Masses", + "PartType0/ElementMassFractionsDiffuse", + ], + output_physical=True, + a_scale_exponent=0, + ), + "DopplerB": Property( + name="DopplerB", + shape=1, + dtype=np.float32, + unit="dimensionless", + description="Kinetic Sunyaey-Zel'dovich effect, assuming a line of sight towards the position of the first lightcone observer.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Coordinates", + "PartType0/Velocities", + "PartType0/ElectronNumberDensities", + "PartType0/Densities", + ], + output_physical=False, + a_scale_exponent=1, + ), + "DtoTgas": Property( + name="DiscToTotalGasMassFraction", + shape=1, + dtype=np.float32, + unit="dimensionless", + description="Fraction of the total gas mass that is in the disc.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Coordinates", + "PartType0/Masses", + "PartType0/Velocities", + ], + output_physical=True, + a_scale_exponent=0, + ), + "DtoTstar": Property( + name="DiscToTotalStellarMassFraction", + shape=1, + dtype=np.float32, + unit="dimensionless", + description="Fraction of the total stellar mass that is in the disc.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType4/Coordinates", + "PartType4/Velocities", + "PartType4/Masses", + ], + output_physical=True, + a_scale_exponent=0, + ), + "DtoTstar_luminosity_weighted_luminosity_ratio": Property( + name="DiscToTotalLuminosityRatioLuminosityWeighted", + shape=9, + dtype=np.float32, + unit="dimensionless", + description="Fraction of the total stellar luminosity for each band that is in the disc. The band uses its own self-defined luminosity-weighted angular momentum", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType4/Coordinates", + "PartType4/Velocities", + "PartType4/Masses", + "PartType4/Luminosities", + ], + output_physical=True, + a_scale_exponent=0, + ), + "DustMass": Property( + name="DustMass", + shape=1, + dtype=np.float32, + unit="snap_mass", + description="Total dust mass.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Masses", + "PartType0/TotalDustMassFractions", + ], + output_physical=True, + a_scale_exponent=0, + ), + "DtoTstar_luminosity_weighted_mass_ratio": Property( + name="DiscToTotalMassRatioLuminosityWeighted", + shape=9, + dtype=np.float32, + unit="dimensionless", + description="Fraction of the total stellar mass that is in the disc. Each band uses its own self-defined luminosity-weighted angular momentum", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType4/Coordinates", + "PartType4/Velocities", + "PartType4/Masses", + "PartType4/Luminosities", + ], + output_physical=True, + a_scale_exponent=0, + ), + "kappa_corot_star_luminosity_weighted": Property( + name="KappaCorotStarsLuminosityWeighted", + shape=9, + dtype=np.float32, + unit="dimensionless", + description="Kappa-corot for stars, relative to the centre of potential and the centre of mass velocity of the stars. A value is output for each of the saved luminosity bands, as each is used to define the luminosity-weighted angular momentum.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType4/Coordinates", + "PartType4/Masses", + "PartType4/Velocities", + "PartType4/Luminosities", + ], + output_physical=True, + a_scale_exponent=0, + ), + "Lstar_luminosity_weighted": Property( + name="AngularMomentumStarsLuminosityWeighted", + shape=27, # 3D vector for each of the 9 GAMA bands + dtype=np.float32, + unit="snap_mass*snap_length**2/snap_time", + description="Luminosity-weighted total angular momentum of the stars, relative to the centre of potential and stellar centre of mass velocity. A different vector is computed for each of the saved luminosity bands.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType4/Coordinates", + "PartType4/Masses", + "PartType4/Velocities", + "PartType4/Luminosities", + ], + output_physical=True, + a_scale_exponent=1, + ), + "DustGraphiteMass": Property( + name="DustGraphiteMass", + shape=1, + dtype=np.float32, + unit="snap_mass", + description="Total dust mass in graphite grains.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType0/Masses", "PartType0/DustMassFractions"], + output_physical=True, + a_scale_exponent=0, + ), + "DustGraphiteMassInAtomicGas": Property( + name="DustGraphiteMassInAtomicGas", + shape=1, + dtype=np.float32, + unit="snap_mass", + description="Total dust mass in graphite grains in atomic gas (estimated from hydrogen).", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Masses", + "PartType0/DustMassFractions", + "PartType0/ElementMassFractions", + "PartType0/SpeciesFractions", + ], + output_physical=True, + a_scale_exponent=0, + ), + "DustGraphiteMassInMolecularGas": Property( + name="DustGraphiteMassInMolecularGas", + shape=1, + dtype=np.float32, + unit="snap_mass", + description="Total dust mass in graphite grains in molecular gas (estimated from hydrogen).", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Masses", + "PartType0/DustMassFractions", + "PartType0/SpeciesFractions", + "PartType0/ElementMassFractions", + ], + output_physical=True, + a_scale_exponent=0, + ), + "DustGraphiteMassInColdDenseGas": Property( + name="DustGraphiteMassInColdDenseGas", + shape=1, + dtype=np.float32, + unit="snap_mass", + description="Total dust mass in graphite grains in cold, dense gas.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Masses", + "PartType0/DustMassFractions", + "PartType0/Densities", + "PartType0/Temperatures", + ], + output_physical=True, + a_scale_exponent=0, + ), + "DustLargeGrainMass": Property( + name="DustLargeGrainMass", + shape=1, + dtype=np.float32, + unit="snap_mass", + description="Total dust mass in large grains.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType0/Masses", "PartType0/DustMassFractions"], + output_physical=True, + a_scale_exponent=0, + ), + "DustLargeGrainMassInMolecularGas": Property( + name="DustLargeGrainMassInMolecularGas", + shape=1, + dtype=np.float32, + unit="snap_mass", + description="Total dust mass in large grains in molecular gas (estimated from hydrogen).", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Masses", + "PartType0/DustMassFractions", + "PartType0/SpeciesFractions", + "PartType0/ElementMassFractions", + ], + output_physical=True, + a_scale_exponent=0, + ), + "DustLargeGrainMassInColdDenseGas": Property( + name="DustLargeGrainMassInColdDenseGas", + shape=1, + dtype=np.float32, + unit="snap_mass", + description="Total dust mass in large grains in cold, dense gas.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Masses", + "PartType0/DustMassFractions", + "PartType0/Densities", + "PartType0/Temperatures", + ], + output_physical=True, + a_scale_exponent=0, + ), + "DustLargeGrainMassSFRWeighted": Property( + name="DustLargeGrainMassSFRWeighted", + shape=1, + dtype=np.float32, + unit="snap_mass", + description="The dust mass in large grains, weighted by the SFR of the particles.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Masses", + "PartType0/DustMassFractions", + "PartType0/StarFormationRates", + ], + output_physical=True, + a_scale_exponent=0, + ), + "DustSilicatesMass": Property( + name="DustSilicatesMass", + shape=1, + dtype=np.float32, + unit="snap_mass", + description="Total dust mass in silicate grains.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType0/Masses", "PartType0/DustMassFractions"], + output_physical=True, + a_scale_exponent=0, + ), + "DustSilicatesMassInAtomicGas": Property( + name="DustSilicatesMassInAtomicGas", + shape=1, + dtype=np.float32, + unit="snap_mass", + description="Total dust mass in silicate grains in atomic gas (estimated from hydrogen).", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Masses", + "PartType0/DustMassFractions", + "PartType0/SpeciesFractions", + "PartType0/ElementMassFractions", + ], + output_physical=True, + a_scale_exponent=0, + ), + "DustSilicatesMassInMolecularGas": Property( + name="DustSilicatesMassInMolecularGas", + shape=1, + dtype=np.float32, + unit="snap_mass", + description="Total dust mass in silicate grains in molecular gas (estimated from hydrogen).", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Masses", + "PartType0/DustMassFractions", + "PartType0/SpeciesFractions", + "PartType0/ElementMassFractions", + ], + output_physical=True, + a_scale_exponent=0, + ), + "DustSilicatesMassInColdDenseGas": Property( + name="DustSilicatesMassInColdDenseGas", + shape=1, + dtype=np.float32, + unit="snap_mass", + description="Total dust mass in silicate grains in cold, dense gas.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Masses", + "PartType0/DustMassFractions", + "PartType0/Densities", + "PartType0/Temperatures", + ], + output_physical=True, + a_scale_exponent=0, + ), + "DustSmallGrainMass": Property( + name="DustSmallGrainMass", + shape=1, + dtype=np.float32, + unit="snap_mass", + description="Total dust mass in small grains.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Masses", + "PartType0/DustMassFractions", + "PartType0/ElementMassFractions", + ], + output_physical=True, + a_scale_exponent=0, + ), + "DustSmallGrainMassInMolecularGas": Property( + name="DustSmallGrainMassInMolecularGas", + shape=1, + dtype=np.float32, + unit="snap_mass", + description="Total dust mass in small grains in molecular gas (estimated from hydrogen).", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Masses", + "PartType0/DustMassFractions", + "PartType0/SpeciesFractions", + "PartType0/ElementMassFractions", + ], + output_physical=True, + a_scale_exponent=0, + ), + "DustSmallGrainMassInColdDenseGas": Property( + name="DustSmallGrainMassInColdDenseGas", + shape=1, + dtype=np.float32, + unit="snap_mass", + description="Total dust mass in small grains in cold, dense gas.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Masses", + "PartType0/DustMassFractions", + "PartType0/Densities", + "PartType0/Temperatures", + ], + output_physical=True, + a_scale_exponent=0, + ), + "DustSmallGrainMassSFRWeighted": Property( + name="DustSmallGrainMassSFRWeighted", + shape=1, + dtype=np.float32, + unit="snap_mass", + description="The dust mass in small grains, weighted by the SFR of the particles.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Masses", + "PartType0/DustMassFractions", + "PartType0/StarFormationRates", + ], + output_physical=True, + a_scale_exponent=0, + ), + "KineticEnergyGas": Property( + name="KineticEnergyGas", + shape=1, + dtype=np.float32, + unit="snap_mass*snap_length**2/snap_time**2", + description="Total kinetic energy of the gas, relative to the gas centre of mass velocity.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Masses", + "PartType0/Velocities", + "PartType0/Coordinates", + ], + output_physical=True, + a_scale_exponent=None, + ), + "KineticEnergyStars": Property( + name="KineticEnergyStars", + shape=1, + dtype=np.float32, + unit="snap_mass*snap_length**2/snap_time**2", + description="Total kinetic energy of the stars, relative to the stellar centre of mass velocity.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType4/Masses", + "PartType4/Velocities", + "PartType4/Coordinates", + ], + output_physical=True, + a_scale_exponent=None, + ), + "KineticEnergyTotal": Property( + name="KineticEnergyTotal", + shape=1, + dtype=np.float32, + unit="snap_mass*snap_length**2/snap_time**2", + description="Total kinetic energy of the particles, relative to the centre of mass velocity.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Masses", + "PartType0/Velocities", + "PartType0/Coordinates", + "PartType1/Masses", + "PartType1/Velocities", + "PartType1/Coordinates", + "PartType4/Masses", + "PartType4/Velocities", + "PartType4/Coordinates", + "PartType5/DynamicalMasses", + "PartType5/Velocities", + "PartType5/Coordinates", + ], + output_physical=True, + a_scale_exponent=None, + ), + "PotentialEnergyTotal": Property( + name="PotentialEnergyTotal", + shape=1, + dtype=np.float32, + unit="snap_mass*snap_length**2/snap_time**2", + description="Total potential energy of the subhalo.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/SpecificPotentialEnergies", + "PartType0/Masses", + "PartType1/SpecificPotentialEnergies", + "PartType1/Masses", + "PartType4/SpecificPotentialEnergies", + "PartType4/Masses", + "PartType5/SpecificPotentialEnergies", + "PartType5/DynamicalMasses", + ], + output_physical=True, + a_scale_exponent=None, + ), + "ThermalEnergyGas": Property( + name="ThermalEnergyGas", + shape=1, + dtype=np.float32, + unit="snap_mass*snap_length**2/snap_time**2", + description="Total thermal energy of the gas.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Densities", + "PartType0/Pressures", + "PartType0/Masses", + ], + output_physical=True, + a_scale_exponent=None, + ), + "GasInertiaTensor": Property( + name="GasInertiaTensor", + shape=6, + dtype=np.float32, + unit="snap_length**2", + description="3D inertia tensor computed iteratively from the gas mass distribution, relative to the halo centre. Diagonal components and one off-diagonal triangle as (1,1), (2,2), (3,3), (1,2), (1,3), (2,3). Only calculated when we have more than 20 particles.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType0/Coordinates", "PartType0/Masses"], + output_physical=True, + a_scale_exponent=2, + ), + "GasInertiaTensorReduced": Property( + name="GasInertiaTensorReduced", + shape=6, + dtype=np.float32, + unit="dimensionless", + description="Reduced 3D inertia tensor computed iteratively from the gas mass distribution, relative to the halo centre. Diagonal components and one off-diagonal triangle as (1,1), (2,2), (3,3), (1,2), (1,3), (2,3). Only calculated when we have more than 20 particles.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType0/Coordinates", "PartType0/Masses"], + output_physical=True, + a_scale_exponent=0, + ), + "GasInertiaTensorNoniterative": Property( + name="GasInertiaTensorNoniterative", + shape=6, + dtype=np.float32, + unit="snap_length**2", + description="3D inertia tensor computed in a single iteration from the gas mass distribution, relative to the halo centre. Diagonal components and one off-diagonal triangle as (1,1), (2,2), (3,3), (1,2), (1,3), (2,3). Only calculated when we have more than 20 particles.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType0/Coordinates", "PartType0/Masses"], + output_physical=True, + a_scale_exponent=2, + ), + "GasInertiaTensorReducedNoniterative": Property( + name="GasInertiaTensorReducedNoniterative", + shape=6, + dtype=np.float32, + unit="dimensionless", + description="Reduced 3D inertia tensor computed in a single iteration from the gas mass distribution, relative to the halo centre. Diagonal components and one off-diagonal triangle as (1,1), (2,2), (3,3), (1,2), (1,3), (2,3). Only calculated when we have more than 20 particles.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType0/Coordinates", "PartType0/Masses"], + output_physical=True, + a_scale_exponent=0, + ), + "HalfMassRadiusBaryon": Property( + name="HalfMassRadiusBaryons", + shape=1, + dtype=np.float32, + unit="snap_length", + description="Baryonic (gas and stars) half mass radius.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Coordinates", + "PartType0/Masses", + "PartType4/Coordinates", + "PartType4/Masses", + ], + output_physical=False, + a_scale_exponent=1, + ), + "HalfMassRadiusDM": Property( + name="HalfMassRadiusDarkMatter", + shape=1, + dtype=np.float32, + unit="snap_length", + description="Dark matter half mass radius.", + lossy_compression_filter="FMantissa9", + dmo_property=True, + particle_properties=["PartType1/Coordinates", "PartType1/Masses"], + output_physical=False, + a_scale_exponent=1, + ), + "HalfMassRadiusDust": Property( + name="HalfMassRadiusDust", + shape=1, + dtype=np.float32, + unit="snap_length", + description="Dust half mass radius.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Coordinates", + "PartType0/Masses", + "PartType0/TotalDustMassFractions", + ], + output_physical=False, + a_scale_exponent=1, + ), + "HalfMassRadiusAtomicHydrogen": Property( + name="HalfMassRadiusAtomicHydrogen", + shape=1, + dtype=np.float32, + unit="snap_length", + description="Atomic hydrogen half mass radius.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Coordinates", + "PartType0/Masses", + "PartType0/SpeciesFractions", + "PartType0/ElementMassFractions", + ], + output_physical=False, + a_scale_exponent=1, + ), + "HalfMassRadiusMolecularHydrogen": Property( + name="HalfMassRadiusMolecularHydrogen", + shape=1, + dtype=np.float32, + unit="snap_length", + description="Molecular hydrogen half mass radius.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Coordinates", + "PartType0/Masses", + "PartType0/SpeciesFractions", + "PartType0/ElementMassFractions", + ], + output_physical=False, + a_scale_exponent=1, + ), + "HalfMassRadiusGas": Property( + name="HalfMassRadiusGas", + shape=1, + dtype=np.float32, + unit="snap_length", + description="Gas half mass radius.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType0/Coordinates", "PartType0/Masses"], + output_physical=False, + a_scale_exponent=1, + ), + "HalfMassRadiusStar": Property( + name="HalfMassRadiusStars", + shape=1, + dtype=np.float32, + unit="snap_length", + description="Stellar half mass radius.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType4/Coordinates", "PartType4/Masses"], + output_physical=False, + a_scale_exponent=1, + ), + "HalfLightRadiusStar": Property( + name="HalfLightRadiusStars", + shape=9, + dtype=np.float32, + unit="snap_length", + description="Stellar half light radius in the 9 GAMA bands.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType4/Coordinates", + "PartType4/Masses", + "PartType4/Luminosities", + ], + output_physical=False, + a_scale_exponent=1, + ), + "HalfMassRadiusTot": Property( + name="HalfMassRadiusTotal", + shape=1, + dtype=np.float32, + unit="snap_length", + description="Total half mass radius.", + lossy_compression_filter="FMantissa9", + dmo_property=True, + particle_properties=[ + "PartType0/Coordinates", + "PartType0/Masses", + "PartType1/Coordinates", + "PartType1/Masses", + "PartType4/Coordinates", + "PartType4/Masses", + "PartType5/Coordinates", + "PartType5/DynamicalMasses", + ], + output_physical=False, + a_scale_exponent=1, + ), + "EncloseRadius": Property( + name="EncloseRadius", + shape=1, + dtype=np.float32, + unit="snap_length", + description="Radius of the particle furthest from the halo centre", + lossy_compression_filter="FMantissa9", + dmo_property=True, + particle_properties=[ + "PartType0/Coordinates", + "PartType1/Coordinates", + "PartType4/Coordinates", + "PartType5/Coordinates", + ], + output_physical=False, + a_scale_exponent=1, + ), + "HeliumMass": Property( + name="HeliumMass", + shape=1, + dtype=np.float32, + unit="snap_mass", + description="Total gas mass in helium.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType0/Masses", "PartType0/ElementMassFractions"], + output_physical=True, + a_scale_exponent=0, + ), + "HydrogenMass": Property( + name="HydrogenMass", + shape=1, + dtype=np.float32, + unit="snap_mass", + description="Total gas mass in hydrogen.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType0/Masses", "PartType0/ElementMassFractions"], + output_physical=True, + a_scale_exponent=0, + ), + "IonisedHydrogenMass": Property( + name="IonisedHydrogenMass", + shape=1, + dtype=np.float32, + unit="snap_mass", + description="Total gas mass in ionised hydrogen.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Masses", + "PartType0/SpeciesFractions", + "PartType0/ElementMassFractions", + ], + output_physical=True, + a_scale_exponent=0, + ), + "LastSupernovaEventMaximumGasDensity": Property( + name="LastSupernovaEventMaximumGasDensity", + shape=1, + dtype=np.float32, + unit="snap_mass/snap_length**3", + description="Maximum gas density at the last supernova event for the last supernova event of each gas particle.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/LastSNIIThermalFeedbackDensities", + "PartType0/LastSNIIKineticFeedbackDensities", + ], + output_physical=True, + a_scale_exponent=None, + ), + "Lbaryons": Property( + name="AngularMomentumBaryons", + shape=3, + dtype=np.float32, + unit="snap_mass*snap_length**2/snap_time", + description="Total angular momentum of baryons (gas and stars), relative to the HaloCentre and baryonic centre of mass velocity.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Coordinates", + "PartType0/Masses", + "PartType0/Velocities", + "PartType4/Coordinates", + "PartType4/Masses", + "PartType4/Velocities", + ], + output_physical=True, + a_scale_exponent=1, + ), + "Ldm": Property( + name="AngularMomentumDarkMatter", + shape=3, + dtype=np.float32, + unit="snap_mass*snap_length**2/snap_time", + description="Total angular momentum of the dark matter, relative to the HaloCentre and DM centre of mass velocity.", + lossy_compression_filter="FMantissa9", + dmo_property=True, + particle_properties=[ + "PartType1/Coordinates", + "PartType1/Masses", + "PartType1/Velocities", + ], + output_physical=True, + a_scale_exponent=1, + ), + "Lgas": Property( + name="AngularMomentumGas", + shape=3, + dtype=np.float32, + unit="snap_mass*snap_length**2/snap_time", + description="Total angular momentum of the gas, relative to the HaloCentre and gas centre of mass velocity.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Coordinates", + "PartType0/Masses", + "PartType0/Velocities", + ], + output_physical=True, + a_scale_exponent=1, + ), + "MedianStellarBirthDensity": Property( + name="MedianStellarBirthDensity", + shape=1, + dtype=np.float32, + unit="snap_mass/snap_length**3", + description="Median density of gas particles that were converted into a star particle.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType4/BirthDensities"], + output_physical=True, + a_scale_exponent=None, + ), + "MedianStellarBirthTemperature": Property( + name="MedianStellarBirthTemperature", + shape=1, + dtype=np.float32, + unit="snap_temperature", + description="Median temperature of gas particles that were converted into a star particle.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType4/BirthTemperatures"], + output_physical=True, + a_scale_exponent=None, + ), + "MedianStellarBirthPressure": Property( + name="MedianStellarBirthPressure", + shape=1, + dtype=np.float64, + unit="snap_temperature/snap_length**3", + description="Median pressure of gas particles that were converted into a star particle.", + lossy_compression_filter="DMantissa9", + dmo_property=False, + particle_properties=[ + "PartType4/BirthTemperatures", + "PartType4/BirthDensities", + ], + output_physical=True, + a_scale_exponent=None, + ), + "Lstar": Property( + name="AngularMomentumStars", + shape=3, + dtype=np.float32, + unit="snap_mass*snap_length**2/snap_time", + description="Total angular momentum of the stars, relative to the HaloCentre and stellar centre of mass velocity.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType4/Coordinates", + "PartType4/Masses", + "PartType4/Velocities", + ], + output_physical=True, + a_scale_exponent=1, + ), + "StellarRotationalVelocity": Property( + name="StellarRotationalVelocity", + shape=1, + dtype=np.float32, + unit="snap_length/snap_time", + description="Mass weighted mean rotational velocity of the stars, in a cylindrical coordinate system where the axes are centred on the stellar CoM, and the z axis is aligned with the stellar angular momentum.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType4/Coordinates", + "PartType4/Masses", + "PartType4/Velocities", + ], + output_physical=True, + a_scale_exponent=0, + ), + "StellarCylindricalVelocityDispersion": Property( + name="StellarCylindricalVelocityDispersion", + shape=1, + dtype=np.float32, + unit="snap_length/snap_time", + description="One-dimensional velocity dispersion of the stars computed in a cylindrical coordinate system where the axes are centred on the stellar CoM, and the z axis is aligned with the stellar angular momentum.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType4/Coordinates", + "PartType4/Masses", + "PartType4/Velocities", + ], + output_physical=True, + a_scale_exponent=0, + ), + "StellarCylindricalVelocityDispersionVertical": Property( + name="StellarCylindricalVelocityDispersionVertical", + shape=1, + dtype=np.float32, + unit="snap_length/snap_time", + description="Velocity dispersion perpendicular to the orbital plane of the stars, computed in a cylindrical coordinate system where the axes are centred on the stellar CoM, and the z axis is aligned with the stellar angular momentum.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType4/Coordinates", + "PartType4/Masses", + "PartType4/Velocities", + ], + output_physical=True, + a_scale_exponent=0, + ), + "StellarCylindricalVelocityDispersionDiscPlane": Property( + name="StellarCylindricalVelocityDispersionDiscPlane", + shape=1, + dtype=np.float32, + unit="snap_length/snap_time", + description="Total velocity dispersion in the orbital plane of the stars, computed in a cylindrical coordinate system where the axes are centred on the stellar CoM, and the z axis is aligned with the stellar angular momentum.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType4/Coordinates", + "PartType4/Masses", + "PartType4/Velocities", + ], + output_physical=True, + a_scale_exponent=0, + ), + "MaximumStellarBirthDensity": Property( + name="MaximumStellarBirthDensity", + shape=1, + dtype=np.float32, + unit="snap_mass/snap_length**3", + description="Maximum density of gas that was converted into a star particle.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType4/BirthDensities"], + output_physical=True, + a_scale_exponent=None, + ), + "MaximumStellarBirthTemperature": Property( + name="MaximumStellarBirthTemperature", + shape=1, + dtype=np.float32, + unit="snap_temperature", + description="Maximum temperature of gas that was converted into a star particle.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType4/BirthTemperatures"], + output_physical=True, + a_scale_exponent=None, + ), + "MaximumStellarBirthPressure": Property( + name="MaximumStellarBirthPressure", + shape=1, + dtype=np.float64, + unit="snap_temperature/snap_length**3", + description="Maximum pressure of gas that was converted into a star particle.", + lossy_compression_filter="DMantissa9", + dmo_property=False, + particle_properties=[ + "PartType4/BirthTemperatures", + "PartType4/BirthDensities", + ], + output_physical=True, + a_scale_exponent=None, + ), + "Mbh_dynamical": Property( + name="BlackHolesDynamicalMass", + shape=1, + dtype=np.float32, + unit="snap_mass", + description="Total BH dynamical mass.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType5/DynamicalMasses"], + output_physical=True, + a_scale_exponent=0, + ), + "Mbh_subgrid": Property( + name="BlackHolesSubgridMass", + shape=1, + dtype=np.float32, + unit="snap_mass", + description="Total BH subgrid mass.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType5/SubgridMasses"], + output_physical=True, + a_scale_exponent=0, + ), + "Mdm": Property( + name="DarkMatterMass", + shape=1, + dtype=np.float32, + unit="snap_mass", + description="Total DM mass.", + lossy_compression_filter="FMantissa9", + dmo_property=True, + particle_properties=["PartType1/Masses"], + output_physical=True, + a_scale_exponent=0, + ), + "Mfrac_satellites": Property( + name="MassFractionSatellites", + shape=1, + dtype=np.float32, + unit="dimensionless", + description="Fraction of mass that is bound to a satellite in the same FOF group.", + lossy_compression_filter="FMantissa9", + dmo_property=True, + particle_properties=[ + "PartType0/Masses", + "PartType1/Masses", + "PartType4/Masses", + "PartType5/DynamicalMasses", + "PartType0/FOFGroupIDs", + "PartType1/FOFGroupIDs", + "PartType4/FOFGroupIDs", + "PartType5/FOFGroupIDs", + "PartType0/GroupNr_bound", + "PartType1/GroupNr_bound", + "PartType4/GroupNr_bound", + "PartType5/GroupNr_bound", + ], + output_physical=True, + a_scale_exponent=0, + ), + "Mfrac_external": Property( + name="MassFractionExternal", + shape=1, + dtype=np.float32, + unit="dimensionless", + description="Fraction of mass that is bound to a satellite outside this FOF group.", + lossy_compression_filter="FMantissa9", + dmo_property=True, + particle_properties=[ + "PartType0/Masses", + "PartType1/Masses", + "PartType4/Masses", + "PartType5/DynamicalMasses", + "PartType0/FOFGroupIDs", + "PartType1/FOFGroupIDs", + "PartType4/FOFGroupIDs", + "PartType5/FOFGroupIDs", + "PartType0/GroupNr_bound", + "PartType1/GroupNr_bound", + "PartType4/GroupNr_bound", + "PartType5/GroupNr_bound", + ], + output_physical=True, + a_scale_exponent=0, + ), + "Mgas": Property( + name="GasMass", + shape=1, + dtype=np.float32, + unit="snap_mass", + description="Total gas mass.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType0/Masses"], + output_physical=True, + a_scale_exponent=0, + ), + "Mgas_SF": Property( + name="StarFormingGasMass", + shape=1, + dtype=np.float32, + unit="snap_mass", + description="Total mass of star-forming gas.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType0/Masses", "PartType0/StarFormationRates"], + output_physical=True, + a_scale_exponent=0, + ), + "Mhotgas": Property( + name="HotGasMass", + shape=1, + dtype=np.float32, + unit="snap_mass", + description="Total mass of gas with a temperature above 1e5 K.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType0/Masses", "PartType0/Temperatures"], + output_physical=True, + a_scale_exponent=0, + ), + "GasMassInColdDenseGas": Property( + name="GasMassInColdDenseGas", + shape=1, + dtype=np.float32, + unit="snap_mass", + description="Total mass of gas in cold, dense gas.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Masses", + "PartType0/Densities", + "PartType0/Temperatures", + ], + output_physical=True, + a_scale_exponent=0, + ), + "MinimumStellarBirthDensity": Property( + name="MinimumStellarBirthDensity", + shape=1, + dtype=np.float32, + unit="snap_mass/snap_length**3", + description="Minimum density of gas that was converted into a star particle.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType4/BirthDensities"], + output_physical=True, + a_scale_exponent=None, + ), + "MinimumStellarBirthTemperature": Property( + name="MinimumStellarBirthTemperature", + shape=1, + dtype=np.float32, + unit="snap_temperature", + description="Minimum temperature of gas that was converted into a star particle.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType4/BirthTemperatures"], + output_physical=True, + a_scale_exponent=None, + ), + "MinimumStellarBirthPressure": Property( + name="MinimumStellarBirthPressure", + shape=1, + dtype=np.float64, + unit="snap_temperature/snap_length**3", + description="Minimum pressure of gas that was converted into a star particle.", + lossy_compression_filter="DMantissa9", + dmo_property=False, + particle_properties=[ + "PartType4/BirthTemperatures", + "PartType4/BirthDensities", + ], + output_physical=True, + a_scale_exponent=None, + ), + "Mnu": Property( + name="RawNeutrinoMass", + shape=1, + dtype=np.float32, + unit="snap_mass", + description="Total neutrino particle mass.", + lossy_compression_filter="FMantissa9", + dmo_property=True, + particle_properties=["PartType6/Masses"], + output_physical=True, + a_scale_exponent=0, + ), + "MnuNS": Property( + name="NoiseSuppressedNeutrinoMass", + shape=1, + dtype=np.float32, + unit="snap_mass", + description="Noise suppressed total neutrino mass.", + lossy_compression_filter="FMantissa9", + dmo_property=True, + particle_properties=["PartType6/Masses", "PartType6/Weights"], + output_physical=True, + a_scale_exponent=0, + ), + "MolecularHydrogenMass": Property( + name="MolecularHydrogenMass", + shape=1, + dtype=np.float32, + unit="snap_mass", + description="Total gas mass in molecular hydrogen.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Masses", + "PartType0/SpeciesFractions", + "PartType0/ElementMassFractions", + ], + output_physical=True, + a_scale_exponent=0, + ), + "Mstar": Property( + name="StellarMass", + shape=1, + dtype=np.float32, + unit="snap_mass", + description="Total stellar mass.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType4/Masses"], + output_physical=True, + a_scale_exponent=0, + ), + "Mstar_init": Property( + name="StellarInitialMass", + shape=1, + dtype=np.float32, + unit="snap_mass", + description="Total stellar initial mass.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType4/InitialMasses"], + output_physical=True, + a_scale_exponent=0, + ), + "Mtot": Property( + name="TotalMass", + shape=1, + dtype=np.float32, + unit="snap_mass", + description="Total mass.", + lossy_compression_filter="FMantissa9", + dmo_property=True, + particle_properties=[ + "PartType0/Masses", + "PartType1/Masses", + "PartType4/Masses", + "PartType5/DynamicalMasses", + ], + output_physical=True, + a_scale_exponent=0, + ), + "Nbh": Property( + name="NumberOfBlackHoleParticles", + shape=1, + dtype=np.uint32, + unit="dimensionless", + description="Number of black hole particles.", + lossy_compression_filter="None", + dmo_property=False, + particle_properties=[], + output_physical=True, + a_scale_exponent=None, + ), + "Ndm": Property( + name="NumberOfDarkMatterParticles", + shape=1, + dtype=np.uint32, + unit="dimensionless", + description="Number of dark matter particles.", + lossy_compression_filter="None", + dmo_property=True, + particle_properties=[], + output_physical=True, + a_scale_exponent=None, + ), + "Ngas": Property( + name="NumberOfGasParticles", + shape=1, + dtype=np.uint32, + unit="dimensionless", + description="Number of gas particles.", + lossy_compression_filter="None", + dmo_property=False, + particle_properties=[], + output_physical=True, + a_scale_exponent=None, + ), + "Nnu": Property( + name="NumberOfNeutrinoParticles", + shape=1, + dtype=np.uint32, + unit="dimensionless", + description="Number of neutrino particles.", + lossy_compression_filter="None", + dmo_property=False, + particle_properties=[], + output_physical=True, + a_scale_exponent=None, + ), + "Nstar": Property( + name="NumberOfStarParticles", + shape=1, + dtype=np.uint32, + unit="dimensionless", + description="Number of star particles.", + lossy_compression_filter="None", + dmo_property=False, + particle_properties=[], + output_physical=True, + a_scale_exponent=None, + ), + "ProjectedTotalInertiaTensor": Property( + name="ProjectedTotalInertiaTensor", + shape=3, + dtype=np.float32, + unit="snap_length**2", + description="2D inertia tensor computed iteratively from the total mass distribution, relative to the halo centre. Diagonal components and one off-diagonal value as (1,1), (2,2), (1,2). Only calculated when we have more than 20 particles.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Coordinates", + "PartType0/Masses", + "PartType1/Coordinates", + "PartType1/Masses", + "PartType4/Coordinates", + "PartType4/Masses", + "PartType5/Coordinates", + "PartType5/DynamicalMasses", + ], + output_physical=True, + a_scale_exponent=2, + ), + "ProjectedTotalInertiaTensorReduced": Property( + name="ProjectedTotalInertiaTensorReduced", + shape=3, + dtype=np.float32, + unit="dimensionless", + description="Reduced 2D inertia tensor computed iteratively from the total mass distribution, relative to the halo centre. Diagonal components and one off-diagonal value as (1,1), (2,2), (1,2). Only calculated when we have more than 20 particles.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Coordinates", + "PartType0/Masses", + "PartType1/Coordinates", + "PartType1/Masses", + "PartType4/Coordinates", + "PartType4/Masses", + "PartType5/Coordinates", + "PartType5/DynamicalMasses", + ], + output_physical=True, + a_scale_exponent=0, + ), + "ProjectedTotalInertiaTensorNoniterative": Property( + name="ProjectedTotalInertiaTensorNoniterative", + shape=3, + dtype=np.float32, + unit="snap_length**2", + description="2D inertia tensor computed in a single iteration from the total mass distribution, relative to the halo centre. Diagonal components and one off-diagonal value as (1,1), (2,2), (1,2). Only calculated when we have more than 20 particles.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Coordinates", + "PartType0/Masses", + "PartType1/Coordinates", + "PartType1/Masses", + "PartType4/Coordinates", + "PartType4/Masses", + "PartType5/Coordinates", + "PartType5/DynamicalMasses", + ], + output_physical=True, + a_scale_exponent=2, + ), + "ProjectedTotalInertiaTensorReducedNoniterative": Property( + name="ProjectedTotalInertiaTensorReducedNoniterative", + shape=3, + dtype=np.float32, + unit="dimensionless", + description="Reduced 2D inertia tensor computed in a single iteration from the total mass distribution, relative to the halo centre. Diagonal components and one off-diagonal value as (1,1), (2,2), (1,2). Only calculated when we have more than 20 particles.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Coordinates", + "PartType0/Masses", + "PartType1/Coordinates", + "PartType1/Masses", + "PartType4/Coordinates", + "PartType4/Masses", + "PartType5/Coordinates", + "PartType5/DynamicalMasses", + ], + output_physical=True, + a_scale_exponent=0, + ), + "ProjectedGasInertiaTensor": Property( + name="ProjectedGasInertiaTensor", + shape=3, + dtype=np.float32, + unit="snap_length**2", + description="2D inertia tensor computed iteratively from the gas mass distribution, relative to the halo centre. Diagonal components and one off-diagonal value as (1,1), (2,2), (1,2). Only calculated when we have more than 20 particles.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType0/Coordinates", "PartType0/Masses"], + output_physical=True, + a_scale_exponent=2, + ), + "ProjectedGasInertiaTensorReduced": Property( + name="ProjectedGasInertiaTensorReduced", + shape=3, + dtype=np.float32, + unit="dimensionless", + description="Reduced 2D inertia tensor computed iteratively from the gas mass distribution, relative to the halo centre. Diagonal components and one off-diagonal value as (1,1), (2,2), (1,2). Only calculated when we have more than 20 particles.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType0/Coordinates", "PartType0/Masses"], + output_physical=True, + a_scale_exponent=0, + ), + "ProjectedGasInertiaTensorNoniterative": Property( + name="ProjectedGasInertiaTensorNoniterative", + shape=3, + dtype=np.float32, + unit="snap_length**2", + description="2D inertia tensor computed in a single iteration from the gas mass distribution, relative to the halo centre. Diagonal components and one off-diagonal value as (1,1), (2,2), (1,2). Only calculated when we have more than 20 particles.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType0/Coordinates", "PartType0/Masses"], + output_physical=True, + a_scale_exponent=2, + ), + "ProjectedGasInertiaTensorReducedNoniterative": Property( + name="ProjectedGasInertiaTensorReducedNoniterative", + shape=3, + dtype=np.float32, + unit="dimensionless", + description="Reduced 2D inertia tensor computed in a single iteration from the gas mass distribution, relative to the halo centre. Diagonal components and one off-diagonal value as (1,1), (2,2), (1,2). Only calculated when we have more than 20 particles.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType0/Coordinates", "PartType0/Masses"], + output_physical=True, + a_scale_exponent=0, + ), + "ProjectedStellarInertiaTensor": Property( + name="ProjectedStellarInertiaTensor", + shape=3, + dtype=np.float32, + unit="snap_length**2", + description="2D inertia tensor computed iteratively from the stellar mass distribution, relative to the halo centre. Diagonal components and one off-diagonal value as (1,1), (2,2), (1,2). Only calculated when we have more than 20 particles.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType4/Coordinates", "PartType4/Masses"], + output_physical=True, + a_scale_exponent=2, + ), + "ProjectedStellarInertiaTensorReduced": Property( + name="ProjectedStellarInertiaTensorReduced", + shape=3, + dtype=np.float32, + unit="dimensionless", + description="Reduced 2D inertia tensor computed iteratively from the stellar mass distribution, relative to the halo centre. Diagonal components and one off-diagonal value as (1,1), (2,2), (1,2). Only calculated when we have more than 20 particles.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType4/Coordinates", "PartType4/Masses"], + output_physical=True, + a_scale_exponent=0, + ), + "ProjectedStellarInertiaTensorNoniterative": Property( + name="ProjectedStellarInertiaTensorNoniterative", + shape=3, + dtype=np.float32, + unit="snap_length**2", + description="2D inertia tensor computed in a single iteration from the stellar mass distribution, relative to the halo centre. Diagonal components and one off-diagonal value as (1,1), (2,2), (1,2). Only calculated when we have more than 20 particles.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType4/Coordinates", "PartType4/Masses"], + output_physical=True, + a_scale_exponent=2, + ), + "ProjectedStellarInertiaTensorReducedNoniterative": Property( + name="ProjectedStellarInertiaTensorReducedNoniterative", + shape=3, + dtype=np.float32, + unit="dimensionless", + description="Reduced 2D inertia tensor computed in a single iteration from the stellar mass distribution, relative to the halo centre. Diagonal components and one off-diagonal value as (1,1), (2,2), (1,2). Only calculated when we have more than 20 particles.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType4/Coordinates", "PartType4/Masses"], + output_physical=True, + a_scale_exponent=0, + ), + "ProjectedStellarInertiaTensorLuminosityWeighted": Property( + name="ProjectedStellarInertiaTensorLuminosityWeighted", + shape=27, + dtype=np.float32, + unit="snap_length**2", + description="2D inertia tensor computed iteratively from the stellar luminosity distribution in different GAMA bands, relative to the halo centre. Diagonal components and one off-diagonal value as (1,1), (2,2), (1,2). Only calculated when we have more than 20 particles.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType4/Coordinates", + "PartType4/Masses", + "PartType4/Luminosities", + ], + output_physical=True, + a_scale_exponent=2, + ), + "ProjectedStellarInertiaTensorReducedLuminosityWeighted": Property( + name="ProjectedStellarInertiaTensorReducedLuminosityWeighted", + shape=27, + dtype=np.float32, + unit="dimensionless", + description="Reduced 2D inertia tensor computed iteratively from the stellar luminosity distribution in different GAMA bands, relative to the halo centre. Diagonal components and one off-diagonal value as (1,1), (2,2), (1,2). Only calculated when we have more than 20 particles.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType4/Coordinates", + "PartType4/Masses", + "PartType4/Luminosities", + ], + output_physical=True, + a_scale_exponent=0, + ), + "ProjectedStellarInertiaTensorNoniterativeLuminosityWeighted": Property( + name="ProjectedStellarInertiaTensorNoniterativeLuminosityWeighted", + shape=27, + dtype=np.float32, + unit="snap_length**2", + description="2D inertia tensor computed in a single iteration from the stellar luminosity distribution in different GAMA bands, relative to the halo centre. Diagonal components and one off-diagonal value as (1,1), (2,2), (1,2). Only calculated when we have more than 20 particles.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType4/Coordinates", + "PartType4/Masses", + "PartType4/Luminosities", + ], + output_physical=True, + a_scale_exponent=2, + ), + "ProjectedStellarInertiaTensorReducedNoniterativeLuminosityWeighted": Property( + name="ProjectedStellarInertiaTensorReducedNoniterativeLuminosityWeighted", + shape=27, + dtype=np.float32, + unit="dimensionless", + description="Reduced 2D inertia tensor computed in a single iteration from the stellar luminosity distribution in different GAMA bands, relative to the halo centre. Diagonal components and one off-diagonal value as (1,1), (2,2), (1,2). Only calculated when we have more than 20 particles.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType4/Coordinates", + "PartType4/Masses", + "PartType4/Luminosities", + ], + output_physical=True, + a_scale_exponent=0, + ), + "SFR": Property( + name="StarFormationRate", + shape=1, + dtype=np.float32, + unit="snap_mass/snap_time", + description="Total star formation rate.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType0/StarFormationRates"], + output_physical=True, + a_scale_exponent=None, + ), + "AveragedStarFormationRate": Property( + name="AveragedStarFormationRate", + shape=2, + dtype=np.float32, + unit="snap_mass/snap_time", + description="Total star formation rate, averaged over past 100Myr and past 10Myr. If the time between this snapshot and the previous one was less than the averaging time, then the value is averaged over the time between the snapshots.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType0/AveragedStarFormationRates"], + output_physical=True, + a_scale_exponent=None, + ), + "StellarInertiaTensor": Property( + name="StellarInertiaTensor", + shape=6, + dtype=np.float32, + unit="snap_length**2", + description="3D inertia tensor computed iteratively from the stellar mass distribution, relative to the halo centre. Diagonal components and one off-diagonal triangle as (1,1), (2,2), (3,3), (1,2), (1,3), (2,3). Only calculated when we have more than 20 particles.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType4/Coordinates", "PartType4/Masses"], + output_physical=True, + a_scale_exponent=2, + ), + "StellarInertiaTensorReduced": Property( + name="StellarInertiaTensorReduced", + shape=6, + dtype=np.float32, + unit="dimensionless", + description="Reduced 3D inertia tensor computed iteratively from the stellar mass distribution, relative to the halo centre. Diagonal components and one off-diagonal triangle as (1,1), (2,2), (3,3), (1,2), (1,3), (2,3). Only calculated when we have more than 20 particles.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType4/Coordinates", "PartType4/Masses"], + output_physical=True, + a_scale_exponent=0, + ), + "StellarInertiaTensorNoniterative": Property( + name="StellarInertiaTensorNoniterative", + shape=6, + dtype=np.float32, + unit="snap_length**2", + description="3D inertia tensor computed in a single iteration from the stellar mass distribution, relative to the halo centre. Diagonal components and one off-diagonal triangle as (1,1), (2,2), (3,3), (1,2), (1,3), (2,3). Only calculated when we have more than 20 particles.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType4/Coordinates", "PartType4/Masses"], + output_physical=True, + a_scale_exponent=2, + ), + "StellarInertiaTensorReducedNoniterative": Property( + name="StellarInertiaTensorReducedNoniterative", + shape=6, + dtype=np.float32, + unit="dimensionless", + description="Reduced 3D inertia tensor computed in a single iteration from the stellar mass distribution, relative to the halo centre. Diagonal components and one off-diagonal triangle as (1,1), (2,2), (3,3), (1,2), (1,3), (2,3). Only calculated when we have more than 20 particles.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType4/Coordinates", "PartType4/Masses"], + output_physical=True, + a_scale_exponent=0, + ), + "StellarInertiaTensorLuminosityWeighted": Property( + name="StellarInertiaTensorLuminosityWeighted", + shape=54, # 6 times each GAMA band + dtype=np.float32, + unit="snap_length**2", + description="3D inertia tensor computed iteratively from the stellar luminosity distribution, relative to the halo centre and for each GAMA band. Diagonal components and one off-diagonal triangle as (1,1), (2,2), (3,3), (1,2), (1,3), (2,3). Only calculated when we have more than 20 particles.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType4/Coordinates", + "PartType4/Masses", + "PartType4/Luminosities", + ], + output_physical=True, + a_scale_exponent=2, + ), + "StellarInertiaTensorReducedLuminosityWeighted": Property( + name="StellarInertiaTensorReducedLuminosityWeighted", + shape=54, # 6 times each GAMA band + dtype=np.float32, + unit="dimensionless", + description="Reduced 3D inertia tensor computed iteratively from the stellar luminosity distribution, relative to the halo centre and for each GAMA band. Diagonal components and one off-diagonal triangle as (1,1), (2,2), (3,3), (1,2), (1,3), (2,3). Only calculated when we have more than 20 particles.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType4/Coordinates", + "PartType4/Masses", + "PartType4/Luminosities", + ], + output_physical=True, + a_scale_exponent=0, + ), + "StellarInertiaTensorNoniterativeLuminosityWeighted": Property( + name="StellarInertiaTensorNoniterativeLuminosityWeighted", + shape=54, # 6 times each GAMA band + dtype=np.float32, + unit="snap_length**2", + description="3D inertia tensor computed in a single iteration from the stellar luminosity distribution, relative to the halo centre and for each GAMA band. Diagonal components and one off-diagonal triangle as (1,1), (2,2), (3,3), (1,2), (1,3), (2,3). Only calculated when we have more than 20 particles.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType4/Coordinates", + "PartType4/Masses", + "PartType4/Luminosities", + ], + output_physical=True, + a_scale_exponent=2, + ), + "StellarInertiaTensorReducedNoniterativeLuminosityWeighted": Property( + name="StellarInertiaTensorReducedNoniterativeLuminosityWeighted", + shape=54, # 6 times each GAMA band + dtype=np.float32, + unit="dimensionless", + description="Reduced 3D inertia tensor computed in a single iteration from the stellar luminosity distribution, relative to the halo centre and for each GAMA band. Diagonal components and one off-diagonal triangle as (1,1), (2,2), (3,3), (1,2), (1,3), (2,3). Only calculated when we have more than 20 particles.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType4/Coordinates", + "PartType4/Masses", + "PartType4/Luminosities", + ], + output_physical=True, + a_scale_exponent=0, + ), + "StellarLuminosity": Property( + name="StellarLuminosity", + shape=9, + dtype=np.float32, + unit="dimensionless", + description="Total stellar luminosity in the 9 GAMA bands.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType4/Luminosities"], + output_physical=True, + a_scale_exponent=None, + ), + "Tgas": Property( + name="GasTemperature", + shape=1, + dtype=np.float32, + unit="snap_temperature", + description="Mass-weighted mean gas temperature.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType0/Temperatures"], + output_physical=True, + a_scale_exponent=0, + ), + "Tgas_no_agn": Property( + name="GasTemperatureWithoutRecentAGNHeating", + shape=1, + dtype=np.float32, + unit="snap_temperature", + description="Mass-weighted mean gas temperature, excluding gas that was recently heated by AGN.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Temperatures", + "PartType0/LastAGNFeedbackScaleFactors", + ], + output_physical=True, + a_scale_exponent=0, + ), + "Tgas_no_cool": Property( + name="GasTemperatureWithoutCoolGas", + shape=1, + dtype=np.float32, + unit="snap_temperature", + description="Mass-weighted mean gas temperature, excluding cool gas with a temperature below 1e5 K.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType0/Temperatures"], + output_physical=True, + a_scale_exponent=0, + ), + "Tgas_no_cool_no_agn": Property( + name="GasTemperatureWithoutCoolGasAndRecentAGNHeating", + shape=1, + dtype=np.float32, + unit="snap_temperature", + description="Mass-weighted mean gas temperature, excluding cool gas with a temperature below 1e5 K and gas that was recently heated by AGN.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Temperatures", + "PartType0/LastAGNFeedbackScaleFactors", + ], + output_physical=True, + a_scale_exponent=0, + ), + "Tgas_cy_weighted": Property( + name="GasComptonYTemperature", + shape=1, + dtype=np.float32, + unit="snap_temperature", + description="ComptonY-weighted mean gas temperature.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Temperatures", + "PartType0/ComptonYParameters", + ], + output_physical=True, + a_scale_exponent=0, + ), + "Tgas_cy_weighted_no_agn": Property( + name="GasComptonYTemperatureWithoutRecentAGNHeating", + shape=1, + dtype=np.float32, + unit="snap_temperature", + description="ComptonY-weighted mean gas temperature, excluding gas that was recently heated by AGN.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Temperatures", + "PartType0/ComptonYParameters", + "PartType0/LastAGNFeedbackScaleFactors", + ], + output_physical=True, + a_scale_exponent=0, + ), + "Tgas_cy_weighted_core_excision": Property( + name="GasComptonYTemperatureCoreExcision", + shape=1, + dtype=np.float32, + unit="snap_temperature", + description="ComptonY-weighted mean gas temperature, excluding the inner {core_excision}.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Temperatures", + "PartType0/ComptonYParameters", + "PartType0/Coordinates", + ], + output_physical=True, + a_scale_exponent=0, + ), + "Tgas_cy_weighted_core_excision_no_agn": Property( + name="GasComptonYTemperatureWithoutRecentAGNHeatingCoreExcision", + shape=1, + dtype=np.float32, + unit="snap_temperature", + description="ComptonY-weighted mean gas temperature, excluding the inner {core_excision} and gas that was recently heated by AGN.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Temperatures", + "PartType0/ComptonYParameters", + "PartType0/Coordinates", + "PartType0/LastAGNFeedbackScaleFactors", + ], + output_physical=True, + a_scale_exponent=0, + ), + "Tgas_core_excision": Property( + name="GasTemperatureCoreExcision", + shape=1, + dtype=np.float32, + unit="snap_temperature", + description="Mass-weighted mean gas temperature, excluding the inner {core_excision}.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Temperatures", + "PartType0/Masses", + "PartType0/Coordinates", + ], + output_physical=True, + a_scale_exponent=0, + ), + "Tgas_no_cool_core_excision": Property( + name="GasTemperatureWithoutCoolGasCoreExcision", + shape=1, + dtype=np.float32, + unit="snap_temperature", + description="Mass-weighted mean gas temperature, excluding the inner {core_excision} and gas below 1e5 K.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Temperatures", + "PartType0/Masses", + "PartType0/Coordinates", + ], + output_physical=True, + a_scale_exponent=0, + ), + "Tgas_no_agn_core_excision": Property( + name="GasTemperatureWithoutRecentAGNHeatingCoreExcision", + shape=1, + dtype=np.float32, + unit="snap_temperature", + description="Mass-weighted mean gas temperature, excluding the inner {core_excision}, and gas that was recently heated by AGN.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Temperatures", + "PartType0/Masses", + "PartType0/Coordinates", + "PartType0/LastAGNFeedbackScaleFactors", + ], + output_physical=True, + a_scale_exponent=0, + ), + "Tgas_no_cool_no_agn_core_excision": Property( + name="GasTemperatureWithoutCoolGasAndRecentAGNHeatingCoreExcision", + shape=1, + dtype=np.float32, + unit="snap_temperature", + description="Mass-weighted mean gas temperature, excluding the inner {core_excision}, gas below 1e5 K and gas that was recently heated by AGN.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Temperatures", + "PartType0/Masses", + "PartType0/Coordinates", + "PartType0/LastAGNFeedbackScaleFactors", + ], + output_physical=True, + a_scale_exponent=0, + ), + "TotalInertiaTensor": Property( + name="TotalInertiaTensor", + shape=6, + dtype=np.float32, + unit="snap_length**2", + description="3D inertia tensor computed iteratively from the total mass distribution, relative to the halo centre. Diagonal components and one off-diagonal triangle as (1,1), (2,2), (3,3), (1,2), (1,3), (2,3). Only calculated when we have more than 20 particles.", + lossy_compression_filter="FMantissa9", + dmo_property=True, + particle_properties=[ + "PartType0/Coordinates", + "PartType0/Masses", + "PartType1/Coordinates", + "PartType1/Masses", + "PartType4/Coordinates", + "PartType4/Masses", + "PartType5/Coordinates", + "PartType5/DynamicalMasses", + ], + output_physical=True, + a_scale_exponent=2, + ), + "TotalInertiaTensorReduced": Property( + name="TotalInertiaTensorReduced", + shape=6, + dtype=np.float32, + unit="dimensionless", + description="Reduced 3D inertia tensor computed iteratively from the total mass distribution, relative to the halo centre. Diagonal components and one off-diagonal triangle as (1,1), (2,2), (3,3), (1,2), (1,3), (2,3). Only calculated when we have more than 20 particles.", + lossy_compression_filter="FMantissa9", + dmo_property=True, + particle_properties=[ + "PartType0/Coordinates", + "PartType0/Masses", + "PartType1/Coordinates", + "PartType1/Masses", + "PartType4/Coordinates", + "PartType4/Masses", + "PartType5/Coordinates", + "PartType5/DynamicalMasses", + ], + output_physical=True, + a_scale_exponent=0, + ), + "TotalInertiaTensorNoniterative": Property( + name="TotalInertiaTensorNoniterative", + shape=6, + dtype=np.float32, + unit="snap_length**2", + description="3D inertia tensor computed in a single iteration from the total mass distribution, relative to the halo centre. Diagonal components and one off-diagonal triangle as (1,1), (2,2), (3,3), (1,2), (1,3), (2,3). Only calculated when we have more than 20 particles.", + lossy_compression_filter="FMantissa9", + dmo_property=True, + particle_properties=[ + "PartType0/Coordinates", + "PartType0/Masses", + "PartType1/Coordinates", + "PartType1/Masses", + "PartType4/Coordinates", + "PartType4/Masses", + "PartType5/Coordinates", + "PartType5/DynamicalMasses", + ], + output_physical=True, + a_scale_exponent=2, + ), + "TotalInertiaTensorReducedNoniterative": Property( + name="TotalInertiaTensorReducedNoniterative", + shape=6, + dtype=np.float32, + unit="dimensionless", + description="Reduced 3D inertia tensor computed in a single iteration from the total mass distribution, relative to the halo centre. Diagonal components and one off-diagonal triangle as (1,1), (2,2), (3,3), (1,2), (1,3), (2,3). Only calculated when we have more than 20 particles.", + lossy_compression_filter="FMantissa9", + dmo_property=True, + particle_properties=[ + "PartType0/Coordinates", + "PartType0/Masses", + "PartType1/Coordinates", + "PartType1/Masses", + "PartType4/Coordinates", + "PartType4/Masses", + "PartType5/Coordinates", + "PartType5/DynamicalMasses", + ], + output_physical=True, + a_scale_exponent=0, + ), + "TotalSNIaRate": Property( + name="TotalSNIaRate", + shape=1, + dtype=np.float32, + unit="1/snap_time", + description="Total SNIa rate.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType4/SNIaRates"], + output_physical=True, + a_scale_exponent=None, + ), + "R_vmax_unsoft": Property( + name="MaximumCircularVelocityRadiusUnsoftened", + shape=1, + dtype=np.float32, + unit="snap_length", + description="Radius at which MaximumCircularVelocityUnsoftened is reached.", + lossy_compression_filter="FMantissa9", + dmo_property=True, + particle_properties=[ + "PartType0/Coordinates", + "PartType0/Masses", + "PartType1/Coordinates", + "PartType1/Masses", + "PartType4/Coordinates", + "PartType4/Masses", + "PartType5/Coordinates", + "PartType5/DynamicalMasses", + ], + output_physical=False, + a_scale_exponent=1, + ), + "Vmax_unsoft": Property( + name="MaximumCircularVelocityUnsoftened", + shape=1, + dtype=np.float32, + unit="snap_length/snap_time", + description="Maximum circular velocity when not accounting for particle softening lengths.", + lossy_compression_filter="FMantissa9", + dmo_property=True, + particle_properties=[ + "PartType0/Coordinates", + "PartType0/Masses", + "PartType1/Coordinates", + "PartType1/Masses", + "PartType4/Coordinates", + "PartType4/Masses", + "PartType5/Coordinates", + "PartType5/DynamicalMasses", + ], + output_physical=True, + a_scale_exponent=0, + ), + "R_vmax_soft": Property( + name="MaximumCircularVelocityRadius", + shape=1, + dtype=np.float32, + unit="snap_length", + description="Radius at which MaximumCircularVelocity is reached.", + lossy_compression_filter="FMantissa9", + dmo_property=True, + particle_properties=[ + "PartType0/Coordinates", + "PartType0/Masses", + "PartType1/Coordinates", + "PartType1/Masses", + "PartType4/Coordinates", + "PartType4/Masses", + "PartType5/Coordinates", + "PartType5/DynamicalMasses", + ], + output_physical=False, + a_scale_exponent=1, + ), + "Vmax_soft": Property( + name="MaximumCircularVelocity", + shape=1, + dtype=np.float32, + unit="snap_length/snap_time", + description="Maximum circular velocity when accounting for particle softening lengths.", + lossy_compression_filter="FMantissa9", + dmo_property=True, + particle_properties=[ + "PartType0/Coordinates", + "PartType0/Masses", + "PartType1/Coordinates", + "PartType1/Masses", + "PartType4/Coordinates", + "PartType4/Masses", + "PartType5/Coordinates", + "PartType5/DynamicalMasses", + ], + output_physical=True, + a_scale_exponent=0, + ), + "DM_R_vmax_soft": Property( + name="MaximumDarkMatterCircularVelocityRadius", + shape=1, + dtype=np.float32, + unit="snap_length", + description="Radius at which MaximumDarkMatterCircularVelocity is reached.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType1/Coordinates", "PartType1/Masses"], + output_physical=False, + a_scale_exponent=1, + ), + "DM_Vmax_soft": Property( + name="MaximumDarkMatterCircularVelocity", + shape=1, + dtype=np.float32, + unit="snap_length/snap_time", + description="Maximum circular velocity calculated using dark matter particles when accounting for particle softening lengths..", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType1/Coordinates", "PartType1/Masses"], + output_physical=True, + a_scale_exponent=0, + ), + "Xraylum": Property( + name="XRayLuminosity", + shape=3, + dtype=np.float64, + unit="snap_mass*snap_length**2/snap_time**3", + description="Total observer-frame Xray luminosity in three bands.", + lossy_compression_filter="DMantissa9", + dmo_property=False, + particle_properties=["PartType0/XrayLuminosities"], + output_physical=True, + a_scale_exponent=None, + ), + "Xraylum_restframe": Property( + name="XRayLuminosityInRestframe", + shape=3, + dtype=np.float64, + unit="snap_mass*snap_length**2/snap_time**3", + description="Total rest-frame Xray luminosity in three bands.", + lossy_compression_filter="DMantissa9", + dmo_property=False, + particle_properties=["PartType0/XrayLuminositiesRestframe"], + output_physical=True, + a_scale_exponent=0, + ), + "Xraylum_no_agn": Property( + name="XRayLuminosityWithoutRecentAGNHeating", + shape=3, + dtype=np.float64, + unit="snap_mass*snap_length**2/snap_time**3", + description="Total observer-frame Xray luminosity in three bands. Excludes gas that was recently heated by AGN.", + lossy_compression_filter="DMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/XrayLuminosities", + "PartType0/LastAGNFeedbackScaleFactors", + "PartType0/Temperatures", + ], + output_physical=True, + a_scale_exponent=None, + ), + "Xraylum_restframe_no_agn": Property( + name="XRayLuminosityInRestframeWithoutRecentAGNHeating", + shape=3, + dtype=np.float64, + unit="snap_mass*snap_length**2/snap_time**3", + description="Total rest-frame Xray luminosity in three bands. Excludes gas that was recently heated by AGN.", + lossy_compression_filter="DMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/XrayLuminositiesRestframe", + "PartType0/LastAGNFeedbackScaleFactors", + "PartType0/Temperatures", + ], + output_physical=True, + a_scale_exponent=0, + ), + "Xraylum_core_excision": Property( + name="XRayLuminosityCoreExcision", + shape=3, + dtype=np.float64, + unit="snap_mass*snap_length**2/snap_time**3", + description="Total observer-frame Xray luminosity in three bands. Excludes gas in the inner {core_excision}", + lossy_compression_filter="DMantissa9", + dmo_property=False, + particle_properties=["PartType0/XrayLuminosities", "PartType0/Coordinates"], + output_physical=True, + a_scale_exponent=None, + ), + "XRayLuminosityNoSat": Property( + name="XRayLuminosityNoSat", + shape=3, + dtype=np.float64, + unit="snap_mass*snap_length**2/snap_time**3", + description="Total observer-frame Xray luminosity in three bands. Excludes particles bound to satellites", + lossy_compression_filter="DMantissa9", + dmo_property=False, + particle_properties=["PartType0/XrayLuminosities"], + output_physical=True, + a_scale_exponent=None, + ), + "XRayLuminosityCoreExcisionNoSat": Property( + name="XRayLuminosityCoreExcisionNoSat", + shape=3, + dtype=np.float64, + unit="snap_mass*snap_length**2/snap_time**3", + description="Total observer-frame Xray luminosity in three bands. Excludes gas in the inner {core_excision}, and excludes particles bound to satellites", + lossy_compression_filter="DMantissa9", + dmo_property=False, + particle_properties=["PartType0/XrayLuminosities"], + output_physical=True, + a_scale_exponent=None, + ), + "Xraylum_restframe_core_excision": Property( + name="XRayLuminosityInRestframeCoreExcision", + shape=3, + dtype=np.float64, + unit="snap_mass*snap_length**2/snap_time**3", + description="Total rest-frame Xray luminosity in three bands. Excludes gas in the inner {core_excision}", + lossy_compression_filter="DMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/XrayLuminositiesRestframe", + "PartType0/Coordinates", + ], + output_physical=True, + a_scale_exponent=0, + ), + "Xraylum_no_agn_core_excision": Property( + name="XRayLuminosityWithoutRecentAGNHeatingCoreExcision", + shape=3, + dtype=np.float64, + unit="snap_mass*snap_length**2/snap_time**3", + description="Total observer-frame Xray luminosity in three bands. Excludes gas that was recently heated by AGN. Excludes gas in the inner {core_excision}", + lossy_compression_filter="DMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/XrayLuminosities", + "PartType0/LastAGNFeedbackScaleFactors", + "PartType0/Temperatures", + "PartType0/Coordinates", + ], + output_physical=True, + a_scale_exponent=None, + ), + "Xraylum_restframe_no_agn_core_excision": Property( + name="XRayLuminosityInRestframeWithoutRecentAGNHeatingCoreExcision", + shape=3, + dtype=np.float64, + unit="snap_mass*snap_length**2/snap_time**3", + description="Total rest-frame Xray luminosity in three bands. Excludes gas that was recently heated by AGN. Excludes gas in the inner {core_excision}", + lossy_compression_filter="DMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/XrayLuminositiesRestframe", + "PartType0/LastAGNFeedbackScaleFactors", + "PartType0/Temperatures", + "PartType0/Coordinates", + ], + output_physical=True, + a_scale_exponent=0, + ), + "Xrayphlum": Property( + name="XRayPhotonLuminosity", + shape=3, + dtype=np.float64, + unit="1/snap_time", + description="Total observer-frame Xray photon luminosity in three bands.", + lossy_compression_filter="DMantissa9", + dmo_property=False, + particle_properties=["PartType0/XrayPhotonLuminosities"], + output_physical=True, + a_scale_exponent=None, + ), + "Xrayphlum_restframe": Property( + name="XRayPhotonLuminosityInRestframe", + shape=3, + dtype=np.float64, + unit="1/snap_time", + description="Total rest-frame Xray photon luminosity in three bands.", + lossy_compression_filter="DMantissa9", + dmo_property=False, + particle_properties=["PartType0/XrayPhotonLuminositiesRestframe"], + output_physical=True, + a_scale_exponent=0, + ), + "Xrayphlum_no_agn": Property( + name="XRayPhotonLuminosityWithoutRecentAGNHeating", + shape=3, + dtype=np.float64, + unit="1/snap_time", + description="Total observer-frame Xray photon luminosity in three bands. Exclude gas that was recently heated by AGN.", + lossy_compression_filter="DMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/XrayPhotonLuminosities", + "PartType0/LastAGNFeedbackScaleFactors", + "PartType0/Temperatures", + ], + output_physical=True, + a_scale_exponent=None, + ), + "Xrayphlum_restframe_no_agn": Property( + name="XRayPhotonLuminosityInRestframeWithoutRecentAGNHeating", + shape=3, + dtype=np.float64, + unit="1/snap_time", + description="Total rest-frame Xray photon luminosity in three bands. Exclude gas that was recently heated by AGN.", + lossy_compression_filter="DMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/XrayPhotonLuminositiesRestframe", + "PartType0/LastAGNFeedbackScaleFactors", + "PartType0/Temperatures", + ], + output_physical=True, + a_scale_exponent=0, + ), + "Xrayphlum_core_excision": Property( + name="XRayPhotonLuminosityCoreExcision", + shape=3, + dtype=np.float64, + unit="1/snap_time", + description="Total observer-frame Xray photon luminosity in three bands. Excludes gas in the inner {core_excision}", + lossy_compression_filter="DMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/XrayPhotonLuminosities", + "PartType0/Temperatures", + "PartType0/Coordinates", + ], + output_physical=True, + a_scale_exponent=None, + ), + "Xrayphlum_restframe_core_excision": Property( + name="XRayPhotonLuminosityInRestframeCoreExcision", + shape=3, + dtype=np.float64, + unit="1/snap_time", + description="Total rest-frame Xray photon luminosity in three bands. Excludes gas in the inner {core_excision}", + lossy_compression_filter="DMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/XrayPhotonLuminositiesRestframe", + "PartType0/ElementMassFractions", + "PartType0/Coordinates", + ], + output_physical=True, + a_scale_exponent=0, + ), + "Xrayphlum_no_agn_core_excision": Property( + name="XRayPhotonLuminosityWithoutRecentAGNHeatingCoreExcision", + shape=3, + dtype=np.float64, + unit="1/snap_time", + description="Total observer-frame Xray photon luminosity in three bands. Exclude gas that was recently heated by AGN. Excludes gas in the inner {core_excision}", + lossy_compression_filter="DMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/XrayPhotonLuminosities", + "PartType0/Temperatures", + "PartType0/LastAGNFeedbackScaleFactors", + "PartType0/Coordinates", + ], + output_physical=True, + a_scale_exponent=None, + ), + "Xrayphlum_restframe_no_agn_core_excision": Property( + name="XRayPhotonLuminosityInRestframeWithoutRecentAGNHeatingCoreExcision", + shape=3, + dtype=np.float64, + unit="1/snap_time", + description="Total rest-frame Xray photon luminosity in three bands. Exclude gas that was recently heated by AGN. Excludes gas in the inner {core_excision}", + lossy_compression_filter="DMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/XrayPhotonLuminositiesRestframe", + "PartType0/Temperatures", + "PartType0/LastAGNFeedbackScaleFactors", + "PartType0/Coordinates", + ], + output_physical=True, + a_scale_exponent=0, + ), + "SpectroscopicLikeTemperature": Property( + name="SpectroscopicLikeTemperature", + shape=1, + dtype=np.float32, + unit="snap_temperature", + description="Spectroscopic-like gas temperature.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Temperatures", + "PartType0/Densities", + "PartType0/Masses", + ], + output_physical=True, + a_scale_exponent=0, + ), + "SpectroscopicLikeTemperature_no_agn": Property( + name="SpectroscopicLikeTemperatureWithoutRecentAGNHeating", + shape=1, + dtype=np.float32, + unit="snap_temperature", + description="Spectroscopic-like gas temperature. Exclude gas that was recently heated by AGN", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Temperatures", + "PartType0/Densities", + "PartType0/LastAGNFeedbackScaleFactors", + "PartType0/Masses", + ], + output_physical=True, + a_scale_exponent=0, + ), + "SpectroscopicLikeTemperature_core_excision": Property( + name="SpectroscopicLikeTemperatureCoreExcision", + shape=1, + dtype=np.float32, + unit="snap_temperature", + description="Spectroscopic-like gas temperature. Excludes gas in the inner {core_excision}", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Temperatures", + "PartType0/Densities", + "PartType0/LastAGNFeedbackScaleFactors", + "PartType0/Masses", + "PartType0/Coordinates", + ], + output_physical=True, + a_scale_exponent=0, + ), + "SpectroscopicLikeTemperature_no_agn_core_excision": Property( + name="SpectroscopicLikeTemperatureWithoutRecentAGNHeatingCoreExcision", + shape=1, + dtype=np.float32, + unit="snap_temperature", + description="Spectroscopic-like gas temperature. Exclude gas that was recently heated by AGN. Excludes gas in the inner {core_excision}", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Temperatures", + "PartType0/Densities", + "PartType0/LastAGNFeedbackScaleFactors", + "PartType0/Masses", + "PartType0/Coordinates", + "PartType0/LastAGNFeedbackScaleFactors", + ], + output_physical=True, + a_scale_exponent=0, + ), + "concentration_unsoft": Property( + name="ConcentrationUnsoftened", + shape=1, + dtype=np.float32, + unit="dimensionless", + description="Halo concentration assuming an NFW profile. No particle softening.", + lossy_compression_filter="FMantissa9", + dmo_property=True, + particle_properties=[ + "PartType0/Coordinates", + "PartType0/Masses", + "PartType1/Coordinates", + "PartType1/Masses", + "PartType4/Coordinates", + "PartType4/Masses", + "PartType5/Coordinates", + "PartType5/DynamicalMasses", + ], + output_physical=True, + a_scale_exponent=0, + ), + "concentration_soft": Property( + name="Concentration", + shape=1, + dtype=np.float32, + unit="dimensionless", + description="Halo concentration assuming an NFW profile. Minimum particle radius set to softening length", + lossy_compression_filter="FMantissa9", + dmo_property=True, + particle_properties=[ + "PartType0/Coordinates", + "PartType0/Masses", + "PartType1/Coordinates", + "PartType1/Masses", + "PartType4/Coordinates", + "PartType4/Masses", + "PartType5/Coordinates", + "PartType5/DynamicalMasses", + ], + output_physical=True, + a_scale_exponent=0, + ), + "concentration_dmo_unsoft": Property( + name="DarkMatterConcentrationUnsoftened", + shape=1, + dtype=np.float32, + unit="dimensionless", + description="Concentration of dark matter particles assuming an NFW profile. No particle softening", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType1/Coordinates", "PartType1/Masses"], + output_physical=True, + a_scale_exponent=0, + ), + "concentration_dmo_soft": Property( + name="DarkMatterConcentration", + shape=1, + dtype=np.float32, + unit="dimensionless", + description="Concentration of dark matter particles assuming an NFW profile. Minimum particle radius set to softening length", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType1/Coordinates", "PartType1/Masses"], + output_physical=True, + a_scale_exponent=0, + ), + "DarkMatterMassFlowRate": Property( + name="DarkMatterMassFlowRate", + shape=6, + dtype=np.float32, + unit="snap_mass / snap_time", + description="Mass flow rate of dark matter particles through spherical shells. Contains 6 entries: inflow rate at 0.1R, 0.3R, R, outflow rate at 0.1R, 0.3R, R.", + lossy_compression_filter="FMantissa9", + dmo_property=True, + particle_properties=[ + "PartType1/Coordinates", + "PartType1/Masses", + "PartType1/Velocities", + ], + output_physical=True, + a_scale_exponent=0, + ), + "ColdGasMassFlowRate": Property( + name="ColdGasMassFlowRate", + shape=9, + dtype=np.float32, + unit="snap_mass / snap_time", + description="Mass flow rate of cold gas particles ($\\log T < 3$) through spherical shells. Contains 9 entries: inflow rate at 0.1R, 0.3R, R, outflow rate at 0.1R, 0.3R, R, fast outflow rate at 0.1R, 0.3R, R.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Coordinates", + "PartType0/Masses", + "PartType0/Velocities", + "PartType0/Temperatures", + ], + output_physical=True, + a_scale_exponent=0, + ), + "CoolGasMassFlowRate": Property( + name="CoolGasMassFlowRate", + shape=9, + dtype=np.float32, + unit="snap_mass / snap_time", + description="Mass flow rate of cool gas particles ($3 < \\log T < 5$) through spherical shells. Contains 9 entries: inflow rate at 0.1R, 0.3R, R, outflow rate at 0.1R, 0.3R, R, fast outflow rate at 0.1R, 0.3R, R.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Coordinates", + "PartType0/Masses", + "PartType0/Velocities", + "PartType0/Temperatures", + ], + output_physical=True, + a_scale_exponent=0, + ), + "WarmGasMassFlowRate": Property( + name="WarmGasMassFlowRate", + shape=9, + dtype=np.float32, + unit="snap_mass / snap_time", + description="Mass flow rate of warm gas particles ($5 < \\log T < 7$) through spherical shells. Contains 9 entries: inflow rate at 0.1R, 0.3R, R, outflow rate at 0.1R, 0.3R, R, fast outflow rate at 0.1R, 0.3R, R.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Coordinates", + "PartType0/Masses", + "PartType0/Velocities", + "PartType0/Temperatures", + ], + output_physical=True, + a_scale_exponent=0, + ), + "HotGasMassFlowRate": Property( + name="HotGasMassFlowRate", + shape=9, + dtype=np.float32, + unit="snap_mass / snap_time", + description="Mass flow rate of hot gas particles ($7 < \\log T$) through spherical shells. Contains 9 entries: inflow rate at 0.1R, 0.3R, R, outflow rate at 0.1R, 0.3R, R, fast outflow rate at 0.1R, 0.3R, R.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Coordinates", + "PartType0/Masses", + "PartType0/Velocities", + "PartType0/Temperatures", + ], + output_physical=True, + a_scale_exponent=0, + ), + "HIMassFlowRate": Property( + name="HIMassFlowRate", + shape=6, + dtype=np.float32, + unit="snap_mass / snap_time", + description="Mass flow rate of gas particles through spherical shells weighted by HI fraction. Contains 6 entries: inflow rate at 0.1R, 0.3R, R, outflow rate at 0.1R, 0.3R, R.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Coordinates", + "PartType0/Masses", + "PartType0/Velocities", + "PartType0/SpeciesFractions", + "PartType0/ElementMassFractions", + ], + output_physical=True, + a_scale_exponent=0, + ), + "H2MassFlowRate": Property( + name="H2MassFlowRate", + shape=6, + dtype=np.float32, + unit="snap_mass / snap_time", + description="Mass flow rate of gas particles through spherical shells weighted by H2 fraction. Does not include Helium. Contains 6 entries: inflow rate at 0.1R, 0.3R, R, outflow rate at 0.1R, 0.3R, R.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Coordinates", + "PartType0/Masses", + "PartType0/Velocities", + "PartType0/SpeciesFractions", + "PartType0/ElementMassFractions", + ], + output_physical=True, + a_scale_exponent=0, + ), + "MetalMassFlowRate": Property( + name="MetalMassFlowRate", + shape=6, + dtype=np.float32, + unit="snap_mass / snap_time", + description="Mass flow rate of gas particles through spherical shells weighted by metal fraction. Contains 6 entries: inflow rate at 0.1R, 0.3R, R, outflow rate at 0.1R, 0.3R, R.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Coordinates", + "PartType0/Masses", + "PartType0/Velocities", + "PartType0/MetalMassFractions", + ], + output_physical=True, + a_scale_exponent=0, + ), + "StellarMassFlowRate": Property( + name="StellarMassFlowRate", + shape=6, + dtype=np.float32, + unit="snap_mass / snap_time", + description="Mass flow rate of star particles through spherical shells. Contains 6 entries: inflow rate at 0.1R, 0.3R, R, outflow rate at 0.1R, 0.3R, R.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType4/Coordinates", + "PartType4/Masses", + "PartType4/Velocities", + ], + output_physical=True, + a_scale_exponent=0, + ), + "ColdGasEnergyFlowRate": Property( + name="ColdGasEnergyFlowRate", + shape=9, + dtype=np.float32, + unit="snap_mass*snap_length**2/snap_time**3", + description="Energy flow rate of cold gas particles ($\\log T < 3$) through spherical shells. Contains 9 entries: inflow rate at 0.1R, 0.3R, R, outflow rate at 0.1R, 0.3R, R, fast outflow rate at 0.1R, 0.3R, R.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Coordinates", + "PartType0/Masses", + "PartType0/Velocities", + "PartType0/InternalEnergies", + "PartType0/Temperatures", + ], + output_physical=True, + a_scale_exponent=0, + ), + "CoolGasEnergyFlowRate": Property( + name="CoolGasEnergyFlowRate", + shape=9, + dtype=np.float32, + unit="snap_mass*snap_length**2/snap_time**3", + description="Energy flow rate of cool gas particles ($3 < \\log T < 5$) through spherical shells. Contains 9 entries: inflow rate at 0.1R, 0.3R, R, outflow rate at 0.1R, 0.3R, R, fast outflow rate at 0.1R, 0.3R, R.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Coordinates", + "PartType0/Masses", + "PartType0/Velocities", + "PartType0/InternalEnergies", + "PartType0/Temperatures", + ], + output_physical=True, + a_scale_exponent=0, + ), + "WarmGasEnergyFlowRate": Property( + name="WarmGasEnergyFlowRate", + shape=9, + dtype=np.float32, + unit="snap_mass*snap_length**2/snap_time**3", + description="Energy flow rate of warm gas particles ($5 < \\log T < 7$) through spherical shells. Contains 9 entries: inflow rate at 0.1R, 0.3R, R, outflow rate at 0.1R, 0.3R, R, fast outflow rate at 0.1R, 0.3R, R.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Coordinates", + "PartType0/Masses", + "PartType0/Velocities", + "PartType0/InternalEnergies", + "PartType0/Temperatures", + ], + output_physical=True, + a_scale_exponent=0, + ), + "HotGasEnergyFlowRate": Property( + name="HotGasEnergyFlowRate", + shape=9, + dtype=np.float32, + unit="snap_mass*snap_length**2/snap_time**3", + description="Energy flow rate of hot gas particles ($7 < \\log T$) through spherical shells. Contains 9 entries: inflow rate at 0.1R, 0.3R, R, outflow rate at 0.1R, 0.3R, R, fast outflow rate at 0.1R, 0.3R, R.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Coordinates", + "PartType0/Masses", + "PartType0/Velocities", + "PartType0/InternalEnergies", + "PartType0/Temperatures", + ], + output_physical=True, + a_scale_exponent=0, + ), + "ColdGasMomentumFlowRate": Property( + name="ColdGasMomentumFlowRate", + shape=9, + dtype=np.float32, + unit="snap_mass*snap_length/snap_time**2", + description="Momentum flow rate of cold gas particles ($\\log T < 3$) through spherical shells. Contains 9 entries: inflow rate at 0.1R, 0.3R, R, outflow rate at 0.1R, 0.3R, R, fast outflow rate at 0.1R, 0.3R, R.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Coordinates", + "PartType0/Masses", + "PartType0/Velocities", + "PartType0/InternalEnergies", + "PartType0/Temperatures", + ], + output_physical=True, + a_scale_exponent=0, + ), + "CoolGasMomentumFlowRate": Property( + name="CoolGasMomentumFlowRate", + shape=9, + dtype=np.float32, + unit="snap_mass*snap_length/snap_time**2", + description="Momentum flow rate of cool gas particles ($3 < \\log T < 5$) through spherical shells. Contains 9 entries: inflow rate at 0.1R, 0.3R, R, outflow rate at 0.1R, 0.3R, R, fast outflow rate at 0.1R, 0.3R, R.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Coordinates", + "PartType0/Masses", + "PartType0/Velocities", + "PartType0/InternalEnergies", + "PartType0/Temperatures", + ], + output_physical=True, + a_scale_exponent=0, + ), + "WarmGasMomentumFlowRate": Property( + name="WarmGasMomentumFlowRate", + shape=9, + dtype=np.float32, + unit="snap_mass*snap_length/snap_time**2", + description="Momentum flow rate of warm gas particles ($5 < \\log T < 7$) through spherical shells. Contains 9 entries: inflow rate at 0.1R, 0.3R, R, outflow rate at 0.1R, 0.3R, R, fast outflow rate at 0.1R, 0.3R, R.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Coordinates", + "PartType0/Masses", + "PartType0/Velocities", + "PartType0/InternalEnergies", + "PartType0/Temperatures", + ], + output_physical=True, + a_scale_exponent=0, + ), + "HotGasMomentumFlowRate": Property( + name="HotGasMomentumFlowRate", + shape=9, + dtype=np.float32, + unit="snap_mass*snap_length/snap_time**2", + description="Momentum flow rate of hot gas particles ($7 < \\log T$) through spherical shells. Contains 9 entries: inflow rate at 0.1R, 0.3R, R, outflow rate at 0.1R, 0.3R, R, fast outflow rate at 0.1R, 0.3R, R.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Coordinates", + "PartType0/Masses", + "PartType0/Velocities", + "PartType0/InternalEnergies", + "PartType0/Temperatures", + ], + output_physical=True, + a_scale_exponent=0, + ), + "com": Property( + name="CentreOfMass", + shape=3, + dtype=np.float64, + unit="snap_length", + description="Centre of mass.", + lossy_compression_filter="DScale6", + dmo_property=True, + particle_properties=[ + "PartType0/Coordinates", + "PartType0/Masses", + "PartType1/Coordinates", + "PartType1/Masses", + "PartType4/Coordinates", + "PartType4/Masses", + "PartType5/Coordinates", + "PartType5/DynamicalMasses", + ], + output_physical=False, + a_scale_exponent=1, + ), + "com_gas": Property( + name="GasCentreOfMass", + shape=3, + dtype=np.float64, + unit="snap_length", + description="Centre of mass of gas.", + lossy_compression_filter="DScale6", + dmo_property=False, + particle_properties=["PartType0/Coordinates", "PartType0/Masses"], + output_physical=False, + a_scale_exponent=1, + ), + "com_dm": Property( + name="DarkMatterCentreOfMass", + shape=3, + dtype=np.float64, + unit="snap_length", + description="Centre of mass of dark matter.", + lossy_compression_filter="DScale6", + dmo_property=False, + particle_properties=["PartType1/Coordinates", "PartType1/Masses"], + output_physical=False, + a_scale_exponent=1, + ), + "com_star": Property( + name="StellarCentreOfMass", + shape=3, + dtype=np.float64, + unit="snap_length", + description="Centre of mass of stars.", + lossy_compression_filter="DScale6", + dmo_property=False, + particle_properties=["PartType4/Coordinates", "PartType4/Masses"], + output_physical=False, + a_scale_exponent=1, + ), + "compY": Property( + name="ComptonY", + shape=1, + dtype=np.float64, + unit="snap_length**2", + description="Total Compton y parameter.", + lossy_compression_filter="DMantissa9", + dmo_property=False, + particle_properties=["PartType0/ComptonYParameters"], + output_physical=True, + a_scale_exponent=0, + ), + "compY_no_agn": Property( + name="ComptonYWithoutRecentAGNHeating", + shape=1, + dtype=np.float64, + unit="snap_length**2", + description="Total Compton y parameter. Excludes gas that was recently heated by AGN.", + lossy_compression_filter="DMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/ComptonYParameters", + "PartType0/LastAGNFeedbackScaleFactors", + "PartType0/Temperatures", + ], + output_physical=True, + a_scale_exponent=0, + ), + "gasFefrac": Property( + name="GasMassFractionInIron", + shape=1, + dtype=np.float32, + unit="dimensionless", + description="Total gas mass fraction in iron.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType0/Masses", "PartType0/ElementMassFractions"], + output_physical=True, + a_scale_exponent=0, + ), + "gasFefrac_SF": Property( + name="StarFormingGasMassFractionInIron", + shape=1, + dtype=np.float32, + unit="dimensionless", + description="Total gas mass fraction in iron for gas that is star-forming.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Masses", + "PartType0/ElementMassFractions", + "PartType0/StarFormationRates", + ], + output_physical=True, + a_scale_exponent=0, + ), + "gasOfrac": Property( + name="GasMassFractionInOxygen", + shape=1, + dtype=np.float32, + unit="dimensionless", + description="Total gas mass in oxygen.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType0/Masses", "PartType0/ElementMassFractions"], + output_physical=True, + a_scale_exponent=0, + ), + "gasOfrac_SF": Property( + name="StarFormingGasMassFractionInOxygen", + shape=1, + dtype=np.float32, + unit="dimensionless", + description="Total gas mass fraction in oxygen for gas that is star-forming.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Masses", + "PartType0/ElementMassFractions", + "PartType0/StarFormationRates", + ], + output_physical=True, + a_scale_exponent=0, + ), + "gasmetalfrac": Property( + name="GasMassFractionInMetals", + shape=1, + dtype=np.float32, + unit="dimensionless", + description="Total gas mass fraction in metals.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType0/Masses", "PartType0/MetalMassFractions"], + output_physical=True, + a_scale_exponent=0, + ), + "gasmetalfrac_SF": Property( + name="StarFormingGasMassFractionInMetals", + shape=1, + dtype=np.float32, + unit="dimensionless", + description="Total gas mass fraction in metals for gas that is star-forming.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Masses", + "PartType0/MetalMassFractions", + "PartType0/StarFormationRates", + ], + output_physical=True, + a_scale_exponent=0, + ), + "kappa_corot_baryons": Property( + name="KappaCorotBaryons", + shape=1, + dtype=np.float32, + unit="dimensionless", + description="Kappa-corot for baryons (gas and stars), relative to the HaloCentre and the centre of mass velocity of the baryons.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Coordinates", + "PartType0/Masses", + "PartType0/Velocities", + "PartType4/Coordinates", + "PartType4/Masses", + "PartType4/Velocities", + ], + output_physical=True, + a_scale_exponent=0, + ), + "kappa_corot_gas": Property( + name="KappaCorotGas", + shape=1, + dtype=np.float32, + unit="dimensionless", + description="Kappa-corot for gas, relative to the HaloCentre and the centre of mass velocity of the gas.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Coordinates", + "PartType0/Masses", + "PartType0/Velocities", + ], + output_physical=True, + a_scale_exponent=0, + ), + "kappa_corot_star": Property( + name="KappaCorotStars", + shape=1, + dtype=np.float32, + unit="dimensionless", + description="Kappa-corot for stars, relative to the HaloCentre and the centre of mass velocity of the stars.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType4/Coordinates", + "PartType4/Masses", + "PartType4/Velocities", + ], + output_physical=True, + a_scale_exponent=0, + ), + "proj_veldisp_dm": Property( + name="DarkMatterProjectedVelocityDispersion", + shape=1, + dtype=np.float32, + unit="snap_length/snap_time", + description="Mass-weighted velocity dispersion of the DM along the projection axis, relative to the DM centre of mass velocity.", + lossy_compression_filter="FMantissa9", + dmo_property=True, + particle_properties=["PartType1/Velocities"], + output_physical=True, + a_scale_exponent=0, + ), + "proj_veldisp_gas": Property( + name="GasProjectedVelocityDispersion", + shape=1, + dtype=np.float32, + unit="snap_length/snap_time", + description="Mass-weighted velocity dispersion of the gas along the projection axis, relative to the gas centre of mass velocity.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType0/Velocities"], + output_physical=True, + a_scale_exponent=0, + ), + "proj_veldisp_star": Property( + name="StellarProjectedVelocityDispersion", + shape=1, + dtype=np.float32, + unit="snap_length/snap_time", + description="Mass-weighted velocity dispersion of the stars along the projection axis, relative to the stellar centre of mass velocity.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType4/Velocities"], + output_physical=True, + a_scale_exponent=0, + ), + "r": Property( + name="SORadius", + shape=1, + dtype=np.float32, + unit="snap_length", + description="Radius of a sphere {label}", + lossy_compression_filter="FMantissa9", + dmo_property=True, + particle_properties=[ + "PartType0/Coordinates", + "PartType0/Masses", + "PartType1/Coordinates", + "PartType1/Masses", + "PartType4/Coordinates", + "PartType4/Masses", + "PartType5/Coordinates", + "PartType5/DynamicalMasses", + "PartType6/Coordinates", + "PartType6/Masses", + "PartType6/Weights", + ], + output_physical=False, + a_scale_exponent=1, + ), + "spin_parameter": Property( + name="SpinParameter", + shape=1, + dtype=np.float32, + unit="dimensionless", + description="Bullock et al. (2001) spin parameter.", + lossy_compression_filter="FMantissa9", + dmo_property=True, + particle_properties=[ + "PartType0/Coordinates", + "PartType0/Masses", + "PartType0/Velocities", + "PartType1/Coordinates", + "PartType1/Masses", + "PartType1/Velocities", + "PartType4/Coordinates", + "PartType4/Masses", + "PartType4/Velocities", + "PartType5/Coordinates", + "PartType5/DynamicalMasses", + "PartType5/Velocities", + ], + output_physical=True, + a_scale_exponent=0, + ), + "starFefrac": Property( + name="StellarMassFractionInIron", + shape=1, + dtype=np.float32, + unit="dimensionless", + description="Total stellar mass fraction in iron.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType4/Masses", "PartType4/ElementMassFractions"], + output_physical=True, + a_scale_exponent=0, + ), + "starMgfrac": Property( + name="StellarMassFractionInMagnesium", + shape=1, + dtype=np.float32, + unit="dimensionless", + description="Total stellar mass fraction in magnesium.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType4/Masses", "PartType4/ElementMassFractions"], + output_physical=True, + a_scale_exponent=0, + ), + "starOfrac": Property( + name="StellarMassFractionInOxygen", + shape=1, + dtype=np.float32, + unit="dimensionless", + description="Total stellar mass fraction in oxygen.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType4/Masses", "PartType4/ElementMassFractions"], + output_physical=True, + a_scale_exponent=0, + ), + "starmetalfrac": Property( + name="StellarMassFractionInMetals", + shape=1, + dtype=np.float32, + unit="dimensionless", + description="Total stellar mass fraction in metals.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType4/Masses", "PartType4/MetalMassFractions"], + output_physical=True, + a_scale_exponent=0, + ), + "stellar_age_lw": Property( + name="LuminosityWeightedMeanStellarAge", + shape=1, + dtype=np.float32, + unit="snap_time", + description="Luminosity weighted mean stellar age. The weight is the r band luminosity.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType4/Luminosities", + "PartType4/BirthScaleFactors", + ], + output_physical=True, + a_scale_exponent=None, + ), + "stellar_age_mw": Property( + name="MassWeightedMeanStellarAge", + shape=1, + dtype=np.float32, + unit="snap_time", + description="Mass weighted mean stellar age.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType4/Masses", "PartType4/BirthScaleFactors"], + output_physical=True, + a_scale_exponent=None, + ), + "vcom": Property( + name="CentreOfMassVelocity", + shape=3, + dtype=np.float32, + unit="snap_length/snap_time", + description="Centre of mass velocity.", + lossy_compression_filter="DScale1", + dmo_property=True, + particle_properties=[ + "PartType0/Masses", + "PartType0/Velocities", + "PartType1/Masses", + "PartType1/Velocities", + "PartType4/Masses", + "PartType4/Velocities", + "PartType5/DynamicalMasses", + "PartType5/Velocities", + ], + output_physical=True, + a_scale_exponent=0, + ), + "vcom_gas": Property( + name="GasCentreOfMassVelocity", + shape=3, + dtype=np.float32, + unit="snap_length/snap_time", + description="Centre of mass velocity of gas.", + lossy_compression_filter="DScale1", + dmo_property=False, + particle_properties=["PartType0/Masses", "PartType0/Velocities"], + output_physical=True, + a_scale_exponent=0, + ), + "vcom_star": Property( + name="StellarCentreOfMassVelocity", + shape=3, + dtype=np.float32, + unit="snap_length/snap_time", + description="Centre of mass velocity of stars.", + lossy_compression_filter="DScale1", + dmo_property=False, + particle_properties=["PartType4/Masses", "PartType4/Velocities"], + output_physical=True, + a_scale_exponent=0, + ), + "veldisp_matrix_dm": Property( + name="DarkMatterVelocityDispersionMatrix", + shape=6, + dtype=np.float32, + unit="snap_length**2/snap_time**2", + description="Mass-weighted velocity dispersion of the dark matter. Measured relative to the DM centre of mass velocity. The order of the components of the dispersion tensor is XX YY ZZ XY XZ YZ.", + lossy_compression_filter="FMantissa9", + dmo_property=True, + particle_properties=["PartType1/Masses", "PartType1/Velocities"], + output_physical=True, + a_scale_exponent=0, + ), + "veldisp_matrix_gas": Property( + name="GasVelocityDispersionMatrix", + shape=6, + dtype=np.float32, + unit="snap_length**2/snap_time**2", + description="Mass-weighted velocity dispersion of the gas. Measured relative to the gas centre of mass velocity. The order of the components of the dispersion tensor is XX YY ZZ XY XZ YZ.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType0/Masses", "PartType0/Velocities"], + output_physical=True, + a_scale_exponent=0, + ), + "veldisp_matrix_star": Property( + name="StellarVelocityDispersionMatrix", + shape=6, + dtype=np.float32, + unit="snap_length**2/snap_time**2", + description="Mass-weighted velocity dispersion of the stars. Measured relative to the stellar centre of mass velocity. The order of the components of the dispersion tensor is XX YY ZZ XY XZ YZ.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType4/Masses", "PartType4/Velocities"], + output_physical=True, + a_scale_exponent=0, + ), + "LinearMassWeightedOxygenOverHydrogenOfGas": Property( + name="LinearMassWeightedOxygenOverHydrogenOfGas", + shape=1, + dtype=np.float32, + unit="dimensionless", + description="Linear sum of the oxygen over hydrogen ratio of gas, multiplied with the gas mass.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Masses", + "PartType0/ElementMassFractions", + "PartType0/Temperatures", + "PartType0/Densities", + ], + output_physical=True, + a_scale_exponent=0, + ), + "LinearMassWeightedNitrogenOverOxygenOfGas": Property( + name="LinearMassWeightedNitrogenOverOxygenOfGas", + shape=1, + dtype=np.float32, + unit="dimensionless", + description="Linear sum of the total nitrogen over oxygen ratio of gas, multiplied with the gas mass.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Masses", + "PartType0/ElementMassFractions", + "PartType0/Temperatures", + "PartType0/Densities", + ], + output_physical=True, + a_scale_exponent=0, + ), + "LinearMassWeightedCarbonOverOxygenOfGas": Property( + name="LinearMassWeightedCarbonOverOxygenOfGas", + shape=1, + dtype=np.float32, + unit="dimensionless", + description="Linear sum of the total carbon over oxygen ratio of gas, multiplied with the gas mass.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Masses", + "PartType0/ElementMassFractions", + "PartType0/Temperatures", + "PartType0/Densities", + ], + output_physical=True, + a_scale_exponent=0, + ), + "LinearMassWeightedDiffuseNitrogenOverOxygenOfGas": Property( + name="LinearMassWeightedDiffuseNitrogenOverOxygenOfGas", + shape=1, + dtype=np.float32, + unit="dimensionless", + description="Linear sum of the diffuse nitrogen over oxygen ratio of gas, multiplied with the gas mass.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Masses", + "PartType0/ElementMassFractionsDiffuse", + "PartType0/Temperatures", + "PartType0/Densities", + ], + output_physical=True, + a_scale_exponent=0, + ), + "LinearMassWeightedDiffuseCarbonOverOxygenOfGas": Property( + name="LinearMassWeightedDiffuseCarbonOverOxygenOfGas", + shape=1, + dtype=np.float32, + unit="dimensionless", + description="Linear sum of the diffuse carbon over oxygen ratio of gas, multiplied with the gas mass.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Masses", + "PartType0/ElementMassFractionsDiffuse", + "PartType0/Temperatures", + "PartType0/Densities", + ], + output_physical=True, + a_scale_exponent=0, + ), + "LinearMassWeightedDiffuseOxygenOverHydrogenOfGas": Property( + name="LinearMassWeightedDiffuseOxygenOverHydrogenOfGas", + shape=1, + dtype=np.float32, + unit="dimensionless", + description="Linear sum of the diffuse oxygen over hydrogen ratio of gas, multiplied with the gas mass.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Masses", + "PartType0/ElementMassFractionsDiffuse", + "PartType0/Temperatures", + "PartType0/Densities", + ], + output_physical=True, + a_scale_exponent=0, + ), + "LogarithmicMassWeightedDiffuseNitrogenOverOxygenOfGasLowLimit": Property( + name="LogarithmicMassWeightedDiffuseNitrogenOverOxygenOfGasLowLimit", + shape=1, + dtype=np.float32, + unit="dimensionless", + description="Logarithmic sum of the diffuse nitrogen over oxygen ratio of gas, multiplied with the gas mass. Imposes a lower limit of 1.e-4 times solar N/O.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Masses", + "PartType0/ElementMassFractionsDiffuse", + "PartType0/Temperatures", + "PartType0/Densities", + ], + output_physical=True, + a_scale_exponent=0, + ), + "LogarithmicMassWeightedDiffuseNitrogenOverOxygenOfGasHighLimit": Property( + name="LogarithmicMassWeightedDiffuseNitrogenOverOxygenOfGasHighLimit", + shape=1, + dtype=np.float32, + unit="dimensionless", + description="Logarithmic sum of the diffuse nitrogen over oxygen ratio of gas, multiplied with the gas mass. Imposes a lower limit of 1.e-3 times solar N/O.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Masses", + "PartType0/ElementMassFractionsDiffuse", + "PartType0/Temperatures", + "PartType0/Densities", + ], + output_physical=True, + a_scale_exponent=0, + ), + "LogarithmicMassWeightedDiffuseCarbonOverOxygenOfGasLowLimit": Property( + name="LogarithmicMassWeightedDiffuseCarbonOverOxygenOfGasLowLimit", + shape=1, + dtype=np.float32, + unit="dimensionless", + description="Logarithmic sum of the diffuse carbon over oxygen ratio of gas, multiplied with the gas mass. Imposes a lower limit of 1.e-4 times solar C/O.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Masses", + "PartType0/ElementMassFractionsDiffuse", + "PartType0/Temperatures", + "PartType0/Densities", + ], + output_physical=True, + a_scale_exponent=0, + ), + "LogarithmicMassWeightedDiffuseCarbonOverOxygenOfGasHighLimit": Property( + name="LogarithmicMassWeightedDiffuseCarbonOverOxygenOfGasHighLimit", + shape=1, + dtype=np.float32, + unit="dimensionless", + description="Logarithmic sum of the diffuse carbon over oxygen ratio of gas, multiplied with the gas mass. Imposes a lower limit of 1.e-3 times solar C/O.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Masses", + "PartType0/ElementMassFractionsDiffuse", + "PartType0/Temperatures", + "PartType0/Densities", + ], + output_physical=True, + a_scale_exponent=0, + ), + "LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfGasLowLimit": Property( + name="LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfGasLowLimit", + shape=1, + dtype=np.float32, + unit="dimensionless", + description="Logarithmic sum of the diffuse oxygen over hydrogen ratio of gas, multiplied with the gas mass. Imposes a lower limit of 1.e-4 times solar O/H.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Masses", + "PartType0/ElementMassFractionsDiffuse", + "PartType0/Temperatures", + "PartType0/Densities", + ], + output_physical=True, + a_scale_exponent=0, + ), + "LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfGasHighLimit": Property( + name="LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfGasHighLimit", + shape=1, + dtype=np.float32, + unit="dimensionless", + description="Logarithmic sum of the diffuse oxygen over hydrogen ratio of gas, multiplied with the gas mass. Imposes a lower limit of 1.e-3 times solar O/H.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Masses", + "PartType0/ElementMassFractionsDiffuse", + "PartType0/Temperatures", + "PartType0/Densities", + ], + output_physical=True, + a_scale_exponent=0, + ), + "LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfAtomicGasLowLimit": Property( + name="LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfAtomicGasLowLimit", + shape=1, + dtype=np.float32, + unit="dimensionless", + description="Logarithmic sum of the diffuse oxygen over hydrogen ratio of atomic gas, multiplied with the gas mass. Imposes a lower limit of 1.e-4 times solar O/H.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Masses", + "PartType0/ElementMassFractionsDiffuse", + "PartType0/ElementMassFractions", + "PartType0/SpeciesFractions", + "PartType0/Temperatures", + "PartType0/Densities", + ], + output_physical=True, + a_scale_exponent=0, + ), + "LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfAtomicGasHighLimit": Property( + name="LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfAtomicGasHighLimit", + shape=1, + dtype=np.float32, + unit="dimensionless", + description="Logarithmic sum of the diffuse oxygen over hydrogen ratio of atomic gas, multiplied with the gas mass. Imposes a lower limit of 1.e-3 times solar O/H.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Masses", + "PartType0/ElementMassFractionsDiffuse", + "PartType0/ElementMassFractions", + "PartType0/SpeciesFractions", + "PartType0/Temperatures", + "PartType0/Densities", + ], + output_physical=True, + a_scale_exponent=0, + ), + "LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfMolecularGasLowLimit": Property( + name="LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfMolecularGasLowLimit", + shape=1, + dtype=np.float32, + unit="dimensionless", + description="Logarithmic sum of the diffuse oxygen over hydrogen ratio of molecular gas, multiplied with the gas mass. Imposes a lower limit of 1.e-4 times solar O/H.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Masses", + "PartType0/ElementMassFractionsDiffuse", + "PartType0/ElementMassFractions", + "PartType0/SpeciesFractions", + "PartType0/Temperatures", + "PartType0/Densities", + ], + output_physical=True, + a_scale_exponent=0, + ), + "LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfMolecularGasHighLimit": Property( + name="LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfMolecularGasHighLimit", + shape=1, + dtype=np.float32, + unit="dimensionless", + description="Logarithmic sum of the diffuse oxygen over hydrogen ratio of molecular gas, multiplied with the gas mass. Imposes a lower limit of 1.e-3 times solar O/H.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Masses", + "PartType0/ElementMassFractionsDiffuse", + "PartType0/ElementMassFractions", + "PartType0/SpeciesFractions", + "PartType0/Temperatures", + "PartType0/Densities", + ], + output_physical=True, + a_scale_exponent=0, + ), + "LinearMassWeightedIronOverHydrogenOfStars": Property( + name="LinearMassWeightedIronOverHydrogenOfStars", + shape=1, + dtype=np.float32, + unit="dimensionless", + description="Linear sum of the iron over hydrogen ratio of stars, multiplied with the stellar mass.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType4/Masses", "PartType4/ElementMassFractions"], + output_physical=True, + a_scale_exponent=0, + ), + "LogarithmicMassWeightedIronOverHydrogenOfStarsLowLimit": Property( + name="LogarithmicMassWeightedIronOverHydrogenOfStarsLowLimit", + shape=1, + dtype=np.float32, + unit="dimensionless", + description="Logarithmic sum of the iron over hydrogen ratio of stars, multiplied with the stellar mass. Imposes a lower limit of 1.e-4 times solar Fe/H.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType4/Masses", "PartType4/ElementMassFractions"], + output_physical=True, + a_scale_exponent=0, + ), + "LogarithmicMassWeightedIronOverHydrogenOfStarsHighLimit": Property( + name="LogarithmicMassWeightedIronOverHydrogenOfStarsHighLimit", + shape=1, + dtype=np.float32, + unit="dimensionless", + description="Logarithmic sum of the iron over hydrogen ratio of stars, multiplied with the stellar mass. Imposes a lower limit of 1.e-3 times solar Fe/H.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType4/Masses", "PartType4/ElementMassFractions"], + output_physical=True, + a_scale_exponent=0, + ), + "LinearMassWeightedMagnesiumOverHydrogenOfStars": Property( + name="LinearMassWeightedMagnesiumOverHydrogenOfStars", + shape=1, + dtype=np.float32, + unit="dimensionless", + description="Linear sum of the magnesium over hydrogen ratio of stars, multiplied with the stellar mass.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType4/Masses", "PartType4/ElementMassFractions"], + output_physical=True, + a_scale_exponent=0, + ), + "LogarithmicMassWeightedMagnesiumOverHydrogenOfStarsLowLimit": Property( + name="LogarithmicMassWeightedMagnesiumOverHydrogenOfStarsLowLimit", + shape=1, + dtype=np.float32, + unit="dimensionless", + description="Logarithmic sum of the magnesium over hydrogen ratio of stars, multiplied with the stellar mass. Imposes a lower limit of 1.e-4 times solar Fe/H.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType4/Masses", "PartType4/ElementMassFractions"], + output_physical=True, + a_scale_exponent=0, + ), + "LogarithmicMassWeightedMagnesiumOverHydrogenOfStarsHighLimit": Property( + name="LogarithmicMassWeightedMagnesiumOverHydrogenOfStarsHighLimit", + shape=1, + dtype=np.float32, + unit="dimensionless", + description="Logarithmic sum of the magnesium over hydrogen ratio of stars, multiplied with the stellar mass. Imposes a lower limit of 1.e-3 times solar Fe/H.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=["PartType4/Masses", "PartType4/ElementMassFractions"], + output_physical=True, + a_scale_exponent=0, + ), + "GasMassInColdDenseDiffuseMetals": Property( + name="GasMassInColdDenseDiffuseMetals", + shape=1, + dtype=np.float32, + unit="snap_mass", + description="Sum of the diffuse metal mass in cold, dense gas.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType0/Masses", + "PartType0/MetalMassFractions", + "PartType0/DustMassFractions", + "PartType0/Temperatures", + "PartType0/Densities", + ], + output_physical=True, + a_scale_exponent=0, + ), + "LogarithmicMassWeightedIronFromSNIaOverHydrogenOfStarsLowLimit": Property( + name="LogarithmicMassWeightedIronFromSNIaOverHydrogenOfStarsLowLimit", + shape=1, + dtype=np.float32, + unit="dimensionless", + description="Logarithmic sum of the iron over hydrogen ratio of stars, multiplied with the stellar mass, where only iron from SNIa is included. Imposes a lower limit of 1.e-4 times solar Fe/H.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType4/Masses", + "PartType4/ElementMassFractions", + "PartType4/IronMassFractionsFromSNIa", + ], + output_physical=True, + a_scale_exponent=0, + ), + "LinearMassWeightedIronFromSNIaOverHydrogenOfStars": Property( + name="LinearMassWeightedIronFromSNIaOverHydrogenOfStars", + shape=1, + dtype=np.float32, + unit="dimensionless", + description="Sum of the iron over hydrogen ratio of stars, multiplied with the stellar mass, where only iron from SNIa is included.", + lossy_compression_filter="FMantissa9", + dmo_property=False, + particle_properties=[ + "PartType4/Masses", + "PartType4/ElementMassFractions", + "PartType4/IronMassFractionsFromSNIa", + ], + output_physical=True, + a_scale_exponent=0, + ), + # InputHalo properties + "cofp": Property( + name="InputHalos/HaloCentre", + shape=3, + dtype=np.float64, + unit="snap_length", + description="The centre of the subhalo as given by the halo finder. Used as reference for all relative positions. For VR and HBTplus this is equal to the position of the most bound particle in the subhalo.", + lossy_compression_filter="DScale6", + dmo_property=True, + particle_properties=[], + output_physical=False, + a_scale_exponent=1, + ), + "index": Property( + name="InputHalos/HaloCatalogueIndex", + shape=1, + dtype=np.int64, + unit="dimensionless", + description="Index of this halo in the original halo finder catalogue (first halo has index=0).", + lossy_compression_filter="None", + dmo_property=True, + particle_properties=[], + output_physical=True, + a_scale_exponent=None, + ), + "is_central": Property( + name="InputHalos/IsCentral", + shape=1, + dtype=np.int64, + unit="dimensionless", + description="Whether the halo finder flagged the halo as central (1) or satellite (0).", + lossy_compression_filter="None", + dmo_property=True, + particle_properties=[], + output_physical=True, + a_scale_exponent=None, + ), + "nr_bound_part": Property( + name="InputHalos/NumberOfBoundParticles", + shape=1, + dtype=np.int64, + unit="dimensionless", + description="Total number of particles bound to the subhalo.", + lossy_compression_filter="None", + dmo_property=True, + particle_properties=[], + output_physical=True, + a_scale_exponent=None, + ), + # Velociraptor properties + "VR/ID": Property( + name="VR/ID", + shape=1, + dtype=np.uint64, + unit="dimensionless", + description="ID assigned to this halo by VR.", + lossy_compression_filter="None", + dmo_property=True, + particle_properties=[], + output_physical=True, + a_scale_exponent=None, + ), + "VR/Parent_halo_ID": Property( + name="VR/ParentHaloID", + shape=1, + dtype=np.int64, + unit="dimensionless", + description="VR/ID of the direct parent of this halo. -1 for field halos.", + lossy_compression_filter="None", + dmo_property=True, + particle_properties=[], + output_physical=True, + a_scale_exponent=None, + ), + "VR/Structuretype": Property( + name="VR/StructureType", + shape=1, + dtype=np.int32, + unit="dimensionless", + description="Structure type identified by VR. Field halos are 10, higher numbers are for satellites.", + lossy_compression_filter="None", + dmo_property=True, + particle_properties=[], + output_physical=True, + a_scale_exponent=None, + ), + "VR/hostHaloID": Property( + name="VR/HostHaloID", + shape=1, + dtype=np.int64, + unit="dimensionless", + description="VR/ID of the top level parent of this halo. -1 for field halos.", + lossy_compression_filter="None", + dmo_property=True, + particle_properties=[], + output_physical=True, + a_scale_exponent=None, + ), + "VR/numSubStruct": Property( + name="VR/NumberOfSubstructures", + shape=1, + dtype=np.uint64, + unit="dimensionless", + description="Number of sub-structures within this halo.", + lossy_compression_filter="None", + dmo_property=True, + particle_properties=[], + output_physical=True, + a_scale_exponent=None, + ), + # EAGLE SubFind properties + "SubfindEagle/group_nr": Property( + name="SubFind/GroupNumber", + shape=1, + dtype=np.uint64, + unit="dimensionless", + description="Group number of the host of this subhalo", + lossy_compression_filter="None", + dmo_property=True, + particle_properties=[], + output_physical=True, + a_scale_exponent=None, + ), + "SubfindEagle/sub_group_nr": Property( + name="SubFind/SubGroupNumber", + shape=1, + dtype=np.uint64, + unit="dimensionless", + description="Sub group number of this subhalo (within its host group)", + lossy_compression_filter="None", + dmo_property=True, + particle_properties=[], + output_physical=True, + a_scale_exponent=None, + ), + # HBT properties + "HBTplus/Depth": Property( + name="HBTplus/Depth", + shape=1, + dtype=np.uint64, + unit="dimensionless", + description="Level of the subhalo in the merging hierarchy.", + lossy_compression_filter="None", + dmo_property=True, + particle_properties=[], + output_physical=True, + a_scale_exponent=None, + ), + "HBTplus/HostHaloId": Property( + name="HBTplus/HostFOFId", + shape=1, + dtype=np.int64, + unit="dimensionless", + description="ID of the host FOF halo of this subhalo. Hostless halos have HostFOFId == -1", + lossy_compression_filter="None", + dmo_property=True, + particle_properties=[], + output_physical=True, + a_scale_exponent=None, + ), + "HBTplus/TrackId": Property( + name="HBTplus/TrackId", + shape=1, + dtype=np.uint64, + unit="dimensionless", + description="Unique ID for this subhalo which is consistent across snapshots.", + lossy_compression_filter="None", + dmo_property=True, + particle_properties=[], + output_physical=True, + a_scale_exponent=None, + ), + "HBTplus/SnapshotOfBirth": Property( + name="HBTplus/SnapshotOfBirth", + shape=1, + dtype=np.int64, + unit="dimensionless", + description="Snapshot when this subhalo was formed.", + lossy_compression_filter="None", + dmo_property=True, + particle_properties=[], + output_physical=True, + a_scale_exponent=None, + ), + "HBTplus/NestedParentTrackId": Property( + name="HBTplus/NestedParentTrackId", + shape=1, + dtype=np.int64, + unit="dimensionless", + description="TrackId of the parent of this subhalo.", + lossy_compression_filter="None", + dmo_property=True, + particle_properties=[], + output_physical=True, + a_scale_exponent=None, + ), + "HBTplus/DescendantTrackId": Property( + name="HBTplus/DescendantTrackId", + shape=1, + dtype=np.int64, + unit="dimensionless", + description="TrackId of the descendant of this subhalo.", + lossy_compression_filter="None", + dmo_property=True, + particle_properties=[], + output_physical=True, + a_scale_exponent=None, + ), + "HBTplus/LastMaxMass": Property( + name="HBTplus/LastMaxMass", + shape=1, + dtype=np.float32, + unit="snap_mass", + description="Maximum mass of this subhalo across its evolutionary history", + lossy_compression_filter="FMantissa9", + dmo_property=True, + particle_properties=[], + output_physical=True, + a_scale_exponent=0, + ), + "HBTplus/SnapshotOfLastMaxMass": Property( + name="HBTplus/SnapshotOfLastMaxMass", + shape=1, + dtype=np.uint64, + unit="dimensionless", + description="Latest snapshot when this subhalo had its maximum mass.", + lossy_compression_filter="None", + dmo_property=True, + particle_properties=[], + output_physical=True, + a_scale_exponent=None, + ), + "HBTplus/LastMaxVmaxPhysical": Property( + name="HBTplus/LastMaxVmaxPhysical", + shape=1, + dtype=np.float32, + unit="snap_length/snap_time", + description="Largest value of maximum circular velocity of this subhalo across its evolutionary history", + lossy_compression_filter="FMantissa9", + dmo_property=True, + particle_properties=[], + output_physical=True, + a_scale_exponent=None, + ), + "HBTplus/SnapshotOfLastMaxVmax": Property( + name="HBTplus/SnapshotOfLastMaxVmax", + shape=1, + dtype=np.uint64, + unit="dimensionless", + description="Latest snapshot when this subhalo had its largest maximum circular velocity.", + lossy_compression_filter="None", + dmo_property=True, + particle_properties=[], + output_physical=True, + a_scale_exponent=None, + ), + "HBTplus/SnapshotOfLastIsolation": Property( + name="HBTplus/SnapshotOfLastIsolation", + shape=1, + dtype=np.uint64, + unit="dimensionless", + description="Latest snapshot when this subhalo was a central. -1 if the subhalo has always been a central.", + lossy_compression_filter="None", + dmo_property=True, + particle_properties=[], + output_physical=True, + a_scale_exponent=None, + ), + # FOF properties + "FOF/Centres": Property( + name="FOF/Centres", + shape=3, + dtype=np.float64, + unit="snap_length", + description="Centre of mass of the host FOF halo of this subhalo. Zero for satellite and hostless subhalos.", + lossy_compression_filter="DScale6", + dmo_property=True, + particle_properties=[], + output_physical=False, + a_scale_exponent=1, + ), + "FOF/Masses": Property( + name="FOF/Masses", + shape=1, + dtype=np.float32, + unit="snap_mass", + description="Mass of the host FOF halo of this subhalo. Zero for satellite and hostless subhalos.", + lossy_compression_filter="FMantissa9", + dmo_property=True, + particle_properties=[], + output_physical=True, + a_scale_exponent=0, + ), + "FOF/Sizes": Property( + name="FOF/Sizes", + shape=1, + dtype=np.uint64, + unit="dimensionless", + description="Number of particles in the host FOF halo of this subhalo. Zero for satellite and hostless subhalos.", + lossy_compression_filter="None", + dmo_property=True, + particle_properties=[], + output_physical=True, + a_scale_exponent=None, + ), + "FOF/Radii": Property( + name="FOF/Radii", + shape=1, + dtype=np.float32, + unit="snap_length", + description="Radius of the particle furthest from the FOF centre of mass. Zero for satellite and hostless subhalos. Missing for older runs.", + lossy_compression_filter="FMantissa9", + dmo_property=True, + particle_properties=[], + output_physical=False, + a_scale_exponent=1, + ), + # SOAP properties + "SOAP/SubhaloRankByBoundMass": Property( + name="SOAP/SubhaloRankByBoundMass", + shape=1, + dtype=np.int32, + unit="dimensionless", + description="Ranking by mass of the halo within its parent field halo. Zero for the most massive halo in the field halo.", + lossy_compression_filter="None", + dmo_property=True, + particle_properties=[], + output_physical=True, + a_scale_exponent=None, + ), + "SOAP/HostHaloIndex": Property( + name="SOAP/HostHaloIndex", + shape=1, + dtype=np.int64, + unit="dimensionless", + description="Index (within the SOAP arrays) of the top level parent of this subhalo. -1 for hostless halos.", + lossy_compression_filter="None", + dmo_property=True, + particle_properties=[], + output_physical=True, + a_scale_exponent=None, + ), + "SOAP/IncludedInReducedSnapshot": Property( + name="SOAP/IncludedInReducedSnapshot", + shape=1, + dtype=np.int32, + unit="dimensionless", + description="Whether this halo is included in the reduced snapshot.", + lossy_compression_filter="None", + dmo_property=True, + particle_properties=[], + output_physical=True, + a_scale_exponent=None, + ), + "SOAP/ProgenitorIndex": Property( + name="SOAP/ProgenitorIndex", + shape=1, + dtype=np.int32, + unit="dimensionless", + description="Index (within the previous snapshot SOAP arrays) of the main progenitor of this subhalo.", + lossy_compression_filter="None", + dmo_property=True, + particle_properties=[], + output_physical=True, + a_scale_exponent=None, + ), + "SOAP/DescendantIndex": Property( + name="SOAP/DescendantIndex", + shape=1, + dtype=np.int32, + unit="dimensionless", + description="Index (within the next snapshot SOAP arrays) of the main descendant of this subhalo.", + lossy_compression_filter="None", + dmo_property=True, + particle_properties=[], + output_physical=True, + a_scale_exponent=None, + ), + } + + def __init__(self, parameters, snipshot_parameters, units_cgs): + """ + Constructor. + """ + self.properties = {} + self.footnotes = [] + self.parameters = parameters + self.snipshot_parameters = snipshot_parameters + self.units_cgs = units_cgs + + # Check all footnotes actually correspond to a property in the table + for prop_names in self.explanation.values(): + for prop_name in prop_names: + err_msg = f"{prop_name} missing from property_list" + assert prop_name in self.full_property_list, err_msg + + def get_footnotes(self, name: str): + """ + List all of the footnotes for a particular property. Returns an empty + string for properties that have no footnotes. + """ + footnotes = [] + for fnote in self.explanation.keys(): + names = self.explanation[fnote] + if name in names: + try: + i = self.footnotes.index(fnote) + except ValueError: + i = len(self.footnotes) + self.footnotes.append(fnote) + footnotes.append(i + 1) + if len(footnotes) > 0: + footnote_links = [f"\\hyperref[footnote:{i}]{{{i}}}" for i in footnotes] + return f'$^{{{",".join(footnote_links)}}}$' + else: + return "" + + def add_properties(self, halo_property: HaloProperty, halo_type: str): + """ + Add all the properties calculated for a particular halo type to the + internal dictionary. + """ + # Get the property_filters dict, which says whether a property should be included, + # and what category the property is in + props = halo_property.property_list + base_halo_type = halo_property.base_halo_type + if halo_type == "DummyProperties": + property_filters = { + prop.name: prop.name.split("/")[0] for prop in props.values() + } + snipshot_filters = { + prop.name: prop.name.split("/")[0] for prop in props.values() + } + else: + property_filters = self.parameters.get_property_filters( + base_halo_type, [prop.name for prop in props.values()] + ) + snipshot_filters = self.snipshot_parameters.get_property_filters( + base_halo_type, [prop.name for prop in props.values()] + ) + + # Loop through all possible properties for this halo type and add them to the + # table, skipping those that we shouldn't calculate according to the parameter file + for name, prop in props.items(): + + if not property_filters[prop.name]: + continue + prop_cat = property_filters[prop.name] + if snipshot_filters.get(prop.name, False): + assert prop_cat == snipshot_filters[prop.name] + + units = unyt.unyt_quantity(1, units=prop.unit) + if not prop.output_physical: + units = units * unyt.Unit("a") ** prop.a_scale_exponent + prop_unit = units.units.latex_repr.replace( + "\\rm{km} \\cdot \\rm{kpc}", "\\rm{kpc} \\cdot \\rm{km}" + ).replace("\\frac{\\rm{km}^{2}}{\\rm{s}^{2}}", "\\rm{km}^{2} / \\rm{s}^{2}") + + prop_dtype = prop.dtype.__name__ + + if name in self.properties: + # run some checks + if prop.shape != self.properties[name]["shape"]: + print("Shape mismatch!") + print(halo_type, name, prop.shape, self.properties[name]) + exit() + if prop_dtype != self.properties[name]["dtype"]: + print("dtype mismatch!") + print(halo_type, name, prop_dtype, self.properties[name]) + exit() + if prop_unit != self.properties[name]["units"]: + print("Unit mismatch!") + print(halo_type, name, prop_unit, self.properties[name]) + exit() + if prop.description != self.properties[name]["description"]: + print("Description mismatch!") + print( + halo_type, + name, + prop.description, + self.properties[name], + ) + exit() + if prop_cat != self.properties[name]["category"]: + print("Category mismatch!") + print(halo_type, name, prop_cat, self.properties[name]) + exit() + assert prop.name == self.properties[name]["name"] + + if not snipshot_filters[prop.name]: + self.properties[name]["types"].append("SnapshotOnly" + halo_type) + else: + self.properties[name]["types"].append(halo_type) + else: + self.properties[name] = { + "name": prop.name, + "shape": prop.shape, + "dtype": prop_dtype, + "units": prop_unit, + "description": prop.description, + "category": prop_cat, + "compression": prop.lossy_compression_filter, + "dmo": prop.dmo_property, + } + if not snipshot_filters[prop.name]: + self.properties[name]["types"] = ["SnapshotOnly" + halo_type] + else: + self.properties[name]["types"] = [halo_type] + + def generate_tex_files(self, output_dir: str): + """ + Outputs all the .tex files required to generate the documentation. + + The documentation consists of + - a hand-written SOAP.tex file. + - a table.tex + - a footnotes.tex file which will contain the contents of + the various hand-written footnote*.tex files + - a version and time stamp file, called timestamp.tex + - a filters.tex which contains the threshold value of each filter + - a variations.tex file which contains a table of all the halo type + variations present in the parameter file, and the filter for each + + This function regenerates the last 5 files, based on the contents of + the internal property dictionary and the parameter file passed. + """ + + # sort the properties by category and then alphabetically within each + # category + category_order = [ + "basic", + "general", + "gas", + "dm", + "star", + "baryon", + "InputHalos", + "VR", + "HBTplus", + "FOF", + "SOAP", + ] + prop_names = sorted( + self.properties.keys(), + key=lambda key: ( + category_order.index(self.properties[key]["category"]), + self.properties[key]["name"].lower(), + ), + ) + + # generate the LaTeX header for a standalone table file + headstr = """\\documentclass{article} +\\usepackage{amsmath} +\\usepackage{amssymb} +\\usepackage{longtable} +\\usepackage{pifont} +\\usepackage{pdflscape} +\\usepackage{a4wide} +\\usepackage{multirow} +\\usepackage{xcolor} + +\\begin{document}""" + # property table string: table header + tablestr = """\\begin{landscape} +\\begin{longtable}{p{15em}llllllllll} +Name & Shape & Type & Units & SH & ES & IS & EP & SO & Category & Compression\\\\ +\\multicolumn{11}{l}{\\rule{30pt}{0pt}Description}\\\\ +\\hline{}\\endhead{}""" + # keep track of the previous category to draw a line when a category + # is finished + prev_cat = None + for prop_name in prop_names: + prop = self.properties[prop_name] + footnotes = self.get_footnotes(prop_name) + prop_outputname = f"{prop['name']}{footnotes}" + if prop_outputname.split("/")[0] in ["HBTplus", "VR", "FOF"]: + prop_outputname = "InputHalos/" + prop_outputname + prop_outputname = word_wrap_name(prop_outputname) + prop_shape = f'{prop["shape"]}' + prop_dtype = prop["dtype"] + prop_units = ( + f'${prop["units"]}$' if prop["units"] != "" else "dimensionless" + ) + prop_cat = prop["category"] + prop_comp = self.compression_description[prop["compression"]] + prop_description = prop["description"].format( + label="satisfying a spherical overdensity criterion.", + core_excision="excised core", + ) + + checkmark = "\\ding{51}" + xmark = "\\ding{53}" + scissor = "\\ding{36}" + prop_subhalo = checkmark if "SubhaloProperties" in prop["types"] else xmark + prop_subhalo = ( + scissor + if "SnapshotOnlySubhaloProperties" in prop["types"] + else prop_subhalo + ) + prop_exclusive = ( + checkmark if "ExclusiveSphereProperties" in prop["types"] else xmark + ) + prop_exclusive = ( + scissor + if "SnapshotOnlyExclusiveSphereProperties" in prop["types"] + else prop_exclusive + ) + prop_inclusive = ( + checkmark if "InclusiveSphereProperties" in prop["types"] else xmark + ) + prop_inclusive = ( + scissor + if "SnapshotOnlyInclusiveSphereProperties" in prop["types"] + else prop_inclusive + ) + prop_projected = ( + checkmark if "ProjectedApertureProperties" in prop["types"] else xmark + ) + prop_projected = ( + scissor + if "SnapshotOnlyProjectedApertureProperties" in prop["types"] + else prop_projected + ) + prop_SO = checkmark if "SOProperties" in prop["types"] else xmark + prop_SO = ( + scissor if "SnapshotOnlySOProperties" in prop["types"] else prop_SO + ) + table_props = [ + prop_outputname, + prop_shape, + prop_dtype, + prop_units, + prop_subhalo, + prop_exclusive, + prop_inclusive, + prop_projected, + prop_SO, + prop_cat, + prop_comp, + ] + if prop["dmo"]: + print_table_props = [f"{{\\color{{violet}}{v}}}" for v in table_props] + prop_description = f"{{\\color{{violet}}{prop_description}}}" + else: + print_table_props = list(table_props) + if prev_cat is None: + prev_cat = prop_cat + if prop_cat != prev_cat: + prev_cat = prop_cat + tablestr += "\\hline{}" + tablestr += "\\rule{0pt}{4ex}" + tablestr += " & ".join([v for v in print_table_props]) + "\\\\*\n" + tablestr += f"\\multicolumn{{11}}{{p{{15cm}}}}{{\\rule{{30pt}}{{0pt}}{prop_description}}}\\\\\n" + tablestr += """\\end{longtable} +\\end{landscape}""" + # standalone table file footer + tailstr = "\\end{document}" + + # generate the auxilary documentation files + with open(f"{output_dir}/timestamp.tex", "w") as ofile: + ofile.write(get_version_string()) + with open(f"{output_dir}/table.tex", "w") as ofile: + ofile.write(tablestr) + with open(f"{output_dir}/footnotes.tex", "w") as ofile: + for i, fnote in enumerate(self.footnotes): + with open(f"documentation/{fnote}", "r") as ifile: + fnstr = ifile.read() + fnstr = fnstr.replace("$FOOTNOTE_NUMBER$", f"{i+1}") + # Substitute in values from parameter file + if "$LOG_COLD_GAS_TEMP$" in fnstr: + params = self.parameters.get_cold_dense_params() + assert params["initialised"] + T = f'{np.log10(params["maximum_temperature_K"]):.2g}' + fnstr = fnstr.replace("$LOG_COLD_GAS_TEMP$", T) + if "$LOG_COLD_GAS_DENSITY$" in fnstr: + params = self.parameters.get_cold_dense_params() + assert params["initialised"] + rho = ( + f'{np.log10(params["minimum_hydrogen_number_density_cm3"]):.2g}' + ) + fnstr = fnstr.replace("$LOG_COLD_GAS_DENSITY$", rho) + ofile.write(f"{fnstr}\n\n") + + # Particle limits for each filter + with open(f"{output_dir}/filters.tex", "w") as ofile: + for name, filter_info in self.parameters.parameters["filters"].items(): + value = filter_info["limit"] + ofile.write(f"\\newcommand{{\\{name.lower()}filter}}{{{value}}}\n") + + # Create table of variations of each halo type, always add BoundSubhalo + tablestr = """\\pagebreak +\\begin{adjustbox}{tabular=llcl,center} +Group name (HDF5) & Group name (swiftsimio) & Inclusive? & Filter \\\\ +\\hline +\\verb+BoundSubhalo+ & \\verb+bound_subhalo+ & \\ding{53} & - \\\\*\n""" + # Add SO apertures to table + apertures = self.parameters.parameters.get("SOProperties", {}) + for _, variation in apertures.get("variations", {}).items(): + name = "" + if "radius_multiple" in variation: + name += f'{int(variation["radius_multiple"])}xR_' + if variation["type"] == "BN98": + name += f"BN98" + else: + name += f'{variation["value"]:.0f}_{variation["type"]}' + filter = variation.get("filter", "basic") + filter = "-" if filter == "basic" else filter + tablestr += f"\\verb+SO/{name}+ & \\verb+spherical_overdensity_{name.lower()}+& \\ding{{51}} & {filter} \\\\*\n" + # Determine which ExclusiveSphere and InclusiveSphere apertures are present + variations_ES, variations_IS = {}, {} + apertures = self.parameters.parameters.get("ApertureProperties", {}) + for _, variation in apertures.get("variations", {}).items(): + if "radius_in_kpc" in variation: + radius_in_kpc = variation["radius_in_kpc"] + if radius_in_kpc < 1: + aperture_name = f"{int(radius_in_kpc*1000)}pc" + else: + aperture_name = f"{int(radius_in_kpc)}kpc" + else: + prop = variation["property"].split("/")[-1] + multiplier = variation.get("radius_multiple", 1) + if multiplier == 1: + aperture_name = prop + else: + aperture_name = f"{int(multiplier)}x{prop}" + if variation["inclusive"]: + variations_IS[aperture_name] = variation.get("filter", "basic") + else: + variations_ES[aperture_name] = variation.get("filter", "basic") + # Add ExclusiveSphere apertures to table in sorted order + for aperture_name in sorted(variations_ES.keys()): + filter = ( + "-" + if variations_ES[aperture_name] == "basic" + else variations_ES[aperture_name] + ) + tablestr += f"\\verb+ExclusiveSphere/{aperture_name}+ & \\verb+exclusive_sphere_{aperture_name}+ & \\ding{{53}} & {filter} \\\\*\n" + # Add InclusiveSphere apertures to table in sorted order + for radius in sorted(variations_IS.keys()): + filter = ( + "-" + if variations_IS[aperture_name] == "basic" + else variations_IS[aperture_name] + ) + tablestr += f"\\verb+InclusiveSphere/{aperture_name}+ & \\verb+inclusive_sphere_{aperture_name}+ & \\ding{{51}} & {filter} \\\\*\n" + # Determine which projected apertures are present + variations_proj = {} + apertures = self.parameters.parameters.get("ProjectedApertureProperties", {}) + for _, variation in apertures.get("variations", {}).items(): + if "radius_in_kpc" in variation: + radius_in_kpc = variation["radius_in_kpc"] + if radius_in_kpc < 1: + aperture_name = f"{int(radius_in_kpc*1000)}pc" + else: + aperture_name = f"{int(radius_in_kpc)}kpc" + else: + prop = variation["property"].split("/")[-1] + multiplier = variation.get("radius_multiple", 1) + if multiplier == 1: + aperture_name = prop + else: + aperture_name = f"{int(multiplier)}x{prop}" + variations_proj[aperture_name] = variation.get("filter", "basic") + # Add ProjectedApertures to table in sorted order + for aperture_name in sorted(variations_proj.keys()): + filter = ( + "-" + if variations_proj[aperture_name] == "basic" + else variations_proj[aperture_name] + ) + tablestr += f"\\verb+ProjectedAperture/{aperture_name}/projP+ & \\verb+projected_aperture_{aperture_name}_projP+ & \\ding{{53}} & {filter} \\\\*\n" + # Add others groups + tablestr += f"\\verb+SOAP+ & \\verb+soap+ & - & - \\\\*\n" + tablestr += f"\\verb+InputHalos+ & \\verb+input_halos+ & - & - \\\\*\n" + halo_finder = self.parameters.parameters["HaloFinder"]["type"] + tablestr += f"\\verb+InputHalos/{halo_finder}+ & \\verb+input_halos_{halo_finder.lower()}+ & - & - \\\\*\n" + if halo_finder == "HBTplus": + tablestr += ( + f"\\verb+InputHalos/FOF+ & \\verb+input_halos_fof+ & - & - \\\\*\n" + ) + # Finish table and output to file + tablestr += "\\end{adjustbox}\n\\newpage" + with open(f"{output_dir}/variations_table.tex", "w") as ofile: + ofile.write(tablestr) + + # Output base units + with open(f"{output_dir}/units.tex", "w") as ofile: + for name, symbol in [ + ("length", "L"), + ("mass", "M"), + ("time", "t"), + ("temperature", "T"), + ]: + value = self.units_cgs[f"Unit {name} in cgs (U_{symbol})"] + ofile.write(f"\\newcommand{{\\{name}baseunit}}{{{value:.4g}}}\n") + vel_kms = (1 * unyt.snap_length / unyt.snap_time).to("km/s").value + ofile.write(f"\\newcommand{{\\velbaseunit}}{{{vel_kms:.4g}}}\n") + + +class DummyProperties: + """ + Dummy HaloProperty object used to include properties which are not computed + for any halo type (e.g. the 'VR' properties). These are identified from the + property list using the fact that their full output name starts with the + category (e.g. we have 'VR/ID' instead of 'ID') + """ + + base_halo_type = "DummyProperties" + + def __init__(self, halo_finder): + categories = ["SOAP", "InputHalos", halo_finder] + # Currently FOF properties are only stored for HBT + if halo_finder == "HBTplus": + categories += ["FOF"] + self.property_list = { + name: prop + for name, prop in PropertyTable.full_property_list.items() + if prop.name.split("/")[0] in categories + } + + +def get_parameter_file_all_properties(): + """ + Returns a parameter file that can be used to generate a full list of + available properties in SOAP, and also return a standard unit_cgs dict + """ + mock_parameters = { + "HaloFinder": {"type": "HBTplus"}, + "ApertureProperties": {"properties": {}, "variations": {}}, + "ProjectedApertureProperties": {"properties": {}, "variations": {}}, + "SOProperties": {"properties": {}, "variations": {}}, + "SubhaloProperties": {"properties": {}}, + "filters": { + "general": { + "limit": 100, + "properties": [ + "BoundSubhalo/NumberOfGasParticles", + "BoundSubhalo/NumberOfDarkMatterParticles", + "BoundSubhalo/NumberOfStarParticles", + "BoundSubhalo/NumberOfBlackHoleParticles", + ], + "combine_properties": "sum", + }, + "baryon": { + "limit": 100, + "properties": [ + "BoundSubhalo/NumberOfGasParticles", + "BoundSubhalo/NumberOfStarParticles", + ], + "combine_properties": "sum", + }, + "dm": { + "limit": 100, + "properties": ["BoundSubhalo/NumberOfDarkMatterParticles"], + }, + "gas": {"limit": 100, "properties": ["BoundSubhalo/NumberOfGasParticles"]}, + "star": { + "limit": 100, + "properties": ["BoundSubhalo/NumberOfStarParticles"], + }, + }, + "calculations": { + "calculate_missing_properties": True, + "min_read_radius_cmpc": 5, + }, + } + os.makedirs("test_data", exist_ok=True) + with open("test_data/mock_parameter_file.yml", "w") as file: + yaml.dump(mock_parameters, file) + parameters = ParameterFile( + file_name="test_data/mock_parameter_file.yml", snipshot=False + ) + snipshot_parameters = ParameterFile( + file_name="test_data/mock_parameter_file.yml", snipshot=True + ) + units_cgs = { + "Unit current in cgs (U_I)": 1.0, + "Unit length in cgs (U_L)": 3.08567758e24, + "Unit mass in cgs (U_M)": 1.98841e43, + "Unit temperature in cgs (U_T)": 1.0, + "Unit time in cgs (U_t)": 3.08567758e19, + } + return parameters, snipshot_parameters, units_cgs + + +if __name__ == "__main__": + """ + Standalone script execution: + Create a PropertyTable object will all the properties from all the halo + types and print the property table or the documentation. The latter is the + default; the former can be achieved by changing the boolean in the condition + below. + + You must pass a parameter file and a snapshot to run this script + """ + + import sys + import h5py + import yaml + + from core.parameter_file import ParameterFile + + # get all the halo types we only import them here to avoid + # circular imports when this script is imported from another script + from particle_selection.aperture_properties import ( + ExclusiveSphereProperties, + InclusiveSphereProperties, + ) + from particle_selection.projected_aperture_properties import ( + ProjectedApertureProperties, + ) + from particle_selection.SO_properties import SOProperties, CoreExcisedSOProperties + from particle_selection.subhalo_properties import SubhaloProperties + + # Parse parameter file + try: + parameters = ParameterFile(file_name=sys.argv[1], snipshot=False) + snipshot_parameters = ParameterFile(file_name=sys.argv[1], snipshot=True) + + # Parse snapshot file to extract base units + try: + with h5py.File(sys.argv[2]) as snap: + units_cgs = { + name: float(value[0]) for name, value in snap["Units"].attrs.items() + } + except IndexError: + print("No snapshot file passed.") + exit() + + except IndexError: + print("No parameter file passed. Outputting all possible properties") + parameters, snipshot_parameters, units_cgs = get_parameter_file_all_properties() + + unyt.define_unit( + "snap_length", + units_cgs["Unit length in cgs (U_L)"] * unyt.cm, + tex_repr="\\rm{L}", + ) + unyt.define_unit( + "snap_mass", + units_cgs["Unit mass in cgs (U_M)"] * unyt.g, + tex_repr="\\rm{M}", + ) + unyt.define_unit( + "snap_time", + units_cgs["Unit time in cgs (U_t)"] * unyt.s, + tex_repr="\\rm{t}", + ) + unyt.define_unit( + "snap_temperature", + units_cgs["Unit temperature in cgs (U_T)"] * unyt.K, + tex_repr="\\rm{T}", + ) + # Define scale factor unit + unyt.define_unit("a", 1 * unyt.dimensionless, tex_repr="\\rm{a}") + + table = PropertyTable(parameters, snipshot_parameters, units_cgs) + # Add standard halo definitions + table.add_properties(SubhaloProperties, "SubhaloProperties") + table.add_properties(ExclusiveSphereProperties, "ExclusiveSphereProperties") + table.add_properties(InclusiveSphereProperties, "InclusiveSphereProperties") + table.add_properties(ProjectedApertureProperties, "ProjectedApertureProperties") + # Decide whether to add core excised properties + for _, variation in parameters.parameters["SOProperties"]["variations"].items(): + if variation.get("core_excision_fraction", 0): + table.add_properties(CoreExcisedSOProperties, "SOProperties") + break + else: + table.add_properties(SOProperties, "SOProperties") + # Add InputHalos and SOAP properties + table.add_properties( + DummyProperties(parameters.parameters["HaloFinder"]["type"]), "DummyProperties" + ) + + table.generate_tex_files("documentation") diff --git a/compression/README.md b/compression/README.md new file mode 100644 index 00000000..2d1e3fc1 --- /dev/null +++ b/compression/README.md @@ -0,0 +1,31 @@ +# Compression scripts + +### Membership files +Compression of the membership files should be done +using the [h5repack](https://support.hdfgroup.org/documentation/hdf5/latest/_h5_t_o_o_l__r_p__u_g.html) command. + +The script `make_virtual_snapshot.py` can be used to create a virtual hdf5 files that +links the membership files to the snapshot files, such that properties from both can be accesed from a single file. + +### SOAP catalogues + +The script `compress_soap_catalogue.py` creates a compressed output catalogue. It works +by reading the lossy compression filter metadata for each property from the +uncompressed SOAP output, applying it (and GZIP compression), +and updating the metadata to reflect the change. It should be run with MPI, e.g. + +`mpirun -- python compress_soap_catalogue.py ${input_filename} ${output_filename} ${scratch_dir}` + +This scripts outputs temporary files as it runs to the `scratch_dir`, so preferably +this is a filesystem with fast reads/writes (e.g. `/snap8` on cosma). + +The following files in this directory are to help with running this script: + - Lossy compression filters for datasets which had the wrong filter set when SOAP + was run can be placed within `wrong_compression.yml`. The compression script will + use the filters in this file rather than the ones in the original catalogue. + - The script `create_empty_SOAP_catalogue.py`can be used to generate an empty SOAP + catalogue for snapshots that have no halos, since SOAP will not run on those snapshots. + - The file `filters.yml` contains the serialised information for the lossy compression + filters. These were grabbed from a SWIFT snapshot, since those filters are + not available in h5py. The script `extract_filters.py` can generate this file. + diff --git a/compression/README.txt b/compression/README.txt deleted file mode 100644 index c337cca7..00000000 --- a/compression/README.txt +++ /dev/null @@ -1,17 +0,0 @@ -This directory contains a fast compression script that can be used to compress -a SOAP catalogue file. -The script reads the lossy compression filter metadata from the SOAP output, -applies it (and GZIP compression), and updates the metadata to reflect the -change. - -The script requires two additional data files, also found in this directory: - - a file (filters.yml) containing the serialised information for the lossy compression - filters. These were grabbed from a SWIFT snapshot, since those filters are - not available in h5py. The script extract_filters.py can generate this file. - - a file (wrong_compression.yml) that contains updated lossy compression filter - names for datasets which had the wrong filter set when SOAP was run. This should - no longer be necessary, but is good to have just in case. - -This directory additionally contains a script that can be used to generate an -empty SOAP catalogue for snapshots that have no halos, since SOAP will not run -on those snapshots. diff --git a/compression/compress_fast_metadata.py b/compression/compress_soap_catalogue.py similarity index 52% rename from compression/compress_fast_metadata.py rename to compression/compress_soap_catalogue.py index c5d74df0..c19dadf1 100644 --- a/compression/compress_fast_metadata.py +++ b/compression/compress_soap_catalogue.py @@ -1,13 +1,16 @@ -import numpy as np -import h5py -import multiprocessing as mp import argparse import time import os + +os.environ["OPENBLAS_NUM_THREADS"] = "1" import shutil + +import numpy as np +import h5py +from mpi4py import MPI import yaml -script_folder = os.path.realpath(__file__).removesuffix("/compress_fast_metadata.py") +script_folder = os.path.realpath(os.path.dirname(__file__)) with open(f"{script_folder}/filters.yml", "r") as ffile: filterdict = yaml.safe_load(ffile) @@ -98,8 +101,7 @@ def create_lossy_dataset(file, name, shape, filter): h5py.h5d.create(file.id, name.encode("utf-8"), type, space, new_plist, None).close() -def compress_dataset(arguments): - input_name, output_name, dset = arguments +def compress_dataset(input_name, output_name, dset): # Setting hdf5 version of file fapl = h5py.h5p.create(h5py.h5p.FILE_ACCESS) @@ -117,9 +119,6 @@ def compress_dataset(arguments): dset_name = dset.split("/")[-1] if dset_name in compression_fixes: filter = compression_fixes[dset_name] - # TODO: Remove after removing DMantissa21 from property table - if filter == "DMantissa21": - filter = "DMantissa9" data = ifile[dset][:] if filter == "None": if len(data.shape) == 1: @@ -140,99 +139,123 @@ def compress_dataset(arguments): # This is needed if we have used the compression_fixes dictionary if attr == "Lossy compression filter": ofile["data"].attrs[attr] = filter - # TODO: Remove, this was only the case for a small number of catalogues - elif ( - attr - == "Conversion factor to CGS (including cosmological corrections)" - ): - ofile["data"].attrs[ - "Conversion factor to physical CGS (including cosmological corrections)" - ] = filter else: ofile["data"].attrs[attr] = ifile[dset].attrs[attr] return dset -if __name__ == "__main__": +# Assign datasets to ranks +def assign_datasets(nr_files, nr_ranks, comm_rank): + files_on_rank = np.zeros(nr_ranks, dtype=int) + files_on_rank[:] = nr_files // nr_ranks + remainder = nr_files % nr_ranks + if remainder == 1: + files_on_rank[0] += 1 + elif remainder > 1: + for i in range(remainder): + files_on_rank[int((nr_ranks - 1) * i / (remainder - 1))] += 1 + assert np.sum(files_on_rank) == nr_files, f"{nr_files=}, {nr_ranks=}" + first_file = np.cumsum(files_on_rank) - files_on_rank + return first_file[comm_rank], first_file[comm_rank] + files_on_rank[comm_rank] - # disable CBLAS threading to avoid problems when spawning - # parallel numpy imports - os.environ["OPENBLAS_NUM_THREADS"] = "1" - mp.set_start_method("forkserver") +if __name__ == "__main__": + + comm = MPI.COMM_WORLD + comm_rank = comm.Get_rank() + comm_size = comm.Get_size() argparser = argparse.ArgumentParser() - argparser.add_argument("input") - argparser.add_argument("output") - argparser.add_argument("scratch") - argparser.add_argument("--nproc", "-n", type=int, default=1) + argparser.add_argument( + "input", help="Filename of uncompressed input SOAP catalogue" + ) + argparser.add_argument("output", help="Filename of output catalogue") + argparser.add_argument("scratch", help="Directory to store temporary files") args = argparser.parse_args() - # Setting hdf5 version of file - fapl = h5py.h5p.create(h5py.h5p.FILE_ACCESS) - fapl.set_libver_bounds(h5py.h5f.LIBVER_V18, h5py.h5f.LIBVER_LATEST) - fid = h5py.h5f.create( - args.output.encode("utf-8"), flags=h5py.h5f.ACC_TRUNC, fapl=fapl - ) + mastertic = time.time() - print(f"Copying over groups to {args.output} and listing datasets...") - tic = time.time() - mastertic = tic - with h5py.File(args.input, "r") as ifile, h5py.File(args.output, "r+") as ofile: - h5copy = H5copier(ifile, ofile) - ifile.visititems(h5copy) - original_size = h5copy.get_total_size() - original_size_bytes = h5copy.get_total_size_bytes() - total_time = h5copy.get_total_time() - toc = time.time() - print(f"File structure copy took {1000.*(toc-tic):.2f} ms.") + if comm_rank == 0: + try: + print(f"Creating output file at {args.output}") + # Setting hdf5 version of file + fapl = h5py.h5p.create(h5py.h5p.FILE_ACCESS) + fapl.set_libver_bounds(h5py.h5f.LIBVER_V18, h5py.h5f.LIBVER_LATEST) + fid = h5py.h5f.create( + args.output.encode("utf-8"), flags=h5py.h5f.ACC_TRUNC, fapl=fapl + ) - tmpdir = ( - f"{args.scratch}/{os.path.basename(args.output).removesuffix('.hdf5')}_temp" - ) - print( - f"Copying over datasets to temporary files in {tmpdir} using {args.nproc} processes..." - ) + print(f"Creating groups and datasets in output file") + tic = time.time() + with ( + h5py.File(args.input, "r") as ifile, + h5py.File(args.output, "r+") as ofile, + ): + h5copy = H5copier(ifile, ofile) + ifile.visititems(h5copy) + original_size = h5copy.get_total_size() + original_size_bytes = h5copy.get_total_size_bytes() + total_time = h5copy.get_total_time() + toc = time.time() + print(f"File structure copy took {toc-tic:.2f} s.") + + tmp_dir = f"{args.scratch}/{os.path.basename(args.output).removesuffix('.hdf5')}_temp" + os.makedirs(tmp_dir, exist_ok=True) + + datasets = h5copy.dsets.copy() + + except Exception as e: + print(f"Error: {e}") + comm.Abort(1) + else: + tmp_dir = None + datasets = None + tmp_dir = comm.bcast(tmp_dir) + datasets = comm.bcast(datasets) + + if comm_rank == 0: + print( + f"Creating compressed datasets in temporary files using {comm_size} ranks" + ) tic = time.time() - os.makedirs(tmpdir, exist_ok=True) - - arguments = [] - for dset in h5copy.dsets: - arguments.append((args.input, f"{tmpdir}/{dset.replace('/','_')}.hdf5", dset)) - - pool = mp.Pool(args.nproc) - count = 0 - ntot = len(arguments) - for dset in pool.imap_unordered(compress_dataset, arguments): - count += 1 - print(f"[{count:04d}/{ntot:04d}] {dset}".ljust(110), end="\r") + i_start, i_end = assign_datasets(len(datasets), comm_size, comm_rank) + for i_dset, dset in enumerate(sorted(datasets)[i_start:i_end]): + tmp_output = f"{tmp_dir}/{dset.replace('/','_')}.hdf5" + compress_dataset(args.input, tmp_output, dset) + print(f"{comm_rank}: [{i_dset+1:04d}/{i_end-i_start:04d}] {dset}") + + comm.barrier() toc = time.time() - print(f"Temporary file writing took {1000.*(toc-tic):.2f} ms.".ljust(110)) + if comm_rank == 0: + print(f"Temporary file writing took {toc-tic:.2f} s.") + print(f"Copying datasets into {args.output}") - print(f"Copying datasets into {args.output} and cleaning up temporary files...") + # Only the first rank writes the final output file tic = time.time() - count = 0 - ntot = len(arguments) - with h5py.File(args.output, "r+") as ofile: - for _, tmpfile, dset in arguments: - count += 1 - print(f"[{count:04d}/{ntot:04d}] {dset}".ljust(110), end="\r") - with h5py.File(tmpfile, "r") as ifile: - ifile.copy(ifile["data"], ofile, dset) - - shutil.rmtree(tmpdir) + if comm_rank == 0: + with h5py.File(args.output, "r+") as ofile: + for i_dset, dset in enumerate(datasets): + tmp_output = f"{tmp_dir}/{dset.replace('/','_')}.hdf5" + with h5py.File(tmp_output, "r") as ifile: + ifile.copy(ifile["data"], ofile, name=dset) + print(f"{comm_rank}: [{i_dset+1:04d}/{len(datasets):04d}] {dset}") + + comm.barrier() toc = time.time() mastertoc = toc - print(f"Temporary file copy took {1000.*(toc-tic):.2f} ms.".ljust(110)) - - with h5py.File(args.output, "r") as ofile: - h5print = H5printer(False) - ofile.visititems(h5print) - new_size = h5print.get_total_size() - new_size_bytes = h5print.get_total_size_bytes() - - print( - f"{original_size} -> {new_size} ({100.*new_size_bytes/original_size_bytes:.2f}%)" - ) - print(f"Total writing time: {1000.*(mastertoc-mastertic):.2f} ms.") + if comm_rank == 0: + print(f"Writing output file took {toc-tic:.2f} s.") + print("Removing temporary files") + shutil.rmtree(tmp_dir) + + with h5py.File(args.output, "r") as ofile: + h5print = H5printer(False) + ofile.visititems(h5print) + new_size = h5print.get_total_size() + new_size_bytes = h5print.get_total_size_bytes() + + frac = new_size_bytes / original_size_bytes + print(f"{original_size} -> {new_size} ({100.*frac:.2f}%)") + print(f"Total writing time: {mastertoc-mastertic:.2f} s.") + print("Done") diff --git a/compression/create_empty_SOAP_catalogue.py b/compression/create_empty_SOAP_catalogue.py index 94c13478..17079f8e 100644 --- a/compression/create_empty_SOAP_catalogue.py +++ b/compression/create_empty_SOAP_catalogue.py @@ -161,8 +161,10 @@ def __call__(self, name, h5obj): assert not os.path.exists(args.outputSOAP) - with h5py.File(args.referenceSOAP, "r") as ifile, h5py.File( - args.snapshot, "r" - ) as snapfile, h5py.File(args.outputSOAP, "w") as ofile: + with ( + h5py.File(args.referenceSOAP, "r") as ifile, + h5py.File(args.snapshot, "r") as snapfile, + h5py.File(args.outputSOAP, "w") as ofile, + ): h5copy = H5copier(ifile, snapfile, ofile, snapnum) ifile.visititems(h5copy) diff --git a/compression/extract_filters.py b/compression/extract_filters.py index 4e6a5b90..2bdee3e1 100644 --- a/compression/extract_filters.py +++ b/compression/extract_filters.py @@ -16,7 +16,7 @@ The chunk size is included in the filter that is output, meaning the output will differ slightly for different SWIFT snapshots. However, this doesn't matter -since we set the chunk size explicitly when we create compressed SOAP catalogues. +since we set the chunk size explicitly when we create compressed SOAP catalogues. """ import sys diff --git a/make_virtual_snapshot.py b/compression/make_virtual_snapshot.py similarity index 50% rename from make_virtual_snapshot.py rename to compression/make_virtual_snapshot.py index 2e065cab..5f6e37f8 100644 --- a/make_virtual_snapshot.py +++ b/compression/make_virtual_snapshot.py @@ -12,12 +12,32 @@ def make_virtual_snapshot(snapshot, membership, output_file, snap_nr): """ # Check which datasets exist in the membership files + # and store their attributes and datatype filename = membership.format(file_nr=0, snap_nr=snap_nr) + dset_attrs = {} + dset_dtype = {} with h5py.File(filename, "r") as infile: - have_grnr_bound = "GroupNr_bound" in infile["PartType1"] - have_grnr_all = "GroupNr_all" in infile["PartType1"] - have_rank_bound = "Rank_bound" in infile["PartType1"] - have_fof_id = "FOFGroupIDs" in infile["PartType1"] + for ptype in range(7): + if not f"PartType{ptype}" in infile: + continue + dset_attrs[f"PartType{ptype}"] = {} + dset_dtype[f"PartType{ptype}"] = {} + for dset in infile[f"PartType{ptype}"].keys(): + attrs = dict(infile[f"PartType{ptype}/{dset}"].attrs) + dtype = infile[f"PartType{ptype}/{dset}"].dtype + + # Some membership files are missing these attributes + if not "Value stored as physical" in attrs: + print(f"Setting comoving attrs for PartType{ptype}/{dset}") + attrs["Value stored as physical"] = [1] + attrs["Property can be converted to comoving"] = [0] + + # Add a flag that these are stored in the membership files + attrs["Auxilary file"] = [1] + + # Store the values we need for later + dset_attrs[f"PartType{ptype}"][dset] = attrs + dset_dtype[f"PartType{ptype}"][dset] = dtype # Copy the input virtual snapshot to the output shutil.copyfile(snapshot, output_file) @@ -29,20 +49,26 @@ def make_virtual_snapshot(snapshot, membership, output_file, snap_nr): file_nr = 0 filenames = [] shapes = [] - dtype = None + counts = [] while True: filename = membership.format(file_nr=file_nr, snap_nr=snap_nr) if os.path.exists(filename): filenames.append(filename) with h5py.File(filename, "r") as infile: shape = {} + count = {} for ptype in range(7): - name = f"PartType{ptype}" - if name in infile: - shape[ptype] = infile[name]["GroupNr_bound"].shape - if dtype is None: - dtype = infile[name]["GroupNr_bound"].dtype + if f"PartType{ptype}" not in dset_attrs: + continue + shape[f"PartType{ptype}"] = {} + # Get the shape for each dataset + for dset in dset_attrs[f"PartType{ptype}"]: + s = infile[f"PartType{ptype}/{dset}"].shape + shape[f"PartType{ptype}"][dset] = s + # Get the number of particles in this chunk file + count[f"PartType{ptype}"] = s[0] shapes.append(shape) + counts.append(count) else: break file_nr += 1 @@ -51,67 +77,51 @@ def make_virtual_snapshot(snapshot, membership, output_file, snap_nr): # Loop over particle types in the output for ptype in range(7): - name = f"PartType{ptype}" - if name in outfile: - # Create virtual layout for new datasets - nr_parts = sum([shape[ptype][0] for shape in shapes]) - full_shape = (nr_parts,) - if have_grnr_all: - layout_grnr_all = h5py.VirtualLayout(shape=full_shape, dtype=dtype) - if have_grnr_bound: - layout_grnr_bound = h5py.VirtualLayout(shape=full_shape, dtype=dtype) - if have_rank_bound: - layout_rank_bound = h5py.VirtualLayout(shape=full_shape, dtype=dtype) - # PartType6 (neutrinos) are not assigned a FOF group - if have_fof_id and (ptype != 6): - layout_fof_id = h5py.VirtualLayout(shape=full_shape, dtype=dtype) - # Loop over input files - offset = 0 - for (filename, shape) in zip(filenames, shapes): - count = shape[ptype][0] - if have_grnr_all: - layout_grnr_all[offset : offset + count] = h5py.VirtualSource( - filename, f"PartType{ptype}/GroupNr_all", shape=shape[ptype] - ) - if have_grnr_bound: - layout_grnr_bound[offset : offset + count] = h5py.VirtualSource( - filename, f"PartType{ptype}/GroupNr_bound", shape=shape[ptype] - ) - if have_rank_bound: - layout_rank_bound[offset : offset + count] = h5py.VirtualSource( - filename, f"PartType{ptype}/Rank_bound", shape=shape[ptype] - ) - if have_fof_id and (ptype != 6): - layout_fof_id[offset : offset + count] = h5py.VirtualSource( - filename, f"PartType{ptype}/FOFGroupIDs", shape=shape[ptype] - ) - offset += count - # Create the virtual datasets - if have_grnr_all: - outfile.create_virtual_dataset( - f"PartType{ptype}/GroupNr_all", layout_grnr_all, fillvalue=-999 - ) - if have_grnr_bound: - outfile.create_virtual_dataset( - f"PartType{ptype}/GroupNr_bound", layout_grnr_bound, fillvalue=-999 + if f"PartType{ptype}" not in dset_attrs: + continue + + # Create virtual layout for new datasets + layouts = {} + nr_parts = sum([count[f"PartType{ptype}"] for count in counts]) + for dset in dset_attrs[f"PartType{ptype}"]: + full_shape = list(shapes[0][f"PartType{ptype}"][dset]) + full_shape[0] = nr_parts + full_shape = tuple(full_shape) + dtype = dset_dtype[f"PartType{ptype}"][dset] + layouts[dset] = h5py.VirtualLayout(shape=full_shape, dtype=dtype) + + # Loop over input files + offset = 0 + for filename, count, shape in zip(filenames, counts, shapes): + n_part = count[f"PartType{ptype}"] + for dset in dset_attrs[f"PartType{ptype}"]: + layouts[dset][offset : offset + n_part] = h5py.VirtualSource( + filename, + f"PartType{ptype}/{dset}", + shape=shape[f"PartType{ptype}"][dset], ) - # Copy GroupNr_bound to HaloCatalogueIndex, since that is the name in SOAP + offset += n_part + + # Create the virtual datasets, renaming datasets if they + # already exist in the snapshot + for dset, attrs in dset_attrs[f"PartType{ptype}"].items(): + if f"PartType{ptype}/{dset}" in outfile: + outfile.move(f"PartType{ptype}/{dset}", f"PartType{ptype}/{dset}_snap") + outfile.create_virtual_dataset( + f"PartType{ptype}/{dset}", layouts[dset], fillvalue=-999 + ) + for k, v in attrs.items(): + outfile[f"PartType{ptype}/{dset}"].attrs[k] = v + + # Copy GroupNr_bound to HaloCatalogueIndex, since that is the name in SOAP + if dset == "GroupNr_bound": outfile.create_virtual_dataset( - f"PartType{ptype}/HaloCatalogueIndex", layout_grnr_bound, fillvalue=-999 + f"PartType{ptype}/HaloCatalogueIndex", + layouts["GroupNr_bound"], + fillvalue=-999, ) for k, v in outfile[f"PartType{ptype}/GroupNr_bound"].attrs.items(): outfile[f"PartType{ptype}/HaloCatalogueIndex"].attrs[k] = v - if have_rank_bound: - outfile.create_virtual_dataset( - f"PartType{ptype}/Rank_bound", layout_rank_bound, fillvalue=-999 - ) - if have_fof_id and (ptype != 6): - outfile.move( - f"PartType{ptype}/FOFGroupIDs", f"PartType{ptype}/FOFGroupIDs_old" - ) - outfile.create_virtual_dataset( - f"PartType{ptype}/FOFGroupIDs", layout_fof_id, fillvalue=-999 - ) # Done outfile.close() @@ -124,7 +134,11 @@ def make_virtual_snapshot(snapshot, membership, output_file, snap_nr): # For description of parameters run the following: $ python make_virtual_snapshot.py --help parser = argparse.ArgumentParser( - description="Link SWIFT snapshots with SOAP membership files" + description=( + "Link SWIFT snapshots with SWIFT auxilary snapshots (snapshot-like" + "files with the same number of particles in the same order as the" + "snapshot, but with less metadata), such as the SOAP memberships" + ) ) parser.add_argument( "virtual_snapshot", diff --git a/update_vds_paths.py b/compression/update_vds_paths.py similarity index 85% rename from update_vds_paths.py rename to compression/update_vds_paths.py index a0e4105a..da24d57d 100644 --- a/update_vds_paths.py +++ b/compression/update_vds_paths.py @@ -83,11 +83,21 @@ def replace_membership_path(old_path): for dset in all_datasets: if dset.is_virtual: name = dset.name.split("/")[-1] - # Data comes from the membership files - if name in ("GroupNr_all", "GroupNr_bound", "Rank_bound", "HaloCatalogueIndex"): + # Check if the dataset comes from a membership file + if dset.attrs.get("Auxilary file", [0])[0] == 1: if membership_dir is not None: update_vds_paths(dset, replace_membership_path) - # FOF IDs come from membership files + # Catch old datasets which didn't have the "Auxilary file" set + elif name in ( + "GroupNr_all", + "GroupNr_bound", + "Rank_bound", + "HaloCatalogueIndex", + "SpecificPotentialEnergies", + ): + if membership_dir is not None: + update_vds_paths(dset, replace_membership_path) + # Catch old case of FOF IDs from membership files elif (name == "FOFGroupIDs") and ("PartType1/FOFGroupIDs_old" in f): if membership_dir is not None: update_vds_paths(dset, replace_membership_path) diff --git a/compression/wrong_compression.yml b/compression/wrong_compression.yml index c9413436..722ccdb0 100644 --- a/compression/wrong_compression.yml +++ b/compression/wrong_compression.yml @@ -1,2 +1,2 @@ -# Add entry as "PropertyName": "FilterName" -# e.g. MostMassiveBlackHolePosition: DScale5 +# Add entry as "PropertyName": "FilterName", e.g. +# MostMassiveBlackHolePosition: DScale5 diff --git a/create_groups.py b/create_groups.py deleted file mode 100644 index 71b62c97..00000000 --- a/create_groups.py +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/env python - -import os.path - - -def find_groups_to_create(paths): - """ - Given a list of paths to HDF5 objects, return a list of the names - of the groups which must be created in the order in which to create - them. - """ - - groups_to_create = set() - for path in paths: - dirname = path - while True: - dirname = os.path.dirname(dirname) - if len(dirname) > 0: - groups_to_create.add(dirname) - else: - break - groups_to_create = list(groups_to_create) - groups_to_create.sort(key=lambda x: len(x.split("/"))) - return groups_to_create diff --git a/documentation/SOAP.tex b/documentation/SOAP.tex index 13fd3eb5..a625d5db 100644 --- a/documentation/SOAP.tex +++ b/documentation/SOAP.tex @@ -1,7 +1,7 @@ \documentclass{article} \usepackage[utf8]{inputenc} \usepackage{longtable} -\usepackage{a4wide} +\usepackage[a4paper, margin=1in]{geometry} \usepackage{amsmath} \usepackage{amssymb} \usepackage{pifont} @@ -15,7 +15,7 @@ \input{units} \title{SOAP -- Spherical Overdensity and Aperture Processor} -\author{Bert Vandenbroucke, Joop Schaye, John Helly, \\Matthieu Schaller, Rob McGibbon} +\author{Rob McGibbon, Bert Vandenbroucke, John Helly, \\Joop Schaye, Matthieu Schaller} \date{} \begin{document} @@ -26,16 +26,25 @@ \section{Introduction} -SOAP computes different types of properties, depending on how particles are included (by radius, in projection...). -For all types, we use the halo membership and halo centre as determined by the input halo catalogue. -This documentation is generated using the SOAP parameter file, and so the properties listed reflect those -present in the current run of SOAP, rather than all possible properties. - -For loading SOAP catalogues we recommend using the swiftsimio package. A simple example is: +SOAP computes properties for different types of halos, which differ in how they decide +on which particles to included (by radius, in projection...). SOAP does not identify +halos itself, instead it uses the halo centre and the boundedness of particles as determined +by an input subhalo catalogue (such as HBT-HERONS, SubFind, etc...). +The purpose of this file is to document the catalogues output by SOAP. For information about running +the code you should refer to the README at \href{https://github.com/SWIFTSIM/SOAP}{https://github.com/SWIFTSIM/SOAP}. +This documentation is generated using the SOAP parameter file, and so the halo types and properties +listed reflect those present in the current run of SOAP, rather than all possible properties. +If you have any questions regarding the halo types or properties mentioned in the document then +please send an email to \href{mailto:mcgibbon@strw.leidenuniv.nl}{mcgibbon@strw.leidenuniv.nl}. +It would be useful for us to know which parts of the document need clarifying. + +SOAP catalogues are output as HDF5 files, with the properties being grouped by halo type. +For loading the catalogues we recommend using the swiftsimio package. A simple example is: \begin{verbatim} import swiftsimio as sw -soap_dir = '/cosma8/data/dp004/flamingo/Runs/L1000N1800/HYDRO_FIDUCIAL/SOAP-HBT/' -data = sw.load(f'{soap_dir}halo_properties_0077.hdf5') +base_dir = '/cosma8/data/dp004/flamingo/Runs' +run = 'L1000N1800/HYDRO_FIDUCIAL' +data = sw.load(f'{base_dir}/{run}/SOAP-HBT/halo_properties_0077.hdf5') # Load a dataset m200 = data.spherical_overdensity_200_crit.total_mass # Specify units @@ -46,78 +55,82 @@ \section{Introduction} z = data.metadata.redshift \end{verbatim} -\section{Property types} +\section{Particle selection for different halo types} + +SOAP computes properties for many different definitions of halo types. The halo type defines how to select the +particles to use when computing the properties. There are two decisions to be made - firstly whether +to include particles which are unbound. Secondly whether to exclude particles which are outside of a certain +aperture. -\paragraph{Subhalo quantities (SH)} are computed for each subhalo identified by the halo finder, irrespective of whether -it is a field halo or a satellite (or even satellite of satellite and so on). They include all particles +\paragraph{Bound subhalo quantities (SH)} are computed for each subhalo identified by the halo finder, irrespective of whether +it is a central or a satellite (or even satellite of satellite and so on). They include all particles that they halo finder has determined are bound to the subhalo. Subhalo properties are contained within the group \verb+BoundSubhalo+ in the output file. -\paragraph{Exclusive sphere quantities (ES)} are similar to subhalo quantities, but they include only the -particles that are bound to the subhalo, and apply an additional radial cut (aperture). We use eight -different aperture radii (10, 30, 50, 100, 300, 500, 1000, 3000 kpc), so that every (sub-)halo has eight of -these. Exclusive sphere properties are contained within a group \verb+ExclusiveSphere/XXXkpc+, where -\verb+XXX+ is the corresponding aperture radius. - -\paragraph{Inclusive sphere quantities (IS)} use the same physical aperture radii as the exclusive sphere -quantities, but include all particles within the radius, regardless of their membership status. They are -stored within a group \verb+InclusiveSphere/XXXkpc+. - -\paragraph{Exclusive projected quantities (EP)} are similar to exclusive sphere quantities, except that their -aperture filter is applied in projection, and this for independent projections along the x-, y- and z-axis. -Along the projection axis, we do not apply any radial cut, so that the depth corresponds to all particles -bound to the (sub-)halo. With four projected aperture radii (10, 30, 50, 100 kpc), we then have twelve sets of -projected aperture quantities for each (sub-)halo. Projected aperture quantities are stored in a group named -\verb+ProjectedAperture/XXXkpc/projP+, where \verb+XXX+ is the corresponding aperture radius, and \verb+P+ +\paragraph{Exclusive sphere quantities (ES)} are similar to subhalo quantities as they include only the +particles that are bound to the subhalo, but they apply an additional radial cut (aperture). Exclusive sphere +properties are contained within the group \verb+ExclusiveSphere+. Groups with the label \verb+XXXkpc+ have an +aperture radius of \verb+XXX+ \textbf{physical} kpc. Groups with a property name have a cut using the value +of the property calculated for the bound subhalo (e.g. the aperture cut for \verb+ExclusiveSphere/HalfMassRadiusTotal+ +is given by the value of \verb+BoundSubhalo/HalfMassRadiusTotal+), meaning a different radial cut is used for each subhalo. + +\paragraph{Inclusive sphere quantities (IS)} apply a aperture radii cut, the same as the exclusive sphere +quantities. However for the inclusive sphere we include all particles within the radius, regardless of their +membership status. If the aperture radius of an inclusive sphere variation is greater than the EncloseRadius +(the maximum distance between a bound particle and the halo centre) of a subhalo, then for that subhalo no +properties are computed for the variation. +The quantities are stored within the group \verb+InclusiveSphere+. + +\paragraph{Exclusive projected quantities (EP)} are similar to exclusive sphere quantities, except that their +aperture cut is applied in projection. For each radii there are three independent projections: along the +x-, y- and z-axis. Along the projection axis, we do not apply any radial cut, meaning the depth corresponds to all particles +bound to the subhalo. Projected aperture quantities are stored in a group named +\verb+ProjectedAperture/XXX/projP+, where \verb+XXX+ is the corresponding aperture cut, and \verb+P+ corresponds to a particular projection direction (\verb+x+, \verb+y+ or \verb+z+). \paragraph{Spherical overdensity properties (SO)} are fundamentally different from the three other types in that their aperture radius is determined from the density profile and is different for different halos. They always include all particles within a sphere around the halo centre, regardless of halo membership. -The radius is either the radius at which the density reaches a certain target value (50 crit, 100 crit, 200 -crit, 500 crit, 1000 crit, 2500 crit, 200 mean, BN98) or a multiple of such a radius (5xR 500 crit). Details -of the spherical overdensity calculation are given at the end of this document. Spherical overdensities are -only computed for centrals, i.e. field halos. The inclusive sphere quantities are stored in a group +The radius is either the radius at which the density reaches a certain target value (e.g. 200 +crit, 100 crit, 200 mean, BN98) or a multiple of such a radius (5xR 500 crit). Details +of the spherical overdensity calculation are given in section \ref{sec:so_calculation}. Spherical overdensities are +only computed for central subhalos, i.e. field halos. The quantities are stored in a group \verb+SO/XXX+, where \verb+XXX+ can be either \verb+XXX_mean+ for density multiples of the mean density, \verb+XXX_crit+ for density multiples of the critical density, \verb+BN98+ for the overdensity definition of -Bryan \& Norman (1998), and \verb+YxR_XXX_ZZZ+ for multiples of some other radius (e.g. \verb+5xR_2500_mean+). +Bryan \& Norman (1998), or \verb+YxR_XXX_ZZZ+ for multiples of some other radius (e.g. \verb+5xR_2500_mean+). The latter can only be computed after the corresponding density multiple SO radius has been computed. This is achieved by ordering the calculations. \paragraph{InputHalos} Some properties are directly copied from the original halo catalogue that was passed to SOAP. These are stored in a separate group, \verb+InputHalos+. -\paragraph{SOAP} Some properties are computed by SOAP using the other halo properties -present in the catalogue. -These are stored in a separate group, \verb+SOAP+. This is just done for convenience; these quantities can be computed from the SOAP output alone. +\paragraph{SOAP} Some properties are computed by SOAP using the other halo properties present in the catalogue. +These are stored in a separate group, \verb+SOAP+. This is just done for convenience; these quantities could be computed from the SOAP output alone. \paragraph{The table below lists} all the groups in the output file which containing datasets. Note that there will be three groups (\verb+x+, \verb+y+ or \verb+z+) for each \verb+ProjectedAperture+ variation. -Each halo variation can have a filter applied to it. If a halo does not satisfy the filter then the variation will -not be calculated for that halo. More information on filters can be found in the next section. +Each halo variation can have a filter applied to it. If a subhalo does not satisfy the filter then the halo variation +will be skipped for that subhalo, and all properties will have a value of zero. More information on filters can be found in the next section. \input{variations_table} -\section{Property categories} +\section{Filters} Halo properties only make sense if the subhalo contains sufficient particles. Halo finders are often run with a -configuration that requires at least 20 particles for a satellite subhalo. +configuration that requires at least 20 particles for a subhalo. However, even for those particle numbers, a lot of the properties computed by SOAP will be zero (e.g. the gas mass within a 10 kpc aperture), or have values that are outliers compared to the full halo population because of undersampling. We can save a lot of disk space by filtering these out by applying appropriate cuts. -Filtering means setting the value of the property to \verb+NaN+; HDF5 file compression then very effectively +Filtering means setting the value of the property to zero; HDF5 file compression then very effectively reduces the data storage required to store these properties, while the size of the arrays that the end user -sees remains unchanged. Evidently, we can also save on computing time by not computing properties that are +sees remains unchanged. We also save on computing time by not computing properties that are filtered out. -Since different properties can have very different requirements, filtering is done in categories, where each -category corresponds to a set of quantities that are filtered using the same criterion. Inclusive, exclusive -or projected quantities with different aperture radii (or overdensity criteria) can be used to create -profiles. In order for these profiles to make sense, we have to apply a consistent cut across all the -different aperture radii (or overdensity criteria) for the same subhalo property type. Or in other words: the -quantities for an inclusive sphere with a 10 kpc aperture radius will use the same filter mask as the -quantities of the inclusive sphere with a 3000 kpc aperture radius, even though the latter by construction has -many more particles. +Since different properties can have very different requirements, and so each property has a filter associated +with it. These filters are consistent across different halos types for the same property, e.g. the properties +for an inclusive sphere with a 10 kpc aperture radius will use the same filters as the quantities of the inclusive +sphere with a 3000 kpc aperture radius, even +though the latter by construction has many more particles. \paragraph{Basic quantities (basic)} are never filtered out, and hence are calculated for all objects in the input halo catalogue. @@ -136,8 +149,7 @@ \section{Property categories} bound to the subhalo. \paragraph{}Note that there are no quantities that use a BH or neutrino particle number filter. - -The particle number thresholds are set in the parameter file. The different categories are summarised in the table below. +The different categories are summarised in the table below. \begin{longtable}{ll} Name & criterion \\ @@ -149,21 +161,26 @@ \section{Property categories} baryon & $N_{\rm{}gas}+N_{\rm{}star} \geq{} \baryonfilter$ \\ \end{longtable} -\section{Overview table} +\section{Property table} The table below lists all the properties that are computed by SOAP when run in HYDRO mode. -For dark matter only (DMO) mode only the properties colored violet/purple are computed. -This table is automatically generated by -SOAP from the source code, so that all names, types, units, categories and descriptions match what is actually -used and output by SOAP. For each quantity, the table indicates for which halo types the property is computed. +For dark matter only (DMO) mode only the properties coloured violet/purple are computed. +This table is automatically generated by SOAP from the source code and parameter file, so that all +names, types, units, categories and descriptions match what is actually +used and output by SOAP. Superscript numbers refer to more detailed explanations for some of the properties and match the numbers in the next section. If swiftsimio has been used to load a catalogue then the fields names are in snake\_case rather -than CamelCase, e.g. \verb+CentreOfMass+ becomes \verb+centre_of_mass+. +than CamelCase, e.g. \verb+CentreOfMass+ becomes \verb+centre_of_mass+. For each quantity, the table +indicates for which halo types the property is computed. If a property is being calculated +for a certain halo type it is marked with a \ding{51}. Properties calculated +for snapshots but not snipshots are indicated with a \ding{36}. -Note that quantities are given in the base units of the simulation snapshot. The attributes of each SOAP dataset contains -all the relevant meta-data to convert between physical and co-moving units, i.e. information about how the -quantity depends on the scale-factor, and what the conversion factor to and from CGS units is. All quantities -are $h$-free. The conversion of the base units to CGS is given by: +\paragraph{}Note that quantities are output in the base units of the simulation snapshot. +We therefore recommend that swiftsimio is used to load the catalogues as it has unit handling built in. +However, the attributes of each SOAP dataset do contain all the relevant meta-data to convert between physical +and co-moving units, i.e. information about how the +quantity depends on the scale-factor, and what the conversion factor to and from CGS units is. \textbf{All quantities +are $h$-free.} The conversion of the base units to CGS is given by: \begin{longtable}{cl} Unit & CGS conversion \\ @@ -176,10 +193,10 @@ \section{Overview table} For example, a property whose units are listed as L/t will have units of velocity, where $1 \, \rm{L/t} = \velbaseunit \, \rm{km/s}$. -The scale factor is explicitly included for comoving properties (e.g. the units of HaloCentre are aL) +The scale factor is explicitly included for comoving properties (e.g. the units of \verb+HaloCentre+ are +listed as aL). -If a property is being calculated for a certain halo type it is marked with a \ding{51}. Properties calculated -for snapshots but not snipshots are indicated with a \ding{36}. +Properties are computed using the value of \verb+HaloCentre+ as the centre of the subhalo, unless otherwise stated. \input{table} @@ -188,6 +205,7 @@ \section{Non-trivial properties} \input{footnotes} \section{Spherical overdensity calculations} +\label{sec:so_calculation} The radius at which the density reaches a certain threshold value is found by linear interpolation of the cumulative mass profile obtained after sorting the particles by radius. The approach we use is different from @@ -281,7 +299,7 @@ \section{Group membership files} The group membership files are HDF5 files with one group for each particle type, named PartType0, PartType1, ... as in the -snapshots. Each group contains the following datasets: +snapshots. The following datasets can be contained in the membership files: \begin{enumerate} \item \verb|GroupNr_bound|: for each particle in the corresponding snapshot @@ -294,25 +312,24 @@ \section{Group membership files} the subhalo it belongs to, or -1 if the particle is not bound to any subhalo. The particle with the most negative total energy has \verb|Rank_bound|=0. -\item \verb|GroupNr_all|: (VELOCIraptor only) for each particle in the - corresponding snapshot - file this contains the array index of the VR group which the - particle belongs to, regardless of whether it is bound or - unbound. Particles in no group have \verb|GroupNr_all|=-1. \item \verb|FOFGroupIDs|: the 3D FOF group the particle is part of. This field is only present if a FOF snapshot is listed in the parameter file. This field is present in the snapshots themselves, but for FLAMINGO hydro simulations the FOF was regenerated. If this field is present it will overwrite the value from the snapshots when SOAP is run. +\item \verb|SpecificPotentialEnergies|: the specific potential energy of + each particle as calculated by the subhalo finder. Note that for HBT + these values may not have the same order as \verb|Rank_bound|. This is because + HBT has a centre refinement step which can reorder the particles in the centre + of the subhalo, but binding energies are not updated as part of this step. \end{enumerate} -The script `make\_virtual\_snapshot.py` will combine snapshot and group membership files +The script \verb|make_virtual_snapshot.py| will combine snapshot and group membership files into a single virtual snapshot file. This virtual file can be read by swiftsimio and gadgetviewer to provide halo membership information alongside other particle properties. Using the virtual file along with the spatial masking functionality within swiftsimio means it is possible to quickly load all the particles bound to a given subhalo. - The \href{https://swiftgalaxy.readthedocs.io/en/latest/index.html}{SWIFTGalaxy package} makes use of the membership files, and is helpful for analysing individual galaxies when manipulating their particles (recentering, rotations, cylindrical coordinates, etc.). diff --git a/documentation/footnote_AngMom.tex b/documentation/footnote_AngMom.tex index 77626a16..a0e6bfaf 100644 --- a/documentation/footnote_AngMom.tex +++ b/documentation/footnote_AngMom.tex @@ -1,4 +1,4 @@ -\paragraph{$^{$FOOTNOTE_NUMBER$}$The angular momentum} of gas, dark matter and stars is computed relative to +\paragraph{$^{$FOOTNOTE_NUMBER$}$The angular momentum}\label{footnote:$FOOTNOTE_NUMBER$} of gas, dark matter and stars is computed relative to the halo centre (cop) and the centre of mass velocity of that particular component, and not to the total centre of mass velocity. The full expression is diff --git a/documentation/footnote_Ekin.tex b/documentation/footnote_Ekin.tex index a79c3b8f..d1f1ea2d 100644 --- a/documentation/footnote_Ekin.tex +++ b/documentation/footnote_Ekin.tex @@ -1,3 +1 @@ -\paragraph{$^{$FOOTNOTE_NUMBER$}$The kinetic energy} of the gas and stars is computed using the same relative -velocities as used for other properties, i.e. relative to the centre of mass velocity of the gas and stars -respectively. +\paragraph{$^{$FOOTNOTE_NUMBER$}$The kinetic energy}\label{footnote:$FOOTNOTE_NUMBER$} is computed using the velocities relative to the centre of mass velocity of all the particles in the aperture. The Hubble flow is included when computing the velocities. diff --git a/documentation/footnote_Etherm.tex b/documentation/footnote_Etherm.tex index 082b308d..828dbe66 100644 --- a/documentation/footnote_Etherm.tex +++ b/documentation/footnote_Etherm.tex @@ -1,4 +1,4 @@ -\paragraph{$^{$FOOTNOTE_NUMBER$}$The thermal energy} of the gas is computed from the density and pressure, +\paragraph{$^{$FOOTNOTE_NUMBER$}$The thermal energy}\label{footnote:$FOOTNOTE_NUMBER$} of the gas is computed from the density and pressure, since the internal energy was not output in the FLAMINGO snapshots. The relevant equation is \begin{equation} diff --git a/documentation/footnote_MBH.tex b/documentation/footnote_MBH.tex index 30db1dc0..2aa71f9d 100644 --- a/documentation/footnote_MBH.tex +++ b/documentation/footnote_MBH.tex @@ -1,2 +1,2 @@ -\paragraph{$^{$FOOTNOTE_NUMBER$}$The most massive black hole} is identified based on the BH subgrid mass (i.e. +\paragraph{$^{$FOOTNOTE_NUMBER$}$The most massive black hole}\label{footnote:$FOOTNOTE_NUMBER$} is identified based on the BH subgrid mass (i.e. the same mass that goes into \verb+BlackHolesSubgridMass+). diff --git a/documentation/footnote_Mnu.tex b/documentation/footnote_Mnu.tex index 091ca38d..a94eef08 100644 --- a/documentation/footnote_Mnu.tex +++ b/documentation/footnote_Mnu.tex @@ -1,4 +1,4 @@ -\paragraph{$^{$FOOTNOTE_NUMBER$}$The neutrino masses} exist in two flavours. \verb+RawNeutrinoMass+ is +\paragraph{$^{$FOOTNOTE_NUMBER$}$The neutrino masses}\label{footnote:$FOOTNOTE_NUMBER$} exist in two flavours. \verb+RawNeutrinoMass+ is obtained by simply summing the neutrino particle masses, while the noise suppressed version, \verb+NoiseSuppressedNeutrinoMass+ is defined as diff --git a/documentation/footnote_SF.tex b/documentation/footnote_SF.tex index 15b2c631..645b6992 100644 --- a/documentation/footnote_SF.tex +++ b/documentation/footnote_SF.tex @@ -1,5 +1,5 @@ \paragraph{$^{$FOOTNOTE_NUMBER$}$When distinguishing between star-forming and non star-forming gas and -computing the total star formation rate,} we have to be careful about the interpretation of the +computing the total star formation rate,}\label{footnote:$FOOTNOTE_NUMBER$} we have to be careful about the interpretation of the \verb+StarFormationRates+ dataset in the snapshots, since negative values in that dataset are used to store another quantity, the last scale factor when that particular gas particle was star-forming. Star-forming gas is then gas for which \verb+StarFormationRates+ is strictly positive, and the total star formation rate is the diff --git a/documentation/footnote_Tgas.tex b/documentation/footnote_Tgas.tex index dcaea436..fa74ed0c 100644 --- a/documentation/footnote_Tgas.tex +++ b/documentation/footnote_Tgas.tex @@ -1,4 +1,4 @@ -\paragraph{$^{$FOOTNOTE_NUMBER$}$The mass-weighted temperature} is computed as +\paragraph{$^{$FOOTNOTE_NUMBER$}$The mass-weighted temperature}\label{footnote:$FOOTNOTE_NUMBER$} is computed as \begin{equation} T = \frac{1}{\sum_i m_i} \sum_i m_i T_i, diff --git a/documentation/footnote_Xray.tex b/documentation/footnote_Xray.tex index 534a2ff9..5d520053 100644 --- a/documentation/footnote_Xray.tex +++ b/documentation/footnote_Xray.tex @@ -1,4 +1,4 @@ -\paragraph{$^{$FOOTNOTE_NUMBER$}$X-ray quantities are} computed directly from the X-ray datasets in the +\paragraph{$^{$FOOTNOTE_NUMBER$}$X-ray quantities are}\label{footnote:$FOOTNOTE_NUMBER$} computed directly from the X-ray datasets in the snapshot. They are either in the emission rest-frame, or in the observed-frame of a $z=0$ observer, using the redshift of the snapshot as the emission redshift . The three bands are always given in the same order as in the snapshot: diff --git a/documentation/footnote_averaged.tex b/documentation/footnote_averaged.tex new file mode 100644 index 00000000..87fcbfdc --- /dev/null +++ b/documentation/footnote_averaged.tex @@ -0,0 +1,4 @@ +\paragraph{$^{$FOOTNOTE_NUMBER$}$Averaged quantities}\label{footnote:$FOOTNOTE_NUMBER$} are calculated by accumulating the quantity over +the 10Myr/100Myr that precedes the writing of a snapshot, and then normalising. For example for SFR we start a clock precisely 10Myr before a snapshot +dump, accumulate SFR * dt at each step during that window, and then divide by 10Myr at the point of writing. + diff --git a/documentation/footnote_circvel.tex b/documentation/footnote_circvel.tex index 1cefd59f..14b13bd9 100644 --- a/documentation/footnote_circvel.tex +++ b/documentation/footnote_circvel.tex @@ -1,4 +1,4 @@ -\paragraph{$^{$FOOTNOTE_NUMBER$}$The maximum circular velocity and the radius where it is reached} are +\paragraph{$^{$FOOTNOTE_NUMBER$}$The maximum circular velocity and the radius where it is reached}\label{footnote:$FOOTNOTE_NUMBER$} are computed using \begin{equation} diff --git a/documentation/footnote_cold_dense.tex b/documentation/footnote_cold_dense.tex new file mode 100644 index 00000000..5cee3c60 --- /dev/null +++ b/documentation/footnote_cold_dense.tex @@ -0,0 +1,2 @@ +\paragraph{$^{$FOOTNOTE_NUMBER$}$Cold dense gas particles}\label{footnote:$FOOTNOTE_NUMBER$} are particles with $T < 10^{$LOG_COLD_GAS_TEMP$}$K) and ($\frac{\rho}{m_{\mathrm{H}}} > 10^{$LOG_COLD_GAS_DENSITY$} \mathrm{cm}^{-3}$). + diff --git a/documentation/footnote_com.tex b/documentation/footnote_com.tex index c53c3a33..746d0894 100644 --- a/documentation/footnote_com.tex +++ b/documentation/footnote_com.tex @@ -1,2 +1,2 @@ -\paragraph{$^{$FOOTNOTE_NUMBER$}$The centre of mass and centre of mass velocity} are computed using all +\paragraph{$^{$FOOTNOTE_NUMBER$}$The centre of mass and centre of mass velocity}\label{footnote:$FOOTNOTE_NUMBER$} are computed using all particle types except neutrinos (since neutrinos can never be bound to a halo). diff --git a/documentation/footnote_compY.tex b/documentation/footnote_compY.tex index 7a8fcf66..61c42877 100644 --- a/documentation/footnote_compY.tex +++ b/documentation/footnote_compY.tex @@ -1,4 +1,4 @@ -\paragraph{$^{$FOOTNOTE_NUMBER$}$The Compton y parameter} is computed as in McCarthy et al. (2017): +\paragraph{$^{$FOOTNOTE_NUMBER$}$The Compton y parameter}\label{footnote:$FOOTNOTE_NUMBER$} is computed as in McCarthy et al. (2017): \begin{equation} y \, {d_A}^2(z) = \sum_i \frac{\sigma{}_T}{m_e c^2} n_{e,i} k_B T_{e,i} V_i, diff --git a/documentation/footnote_concentration.tex b/documentation/footnote_concentration.tex index 6904ed16..c1feee4e 100644 --- a/documentation/footnote_concentration.tex +++ b/documentation/footnote_concentration.tex @@ -1,4 +1,4 @@ -\paragraph{$^{$FOOTNOTE_NUMBER$}$The concentration} is computed using the +\paragraph{$^{$FOOTNOTE_NUMBER$}$The concentration}\label{footnote:$FOOTNOTE_NUMBER$} is computed using the method described in Wang et al. (2023), but using a fifth order polynomial fit to the R1-concentration relation for $1 0.1 \mathrm{cm}^{-3}$). +When calculating gas metallicity we only sum over particles which are cold ($T_i < 10^{$LOG_COLD_GAS_TEMP$}$K) and dense ($\frac{\rho_i}{m_{\mathrm{H}}} > 10^{$LOG_COLD_GAS_DENSITY$} \mathrm{cm}^{-3}$). LogarithmicMassWeighted properties, $Z_{\mathrm{log}}$, are calculated as \begin{equation} diff --git a/documentation/footnote_progenitor_descendant.tex b/documentation/footnote_progenitor_descendant.tex new file mode 100644 index 00000000..d34d4592 --- /dev/null +++ b/documentation/footnote_progenitor_descendant.tex @@ -0,0 +1 @@ +\paragraph{$^{$FOOTNOTE_NUMBER$}$The progenitor/descendant index}\label{footnote:$FOOTNOTE_NUMBER$} of a subhalo points to the subhalo in the previous/next snapshot which has the same HBT TrackId. Therefore this index can only be used to move up/down the main progenitor branch for a subhalo, it provides no information about subhalo mergers. diff --git a/documentation/footnote_proj_veldisp.tex b/documentation/footnote_proj_veldisp.tex index 3da5fae9..a38f5f26 100644 --- a/documentation/footnote_proj_veldisp.tex +++ b/documentation/footnote_proj_veldisp.tex @@ -1,2 +1,2 @@ -\paragraph{$^{$FOOTNOTE_NUMBER$}$The projected velocity dispersion} is computed along the projection axis. +\paragraph{$^{$FOOTNOTE_NUMBER$}$The projected velocity dispersion}\label{footnote:$FOOTNOTE_NUMBER$} is computed along the projection axis. Along this axis, the velocity is a 1D quantity, so that the velocity dispersion is simply 1 value. diff --git a/documentation/footnote_satfrac.tex b/documentation/footnote_satfrac.tex index 5ea82033..0eda2032 100644 --- a/documentation/footnote_satfrac.tex +++ b/documentation/footnote_satfrac.tex @@ -1,6 +1,6 @@ -\paragraph{$^{$FOOTNOTE_NUMBER$}$The satellite mass fractions} is obtained by summing the masses of all +\paragraph{$^{$FOOTNOTE_NUMBER$}$The satellite mass fractions}\label{footnote:$FOOTNOTE_NUMBER$} is obtained by summing the masses of all particles within the inclusive sphere that are bound to a subhalo that is not the central subhalo, and dividing this by $M_{\rm{}SO}$. This uses the same membership information that is also used to decide what particles need to be included in the exclusive sphere and projected aperture properties. For MassFractionSatellites we only consider particles with the same FOF ID as the most bound particle in the central subhalo. For -MassFractionExternal we include all particles with a FOF ID not equal to the most bound particle in the central subhalo. \ No newline at end of file +MassFractionExternal we include all particles with a FOF ID not equal to the most bound particle in the central subhalo. diff --git a/documentation/footnote_spectroscopicliketemperature.tex b/documentation/footnote_spectroscopicliketemperature.tex index 542b1c02..f33f073b 100644 --- a/documentation/footnote_spectroscopicliketemperature.tex +++ b/documentation/footnote_spectroscopicliketemperature.tex @@ -1,4 +1,4 @@ -\paragraph{$^{$FOOTNOTE_NUMBER$}$The spectroscopic-like temperature} is computed as +\paragraph{$^{$FOOTNOTE_NUMBER$}$The spectroscopic-like temperature}\label{footnote:$FOOTNOTE_NUMBER$} is computed as \begin{equation} T_{SL} = \frac{\sum_i \rho_i m_i T_i^{1/4}}{\sum_i \rho_i m_i T_i^{-3/4}} \end{equation} diff --git a/documentation/footnote_spin.tex b/documentation/footnote_spin.tex index 503dd58a..cf2ddc3c 100644 --- a/documentation/footnote_spin.tex +++ b/documentation/footnote_spin.tex @@ -1,4 +1,4 @@ -\paragraph{$^{$FOOTNOTE_NUMBER$}$The spin parameter} is computed following Bullock et al. (2021): +\paragraph{$^{$FOOTNOTE_NUMBER$}$The spin parameter}\label{footnote:$FOOTNOTE_NUMBER$} is computed following Bullock et al. (2021): \begin{equation} \lambda{} = \frac{|\vec{L}_{\rm{}tot}|}{\sqrt{2}M v_{\rm{}max} R}, diff --git a/documentation/footnote_tensor.tex b/documentation/footnote_tensor.tex index 9286757b..c3973b9b 100644 --- a/documentation/footnote_tensor.tex +++ b/documentation/footnote_tensor.tex @@ -1,10 +1,10 @@ -\paragraph{$^{$FOOTNOTE_NUMBER$}$The inertia tensor} for a set of particles is computed as +\paragraph{$^{$FOOTNOTE_NUMBER$}$The inertia tensor}\label{footnote:$FOOTNOTE_NUMBER$} for a set of particles is computed as \begin{equation} I_{ij} = \frac{1}{\sum_k m_k} \sum_k m_k \; r_{k,i} \; r_{k, j} \end{equation} -where the index $k$ loops over all particles, $m_k$ is the mass of particle $k$, and $r_{k, i}$ is the $i$-component of the position vector of particle $k$ relative to the halo centre. We first compute the inertia tensor using all particles within a sphere (with radius equal to the aperture size, except for subhalos where we use the half mass radius of the particles). This is the tensor we output in the non-iterative case. In the iterative case we construct an ellipsoid with a volume equal to the initial sphere, but whose shape is given by the inertia tensor. We then recalculate the inertia tensor using only the particles within the ellipsoid. This process is repeated until the value of the $q$ parameter converges, or we reach 20 iterations. For projected apertures the process is similar, except we use circles and ellipses in the projected plane to determine which particles to include. +where the index $k$ loops over all particles, $m_k$ is the mass of particle $k$, and $r_{k, i}$ is the $i$-component of the position vector of particle $k$ relative to the halo centre. We first compute the inertia tensor using all particles within a sphere (with radius equal to the aperture size, except for subhalos where we use the half mass radius of the particles). This is the tensor we output in the non-iterative case. In the iterative case we construct an ellipsoid with a volume equal to the initial sphere, but whose shape is given by the inertia tensor. We then recalculate the inertia tensor using only the particles within the ellipsoid. This process is repeated until the value of the $q$ parameter converges, or we reach 20 iterations. If at any point during the iterations there is only a single particle within the ellipsoid, we return zero. For projected apertures the process is similar, except we use circles and ellipses in the projected plane to determine which particles to include. The reduced inertia tensor is calculated as @@ -15,3 +15,5 @@ where $r_k$ is the radial distance of the particle. We do not calculate the inertia tensor if there are less than 20 particles within the initial sphere. + +For when calculating the inertia tensor for a bound subhalo we use a sphere with a radius equal to 10 times the half mass radius of the particles being considered. diff --git a/documentation/footnote_veldisp_matrix.tex b/documentation/footnote_veldisp_matrix.tex index 41124325..0b70f970 100644 --- a/documentation/footnote_veldisp_matrix.tex +++ b/documentation/footnote_veldisp_matrix.tex @@ -1,4 +1,4 @@ -\paragraph{$^{$FOOTNOTE_NUMBER$}$The velocity dispersion matrix} is defined as +\paragraph{$^{$FOOTNOTE_NUMBER$}$The velocity dispersion matrix}\label{footnote:$FOOTNOTE_NUMBER$} is defined as \begin{equation} V_{\rm{}disp,comp} = \frac{1}{\sum_{i={\rm{}comp}} m_i} \sum_{i={\rm{}comp}} m_i \vec{v}_{{\rm{}comp},r,i}\vec{v}_{{\rm{}comp},r,i}, diff --git a/format.sh b/format.sh index f02888fb..f532a46d 100755 --- a/format.sh +++ b/format.sh @@ -14,67 +14,20 @@ if [ ! -d black_formatting_env ] then echo "Formatting environment not found, installing it..." python3 -m venv black_formatting_env - ./black_formatting_env/bin/python3 -m pip install click==8.0.4 black==19.3b0 + ./black_formatting_env/bin/python3 -m pip install black fi # Now we know exactly which black to use black="./black_formatting_env/bin/python3 -m black" -# Formatting command -files=$(echo {*.py,compression/*.py}) -cmd="$black -t py38 $files" +# Make sure we don't try and format any virtual environments +files=$(echo {compression/*.py,misc/*.py,SOAP/*.py,SOAP/*/*.py,tests/*.py}) -# Print the help -function show_help { - echo -e "This script formats all Python scripts using black" - echo -e " -h, --help \t Show this help" - echo -e " -c, --check \t Test if the Python scripts are well formatted" -} - -# Parse arguments -TEST=0 -while [[ $# -gt 0 ]] -do - key="$1" - - case $key in - # print the help and exit - -h|--help) - show_help - exit - ;; - # check if the code is well formatted - -c|--check) - TEST=1 - shift - ;; - # unknown option - *) - echo "Argument '$1' not implemented" - show_help - exit - ;; - esac -done - -# Run the required commands -if [[ $TEST -eq 1 ]] -then - # Note trapping the exit status from both commands in the pipe. Also note - # do not use -q in grep as that closes the pipe on first match and we get - # a SIGPIPE error. - echo "Testing if Python scripts are correctly formatted" - $cmd --check - status=$? - - # Check formatting - if [[ ! ${status} -eq 0 ]] - then - echo "ERROR: needs formatting" - exit 1 - else - echo "Everything is correctly formatted" - fi +# Run formatting +if [[ "$1" == "--check" ]]; then + $black -t py310 $files --check else - echo "Formatting all Python scripts" - $cmd + $black -t py310 $files fi + +# Return the black return code +exit $? diff --git a/half_mass_radius.py b/half_mass_radius.py deleted file mode 100644 index a36638ce..00000000 --- a/half_mass_radius.py +++ /dev/null @@ -1,129 +0,0 @@ -#! /usr/bin/env python - -""" -half_mass_radius.py - -Utility functions to compute the half mass radius of a particle distribution. - -We put this in a separate file to facilitate unit testing. -""" - -import numpy as np -import unyt - - -def get_half_mass_radius( - radius: unyt.unyt_array, mass: unyt.unyt_array, total_mass: unyt.unyt_quantity -) -> unyt.unyt_quantity: - """ - Get the half mass radius of the given particle distribution. - - We obtain the half mass radius by sorting the particles on radius and then computing - the cumulative mass profile from this. We then determine in which "bin" the cumulative - mass profile intersects the target half mass value and obtain the corresponding - radius from linear interpolation. - - Parameters: - - radius: unyt.unyt_array - Radii of the particles. - - mass: unyt.unyt_array - Mass of the particles. - - total_mass: unyt.unyt_quantity - Total mass of the particles. Should be mass.sum(). We pass this on as an argument - because this value might already have been computed before. If it was not, then - computing it in the function call is still an efficient way to do this. - - Returns the half mass radius, defined as the radius at which the cumulative mass profile - reaches 0.5*total_mass. - """ - if total_mass == 0.0 * total_mass.units or len(mass) < 1: - return 0.0 * radius.units - - target_mass = 0.5 * total_mass - - isort = np.argsort(radius) - sorted_radius = radius[isort] - # compute sum in double precision to avoid numerical overflow due to - # weird unit conversions in unyt - cumulative_mass = mass[isort].cumsum(dtype=np.float64) - - # consistency check - # np.sum() and np.cumsum() use different orders, so we have to allow for - # some small difference - if cumulative_mass[-1] < 0.999 * total_mass: - raise RuntimeError( - "Masses sum up to less than the given total mass:" - f" cumulative_mass[-1] = {cumulative_mass[-1]}," - f" total_mass = {total_mass}!" - ) - - # find the intersection point - # if that is the first bin, set the lower limits to 0 - ihalf = np.argmax(cumulative_mass >= target_mass) - if ihalf == 0: - rmin = 0.0 * radius.units - Mmin = 0.0 * mass.units - else: - rmin = sorted_radius[ihalf - 1] - Mmin = cumulative_mass[ihalf - 1] - rmax = sorted_radius[ihalf] - Mmax = cumulative_mass[ihalf] - - # now get the radius by linearly interpolating - # if the bin edges coincide (two particles at exactly the same radius) - # then we simply take that radius - if Mmin == Mmax: - half_mass_radius = 0.5 * (rmin + rmax) - else: - half_mass_radius = rmin + (target_mass - Mmin) / (Mmax - Mmin) * (rmax - rmin) - - # consistency check - # we cannot use '>=', since equality would happen if half_mass_radius==0 - if half_mass_radius > sorted_radius[-1]: - raise RuntimeError( - "Half mass radius larger than input radii:" - f" half_mass_radius = {half_mass_radius}," - f" sorted_radius[-1] = {sorted_radius[-1]}!" - f" ihalf = {ihalf}, Npart = {len(radius)}," - f" target_mass = {target_mass}," - f" rmin = {rmin}, rmax = {rmax}," - f" Mmin = {Mmin}, Mmax = {Mmax}," - f" sorted_radius = {sorted_radius}," - f" cumulative_mass = {cumulative_mass}" - ) - - return half_mass_radius - - -def test_get_half_mass_radius(): - """ - Unit test for get_half_mass_radius(). - - We generate 1000 random particle distributions and check that the - half mass radius returned by the function contains less than half - the particles in mass. - """ - np.random.seed(203) - - for i in range(1000): - npart = np.random.choice([1, 10, 100, 1000, 10000]) - - radius = np.random.exponential(1.0, npart) * unyt.kpc - - Mpart = 1.0e9 * unyt.Msun - mass = Mpart * (1.0 + 0.2 * (np.random.random(npart) - 0.5)) - - total_mass = mass.sum() - - half_mass_radius = get_half_mass_radius(radius, mass, total_mass) - - mask = radius <= half_mass_radius - Mtest = mass[mask].sum() - assert Mtest <= 0.5 * total_mass - - fail = False - try: - half_mass_radius = get_half_mass_radius(radius, mass, 2.0 * total_mass) - except RuntimeError: - fail = True - assert fail diff --git a/io_test.py b/io_test.py deleted file mode 100644 index 2ee1d088..00000000 --- a/io_test.py +++ /dev/null @@ -1,98 +0,0 @@ -#!/bin/env python - -import time -import matplotlib.pyplot as plt -import numpy as np - -from mpi4py import MPI - -comm = MPI.COMM_WORLD -comm_rank = comm.Get_rank() -comm_size = comm.Get_size() - -import swift_cells -import shared_mesh -import pytest - - -@pytest.mark.mpi -def test_io(): - - comm.barrier() - t0 = time.time() - - # Open the snapshot - fname = "/cosma8/data/dp004/flamingo/Runs/L1000N0900/HYDRO_FIDUCIAL/snapshots/flamingo_0037/flamingo_0037.{file_nr}.hdf5" - try: - cellgrid = swift_cells.SWIFTCellGrid(fname) - except FileNotFoundError: - if comm_rank == 0: - print("File not found for running io_test") - return - - # Quantities to read - property_names = { - "PartType0": ("Coordinates", "Velocities", "Masses"), - "PartType1": ("Coordinates", "Velocities", "Masses"), - } - - # Specify region to read - pos_min = np.asarray((0.0, 0.0, 0.0)) * cellgrid.get_unit("snap_length") - pos_max = np.asarray((50.0, 50.0, 50.0)) * cellgrid.get_unit("snap_length") - - # Read in the region - mask = cellgrid.empty_mask() - cellgrid.mask_region(mask, pos_min, pos_max) - data = cellgrid.read_masked_cells_to_shared_memory(property_names, mask, comm, 8) - - comm.barrier() - t1 = time.time() - - # Find read rate - nbytes = 0 - for ptype in property_names: - for dataset in property_names[ptype]: - arr = data[ptype][dataset].full - nbytes += arr.data.nbytes - elapsed = t1 - t0 - - if comm_rank == 0: - rate = nbytes / elapsed / (1024 ** 3) - print("Read at %.2f GB/sec on %d ranks" % (rate, comm_size)) - - # Build the shared mesh - mesh = shared_mesh.SharedMesh( - comm, pos=data["PartType1"]["Coordinates"], resolution=256 - ) - if comm_rank == 0: - print("Built mesh") - - comm.barrier() - - if comm_rank == 0: - - # Plot all particles - pos = data["PartType1"]["Coordinates"] - plt.plot(pos.full[:, 0], pos.full[:, 1], "k,", alpha=0.05) - plt.gca().set_aspect("equal") - - # Try selecting a sphere - centre = np.asarray((30, 30, 30)) * cellgrid.get_unit("snap_length") - radius = 10 * cellgrid.get_unit("snap_length") - idx = mesh.query_radius_periodic(centre, radius, pos, cellgrid.boxsize) - plt.plot(pos.full[idx, 0], pos.full[idx, 1], "g,") - - plt.xlim(0, 150) - plt.ylim(0, 150) - plt.savefig(f"io_test.png", dpi=300) - plt.close() - - # Free the shared particle data - for ptype in data: - for name in data[ptype]: - data[ptype][name].free() - mesh.free() - - -if __name__ == "__main__": - test_io() diff --git a/kinematic_properties.py b/kinematic_properties.py deleted file mode 100644 index 7cebf905..00000000 --- a/kinematic_properties.py +++ /dev/null @@ -1,446 +0,0 @@ -#! /usr/bin/env python - -""" -kinematic_properties.py - -Some utility functions to compute kinematic properies for particle -distributions. - -We put them in a separate file to facilitate unit testing. -""" - -import numpy as np -import unyt -from typing import Union, Tuple -from halo_properties import SearchRadiusTooSmallError - - -def get_velocity_dispersion_matrix( - mass_fraction: unyt.unyt_array, - velocity: unyt.unyt_array, - ref_velocity: unyt.unyt_array, -) -> unyt.unyt_array: - """ - Compute the velocity dispersion matrix for the particles with the given - fractional mass (particle mass divided by total mass) and velocity, using - the given reference velocity as the centre of mass velocity. - - The result is a 6 element vector containing the unique components XX, YY, - ZZ, XY, XZ and YZ of the velocity dispersion matrix. - - Parameters: - - mass_fraction: unyt.unyt_array - Fractional mass of the particles (mass/mass.sum()). - - velocity: unyt.unyt_array - Velocity of the particles. - - ref_velocity: unyt.unyt_array - Reference point in velocity space. velocity and ref_velocity are assumed - to use the same reference point upon entry into this function. - - Returns an array with 6 elements: the XX, YY, ZZ, XY, XZ and YZ components - of the velocity dispersion matrix. - """ - - result = unyt.unyt_array(np.zeros(6), dtype=np.float32, units=velocity.units ** 2) - - vrel = velocity - ref_velocity[None, :] - result[0] += (mass_fraction * vrel[:, 0] * vrel[:, 0]).sum() - result[1] += (mass_fraction * vrel[:, 1] * vrel[:, 1]).sum() - result[2] += (mass_fraction * vrel[:, 2] * vrel[:, 2]).sum() - result[3] += (mass_fraction * vrel[:, 0] * vrel[:, 1]).sum() - result[4] += (mass_fraction * vrel[:, 0] * vrel[:, 2]).sum() - result[5] += (mass_fraction * vrel[:, 1] * vrel[:, 2]).sum() - - return result - - -def get_angular_momentum( - mass: unyt.unyt_array, - position: unyt.unyt_array, - velocity: unyt.unyt_array, - ref_position: Union[None, unyt.unyt_array] = None, - ref_velocity: Union[None, unyt.unyt_array] = None, -) -> unyt.unyt_array: - """ - Compute the total angular momentum vector for the particles with the given - masses, positions and velocities, and using the given reference position - and velocity as the centre of mass (velocity). - - Parameters: - - mass: unyt.unyt_array - Masses of the particles. - - position: unyt.unyt_array - Position of the particles. - - velocity: unyt.unyt_array - Velocities of the particles. - - ref_position: unyt.unyt_array or None - Reference position used as centre for the angular momentum calculation. - position and ref_position are assumed to use the same reference point upon - entry into this function. If None, position is assumed to be already using - the desired referece point. - - ref_velocity: unyt.unyt_array or None - Reference point in velocity space for the angular momentum calculation. - velocity and ref_velocity are assumed to use the same reference point upon - entry into this function. If None, velocity is assumed to be already using - the desired reference point. - - Returns the total angular momentum vector. - """ - - if ref_position is None: - prel = position - else: - prel = position - ref_position[None, :] - if ref_velocity is None: - vrel = velocity - else: - vrel = velocity - ref_velocity[None, :] - return (mass[:, None] * np.cross(prel, vrel)).sum(axis=0) - - -def get_angular_momentum_and_kappa_corot( - mass: unyt.unyt_array, - position: unyt.unyt_array, - velocity: unyt.unyt_array, - ref_position: Union[None, unyt.unyt_array] = None, - ref_velocity: Union[None, unyt.unyt_array] = None, - do_counterrot_mass: bool = False, -) -> Union[ - Tuple[unyt.unyt_array, unyt.unyt_quantity], - Tuple[unyt.unyt_array, unyt.unyt_quantity, unyt.unyt_quantity], -]: - """ - Get the total angular momentum vector (as in get_angular_momentum()) and - kappa_corot (Correa et al., 2017) for the particles with the given masses, - positions and velocities, and using the given reference position and - velocity as centre of mass (velocity). - - If both kappa_corot and the angular momentum vector are desired, it is more - efficient to use this function that calling get_angular_momentum() (and - get_kappa_corot(), if that would ever exist). - - Parameters: - - mass: unyt.unyt_array - Masses of the particles. - - position: unyt.unyt_array - Position of the particles. - - velocity: unyt.unyt_array - Velocities of the particles. - - ref_position: unyt.unyt_array or None - Reference position used as centre for the angular momentum calculation. - position and ref_position are assumed to use the same reference point upon - entry into this function. If None, position is assumed to be already using - the desired referece point. - - ref_velocity: unyt.unyt_array or None - Reference point in velocity space for the angular momentum calculation. - velocity and ref_velocity are assumed to use the same reference point upon - entry into this function. If None, velocity is assumed to be already using - the desired reference point. - - do_counterrot_mass: bool - Also compute the counterrotating mass? - - Returns: - - The total angular momentum vector. - - The ratio of the kinetic energy in counterrotating movement and the total - kinetic energy, kappa_corot. - - The total mass of counterrotating particles (if do_counterrot_mass == True). - """ - - kappa_corot = unyt.unyt_array( - 0.0, dtype=np.float32, units="dimensionless", registry=mass.units.registry - ) - - if ref_position is None: - prel = position - else: - prel = position - ref_position[None, :] - if ref_velocity is None: - vrel = velocity - else: - vrel = velocity - ref_velocity[None, :] - - Lpart = mass[:, None] * np.cross(prel, vrel) - Ltot = Lpart.sum(axis=0) - Lnrm = np.linalg.norm(Ltot) - - if do_counterrot_mass: - M_counterrot = unyt.unyt_array( - 0.0, dtype=np.float32, units=mass.units, registry=mass.units.registry - ) - - if Lnrm > 0.0 * Lnrm.units: - K = 0.5 * (mass[:, None] * vrel ** 2).sum() - if K > 0.0 * K.units or do_counterrot_mass: - Ldir = Ltot / Lnrm - Li = (Lpart * Ldir[None, :]).sum(axis=1) - if K > 0.0 * K.units: - r2 = prel[:, 0] ** 2 + prel[:, 1] ** 2 + prel[:, 2] ** 2 - rdotL = (prel * Ldir[None, :]).sum(axis=1) - Ri2 = r2 - rdotL ** 2 - # deal with division by zero (the first particle is guaranteed to - # be in the centre) - mask = Ri2 == 0.0 - Ri2[mask] = 1.0 * Ri2.units - Krot = 0.5 * (Li ** 2 / (mass * Ri2)) - Kcorot = Krot[(~mask) & (Li > 0.0 * Li.units)].sum() - kappa_corot += Kcorot / K - - if do_counterrot_mass: - M_counterrot += mass[Li < 0.0 * Li.units].sum() - - if do_counterrot_mass: - return Ltot, kappa_corot, M_counterrot - else: - return Ltot, kappa_corot - - -def get_vmax( - mass: unyt.unyt_array, radius: unyt.unyt_array, nskip: int = 0 -) -> Tuple[unyt.unyt_quantity, unyt.unyt_quantity]: - """ - Get the maximum circular velocity of a particle distribution. - - The value is computed from the cumulative mass profile after - sorting the particles by radius, as - vmax = sqrt(G*M/r) - - Parameters: - - mass: unyt.unyt_array - Mass of the particles. - - radius: unyt.unyt_array - Radius of the particles. - - nskip: int - Number of particles to skip - - Returns: - - Radius at which the maximum circular velocity is reached. - - Maximum circular velocity. - """ - # obtain the gravitational constant in the right units - # (this is read from the snapshot metadata, and is hence - # guaranteed to be consistent with the value used by SWIFT) - G = unyt.Unit("newton_G", registry=mass.units.registry) - isort = np.argsort(radius) - ordered_radius = radius[isort] - cumulative_mass = mass[isort].cumsum() - nskip = max( - nskip, np.argmin(np.isclose(ordered_radius, 0.0 * ordered_radius.units)) - ) - ordered_radius = ordered_radius[nskip:] - if len(ordered_radius) == 0 or ordered_radius[0] == 0: - return 0.0 * radius.units, np.sqrt(0.0 * G * mass.units / radius.units) - cumulative_mass = cumulative_mass[nskip:] - v_over_G = cumulative_mass / ordered_radius - imax = np.argmax(v_over_G) - return ordered_radius[imax], np.sqrt(v_over_G[imax] * G) - - -def get_inertia_tensor( - mass, - position, - sphere_radius, - search_radius=None, - reduced=False, - max_iterations=20, - min_particles=20, -): - """ - Get the inertia tensor of the given particle distribution, computed as - I_{ij} = m*x_i*x_j / Mtot. - - Parameters: - - mass: unyt.unyt_array - Masses of the particles. - - position: unyt.unyt_array - Positions of the particles. - - sphere_radius: unyt.unyt_quantity - Use all particles within a sphere of this size for the calculation - - search_radius: unyt.unyt_quantity - Radius of the region of the simulation for which we have particle data - This function throws a SearchRadiusTooSmallError if we need particles outside - of this region. - - reduced: bool - Whether to calculate the reduced inertia tensor - - max_iterations: int - The maximum number of iterations to repeat the inertia tensor calculation - - min_particles: int - The number of particles required within the initial sphere. The inertia tensor - is not computed if this threshold is not met. - - Returns the inertia tensor. - """ - - # Check we have at least "min_particles" particles - if mass.shape[0] < min_particles: - return None - - # Remove particles at centre if calculating reduced tensor - if reduced: - norm = np.linalg.norm(position, axis=1) ** 2 - mask = np.logical_not(np.isclose(norm, 0)) - position = position[mask] - mass = mass[mask] - norm = norm[mask] - - # Set stopping criteria - tol = 0.0001 - q = 1000 - - # Ensure we have consistent units - R = sphere_radius.to("kpc") - position = position.to("kpc") - - # Start with a sphere - eig_val = [1, 1, 1] - eig_vec = np.array([[1, 0, 0], [0, 1, 0], [0, 0, 1]]) - - for i_iter in range(max_iterations): - # Calculate shape - old_q = q - q = np.sqrt(eig_val[1] / eig_val[2]) - s = np.sqrt(eig_val[0] / eig_val[2]) - p = np.sqrt(eig_val[0] / eig_val[1]) - - # Break if converged - if abs((old_q - q) / q) < tol: - break - - # Calculate ellipsoid, determine which particles are inside - axis = R * np.array( - [1 * np.cbrt(s * p), 1 * np.cbrt(q / p), 1 / np.cbrt(q * s)] - ) - p = np.dot(position, eig_vec) / axis - r = np.linalg.norm(p, axis=1) - # We want to skip the calculation if we have less than "min_particles" - # inside the initial sphere. We do the check here since this is the first - # time we calculate how many particles are within the sphere. - if (i_iter == 0) and (np.sum(r <= 1) < min_particles): - return None - weight = mass / np.sum(mass[r <= 1]) - weight[r > 1] = 0 - - # Check if we have exceeded the search radius. For subhalo_properties we - # have all the bound particles, and so the search radius doesn't matter - if (search_radius is not None) and (np.max(R) > search_radius): - raise SearchRadiusTooSmallError("Inertia tensor required more particles") - - # Calculate inertia tensor - tensor = weight[:, None, None] * position[:, :, None] * position[:, None, :] - if reduced: - tensor /= norm[:, None, None] - tensor = tensor.sum(axis=0) - eig_val, eig_vec = np.linalg.eigh(tensor.value) - - return np.concatenate([np.diag(tensor), tensor[np.triu_indices(3, 1)]]) - - -def get_projected_inertia_tensor( - mass, position, axis, radius, reduced=False, max_iterations=20, min_particles=20 -): - """ - Takes in the particle distribution projected along a given axis, and calculates the inertia - tensor using the projected values. - - Unlike get_inertia_tensor, we don't need to check if we have exceeded the search radius. This - is because all the bound particles are passed to this function. - - Parameters: - - mass: unyt.unyt_array - Masses of the particles. - - position: unyt.unyt_array - Positions of the particles. - - axis: 0, 1, 2 - Projection axis. Only the coordinates perpendicular to this axis are - taken into account. - - radius: unyt.unyt_quantity - Exclude particles outside this radius for the inertia tensor calculation - - reduced: bool - Whether to calculate the reduced inertia tensor - - max_iterations: int - The maximum number of iterations to repeat the inertia tensor calculation - - min_particles: int - The number of particles required within the initial circle. The inertia tensor - is not computed if this threshold is not met. - - Returns the inertia tensor. - """ - - # Check we have at least "min_particles" particles - if mass.shape[0] < min_particles: - return None - - projected_position = unyt.unyt_array( - np.zeros((position.shape[0], 2)), units=position.units, dtype=position.dtype - ) - if axis == 0: - projected_position[:, 0] = position[:, 1] - projected_position[:, 1] = position[:, 2] - elif axis == 1: - projected_position[:, 0] = position[:, 2] - projected_position[:, 1] = position[:, 0] - elif axis == 2: - projected_position[:, 0] = position[:, 0] - projected_position[:, 1] = position[:, 1] - else: - raise AttributeError(f"Invalid axis: {axis}!") - - # Remove particles at centre if calculating reduced tensor - if reduced: - norm = np.linalg.norm(projected_position, axis=1) ** 2 - mask = np.logical_not(np.isclose(norm, 0)) - projected_position = projected_position[mask] - mass = mass[mask] - norm = norm[mask] - - # Set stopping criteria - tol = 0.0001 - q = 1000 - - # Ensure we have consistent units - R = radius.to("kpc") - projected_position = projected_position.to("kpc") - - # Start with a circle - eig_val = [1, 1] - eig_vec = np.array([[1, 0], [0, 1]]) - - for i_iter in range(max_iterations): - # Calculate shape - old_q = q - q = np.sqrt(eig_val[0] / eig_val[1]) - - # Break if converged - if abs((old_q - q) / q) < tol: - break - - # Calculate ellipse, determine which particles are inside - axis = R * np.array([1 * np.sqrt(q), 1 / np.sqrt(q)]) - p = np.dot(projected_position, eig_vec) / axis - r = np.linalg.norm(p, axis=1) - # We want to skip the calculation if we have less than "min_particles" - # inside the initial circle. We do the check here since this is the first - # time we calculate how many particles are within the circle. - if (i_iter == 0) and (np.sum(r <= 1) < min_particles): - return None - weight = mass / np.sum(mass[r <= 1]) - weight[r > 1] = 0 - - # Calculate inertia tensor - tensor = ( - weight[:, None, None] - * projected_position[:, :, None] - * projected_position[:, None, :] - ) - if reduced: - tensor /= norm[:, None, None] - tensor = tensor.sum(axis=0) - eig_val, eig_vec = np.linalg.eigh(tensor.value) - - return np.concatenate([np.diag(tensor), [tensor[(0, 1)]]]) - - -if __name__ == "__main__": - """ - Standalone version. TODO: add test to check if inertia tensor computation works. - """ - pass diff --git a/lustre.py b/lustre.py deleted file mode 100644 index 8a4102a7..00000000 --- a/lustre.py +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/env python - -import subprocess -import os - - -def setstripe(dirname, stripe_size, stripe_count): - """ - Try to set Lustre striping on a directory - """ - args = [ - "lfs", - "setstripe", - "--stripe-count=%d" % stripe_count, - "--stripe-size=%dM" % stripe_size, - dirname, - ] - try: - subprocess.run(args) - except (FileNotFoundError, subprocess.CalledProcessError): - # if the 'lfs' command is not available, this will generate a - # FileNotFoundError - print("WARNING: failed to set lustre striping on %s" % dirname) - - -def ensure_output_dir(filename): - - # Try to ensure the directory exists - dirname = os.path.dirname(filename) - try: - os.makedirs(dirname) - except OSError: - pass - - # Try to set striping - setstripe(dirname, 32, -1) diff --git a/match_hbt_halos.py b/match_hbt_halos.py deleted file mode 100644 index bd04f94f..00000000 --- a/match_hbt_halos.py +++ /dev/null @@ -1,427 +0,0 @@ -#!/bin/env python - -import os - -import numpy as np -import h5py - -import virgo.mpi.parallel_sort as psort -import virgo.mpi.parallel_hdf5 as phdf5 - -import lustre -import read_hbtplus - -import unyt -import swift_cells - -from mpi4py import MPI - -comm = MPI.COMM_WORLD -comm_rank = comm.Get_rank() -comm_size = comm.Get_size() - -# Maximum number of particle types -NTYPEMAX = 7 - - -def message(s): - if comm_rank == 0: - print(s) - - -def exchange_array(arr, dest, comm): - """ - Carry out an alltoallv on the supplied array, given the MPI rank - to send each element to. - """ - order = np.argsort(dest) - sendbuf = arr[order] - send_count = np.bincount(dest, minlength=comm_size) - send_offset = np.cumsum(send_count) - send_count - recv_count = np.zeros_like(send_count) - comm.Alltoall(send_count, recv_count) - recv_offset = np.cumsum(recv_count) - recv_count - recvbuf = np.ndarray(recv_count.sum(), dtype=arr.dtype) - psort.my_alltoallv( - sendbuf, send_count, send_offset, recvbuf, recv_count, recv_offset, comm=comm - ) - return recvbuf - - -# Define a placeholder unit system, since everything we do is dimensionless. -# However, it would be better to load this from a snapshot. -def define_unit_system(): - - # Create a registry using this base unit system - reg = unyt.unit_registry.UnitRegistry() - - # Add some units which might be useful for dealing with input halo catalogues - unyt.define_unit("swift_mpc", 1.0 * unyt.cm, registry=reg) - unyt.define_unit("swift_msun", 1.0 * unyt.g, registry=reg) - unyt.define_unit("h", 1.0 * unyt.Hz, registry=reg) - - return reg - - -def find_matching_halos( - base_name1, - base_name2, - max_nr_particles, - min_particle_id, - max_particle_id, - field_only, -): - - # We only care about dimensionless quantities here, so - # define a placeholder unit system - registry = define_unit_system() - a_unit = unyt.Unit("cm", registry=registry) ** 0 - boxsize = None - - # Load halo data from the second catalogue - keep_orphans = True - halo_data2 = read_hbtplus.read_hbtplus_catalogue( - comm, base_name2, a_unit, registry, boxsize, keep_orphans - ) - - # Catalogue 2 properties - track_ids2 = halo_data2["TrackId"] - host_ids2 = halo_data2["HostHaloId"] - is_central2 = halo_data2["is_central"] - - # Find the track ID of the central halo that has the same hostHaloID - central_index2 = psort.parallel_match( - host_ids2, host_ids2[is_central2 == 1], comm=comm - ) - host_track_ids2 = psort.fetch_elements( - track_ids2[is_central2 == 1], central_index2, comm=comm - ) - - # Find the index of that central halo (which may differ from the track ID) - host_index2 = psort.parallel_match(host_track_ids2, track_ids2, comm=comm) - - # Free the other halo data - del halo_data2 - - # Find group membership for particles in the first catalogue: - total_nr_halos1, cat1_ids, cat1_grnr_in_cat1, rank_bound1 = read_hbtplus.read_hbtplus_groupnr( - base_name1 - ) - - # Find group membership for particles in the second catalogue - total_nr_halos2, cat2_ids, cat2_grnr_in_cat2, rank_bound2 = read_hbtplus.read_hbtplus_groupnr( - base_name2 - ) - - # Decide range of halos in cat1 which we'll store on each rank: - # This is used to partition the result between MPI ranks. - nr_cat1_tot = total_nr_halos1 - nr_cat1_per_rank = nr_cat1_tot // comm_size - if comm_rank < comm_size - 1: - nr_cat1_local = nr_cat1_per_rank - else: - nr_cat1_local = nr_cat1_tot - (comm_size - 1) * nr_cat1_per_rank - - # Clear group membership for particles with invalid IDs - if min_particle_id != None: - discard = cat1_ids < min_particle_id - cat1_grnr_in_cat1[discard] = -1 - - if max_particle_id != None: - discard = cat1_ids >= max_particle_id - cat1_grnr_in_cat1[discard] = -1 - - # If we're only matching to field halos, then any particles in the second catalogue which - # belong to a halo with hostHaloID != -1 need to have their group membership reset to their - # host halo. - if field_only: - # Find particles in halos in cat2 - in_halo = cat2_grnr_in_cat2 >= 0 - # Fetch host halo array index for each particle in cat2, or -1 if not in a halo - particle_host_index = -np.ones_like(cat2_grnr_in_cat2) - particle_host_index[in_halo] = psort.fetch_elements( - host_index2, cat2_grnr_in_cat2[in_halo], comm=comm - ) - # Where a particle's halo has a host halo, set its group membership to be the host halo - have_host = particle_host_index >= 0 - cat2_grnr_in_cat2[have_host] = particle_host_index[have_host] - - # Discard particles which are in no halo from each catalogue - in_group = cat1_grnr_in_cat1 >= 0 - cat1_ids = cat1_ids[in_group] - cat1_grnr_in_cat1 = cat1_grnr_in_cat1[in_group] - in_group = cat2_grnr_in_cat2 >= 0 - cat2_ids = cat2_ids[in_group] - cat2_grnr_in_cat2 = cat2_grnr_in_cat2[in_group] - - # Now we need to identify the first max_nr_particles remaining particles for each - # halo in catalogue 1. First, find the ranking of each particle within the part of - # its group which is stored on this MPI rank. First particle in a group has rank 0. - unique_grnr, unique_index, unique_count = np.unique( - cat1_grnr_in_cat1, return_index=True, return_counts=True - ) - cat1_rank_in_group = -np.ones_like(cat1_grnr_in_cat1) - for ui, uc in zip(unique_index, unique_count): - cat1_rank_in_group[ui : ui + uc] = np.arange(uc, dtype=int) - assert np.all(cat1_rank_in_group >= 0) - - # Then for the first group on each rank we'll need to add the total number of particles in - # the same group on all lower numbered ranks. Since the particles are sorted by group this - # can only ever be the last group on each lower numbered rank. - if len(unique_grnr) > 0: - # This rank has at least one particle in a group. Store indexes of first and last groups - # and the number of particles from the last group which are stored on this rank. - assert unique_index[0] == 0 - first_grnr = unique_grnr[0] - last_grnr = unique_grnr[-1] - last_grnr_count = unique_count[-1] - else: - # This rank has no particles in groups - first_grnr = -1 - last_grnr = -1 - last_grnr_count = 0 - all_last_grnr = comm.allgather(last_grnr) - all_last_grnr_count = comm.allgather(last_grnr_count) - # Loop over lower numbered ranks - for rank_nr in range(comm_rank): - if first_grnr >= 0 and all_last_grnr[rank_nr] == first_grnr: - cat1_rank_in_group[: unique_count[0]] += all_last_grnr_count[rank_nr] - - # Only keep the first max_nr_particles remaining particles in each group in catalogue 1 - keep = cat1_rank_in_group < max_nr_particles - cat1_ids = cat1_ids[keep] - cat1_grnr_in_cat1 = cat1_grnr_in_cat1[keep] - - # For each particle ID in catalogue 1, try to find the same particle ID in catalogue 2 - ptr = psort.parallel_match(cat1_ids, cat2_ids, comm=comm) - matched = ptr >= 0 - - # For each particle ID in catalogue 1, fetch the group membership of the matching ID in catalogue 2 - cat1_grnr_in_cat2 = -np.ones_like(cat1_grnr_in_cat1) - cat1_grnr_in_cat2[matched] = psort.fetch_elements(cat2_grnr_in_cat2, ptr[matched]) - - # Discard unmatched particles - cat1_grnr_in_cat1 = cat1_grnr_in_cat1[matched] - cat1_grnr_in_cat2 = cat1_grnr_in_cat2[matched] - - # Get sorted, unique (grnr1, grnr2) combinations and counts of how many instances of each we have - assert np.all(cat1_grnr_in_cat1 < 2 ** 32) - assert np.all(cat1_grnr_in_cat1 >= 0) - assert np.all(cat1_grnr_in_cat2 < 2 ** 32) - assert np.all(cat1_grnr_in_cat2 >= 0) - sort_key = (cat1_grnr_in_cat1.astype(np.uint64) << 32) + cat1_grnr_in_cat2.astype( - np.uint64 - ) - unique_value, cat1_count = psort.parallel_unique( - sort_key, comm=comm, return_counts=True, repartition_output=True - ) - cat1_grnr_in_cat1 = (unique_value >> 32).astype( - int - ) # Cast to int because mixing signed and unsigned causes numpy to cast to float! - cat1_grnr_in_cat2 = (unique_value % (1 << 32)).astype(int) - - # Send each (grnr1, grnr2, count) combination to the rank which will store the result for that halo - if nr_cat1_per_rank > 0: - dest = (cat1_grnr_in_cat1 // nr_cat1_per_rank).astype(int) - dest[dest > comm_size - 1] = comm_size - 1 - else: - dest = np.empty_like(cat1_grnr_in_cat1, dtype=int) - dest[:] = comm_size - 1 - recv_grnr_in_cat1 = exchange_array(cat1_grnr_in_cat1, dest, comm) - recv_grnr_in_cat2 = exchange_array(cat1_grnr_in_cat2, dest, comm) - recv_count = exchange_array(cat1_count, dest, comm) - - # Allocate output arrays: - # Each rank has nr_cat1_per_rank halos with any extras on the last rank - first_in_cat1 = comm_rank * nr_cat1_per_rank - result_grnr_in_cat2 = -np.ones( - nr_cat1_local, dtype=int - ) # For each halo in cat1, will store index of match in cat2 - result_count = np.zeros( - nr_cat1_local, dtype=int - ) # Will store number of matching particles - - # Update output arrays using the received data. - for recv_nr in range(len(recv_grnr_in_cat1)): - # Compute local array index of halo to update - local_halo_nr = recv_grnr_in_cat1[recv_nr] - first_in_cat1 - assert local_halo_nr >= 0 - assert local_halo_nr < nr_cat1_local - # Check if the received count is higher than the highest so far - if recv_count[recv_nr] > result_count[local_halo_nr]: - # This received combination has the highest count so far - result_grnr_in_cat2[local_halo_nr] = recv_grnr_in_cat2[recv_nr] - result_count[local_halo_nr] = recv_count[recv_nr] - elif recv_count[recv_nr] == result_count[local_halo_nr]: - # In the event of a tie, go for the lowest group number for reproducibility - if recv_grnr_in_cat2[recv_nr] < result_grnr_in_cat2[local_halo_nr]: - result_grnr_in_cat2[local_halo_nr] = recv_grnr_in_cat2[recv_nr] - result_count[local_halo_nr] = recv_count[recv_nr] - - return result_grnr_in_cat2, result_count - - -def consistent_match(match_index_12, match_index_21): - """ - For each halo in catalogue 1, determine if its match in catalogue 2 - points back at it. - - match_index_12 has one entry for each halo in catalogue 1 and - specifies the matching halo in catalogue 2 (or -1 for not match) - - match_index_21 has one entry for each halo in catalogue 2 and - specifies the matching halo in catalogue 1 (or -1 for not match) - - Returns an array with 1 for a match and 0 otherwise. - """ - - # Find the global array indexes of halos stored on this rank - nr_local_halos = len(match_index_12) - local_halo_offset = comm.scan(nr_local_halos) - nr_local_halos - local_halo_index = np.arange( - local_halo_offset, local_halo_offset + nr_local_halos, dtype=int - ) - - # For each halo, find the halo that its match in the other catalogue was matched with - match_back = -np.ones(nr_local_halos, dtype=int) - has_match = match_index_12 >= 0 - match_back[has_match] = psort.fetch_elements( - match_index_21, match_index_12[has_match], comm=comm - ) - - # If we retrieved our own halo index, we have a match - return np.where(match_back == local_halo_index, 1, 0) - - -def get_match_hbt_halos_args(comm): - """ - Process command line arguments for halo matching program. - - Returns a dict with the argument values, or None on failure. - """ - - from virgo.mpi.util import MPIArgumentParser - - parser = MPIArgumentParser( - comm, description="Find matching halos between snapshots" - ) - parser.add_argument("hbt_basename1", help="Base name of the first set of HBT files") - parser.add_argument( - "hbt_basename2", help="Base name of the second set of HBT files" - ) - parser.add_argument( - "nr_particles", - metavar="N", - type=int, - help="Number of most bound particles to use.", - ) - parser.add_argument("output_file", help="Output file name") - parser.add_argument( - "--min-particle-id", - nargs="*", - type=int, - help="Only use particle with ID >= this", - ) - parser.add_argument( - "--max-particle-id", - nargs="*", - type=int, - help="Only use particle with ID < this", - ) - parser.add_argument( - "--to-field-halos-only", action="store_true", help="Only match to field halos" - ) - args = parser.parse_args() - - return args - - -if __name__ == "__main__": - - # Read command line parameters - args = get_match_hbt_halos_args(comm) - - # Ensure output dir exists - if comm_rank == 0: - lustre.ensure_output_dir(args.output_file) - comm.barrier() - - # For each halo in output 1, find the matching halo in output 2 - message("Matching from first catalogue to second") - match_index_12, count_12 = find_matching_halos( - args.hbt_basename1, - args.hbt_basename2, - args.nr_particles, - args.min_particle_id, - args.max_particle_id, - args.to_field_halos_only, - ) - total_nr_halos = comm.allreduce(len(match_index_12)) - total_nr_matched = comm.allreduce(np.sum(match_index_12 >= 0)) - message(f" Matched {total_nr_matched} of {total_nr_halos} halos") - - # For each halo in output 2, find the matching halo in output 1 - message("Matching from second catalogue to first") - match_index_21, count_21 = find_matching_halos( - args.hbt_basename2, - args.hbt_basename1, - args.nr_particles, - args.min_particle_id, - args.max_particle_id, - args.to_field_halos_only, - ) - total_nr_halos = comm.allreduce(len(match_index_21)) - total_nr_matched = comm.allreduce(np.sum(match_index_21 >= 0)) - message(f" Matched {total_nr_matched} of {total_nr_halos} halos") - - # Check for consistent matches in both directions - message("Checking for consistent matches") - consistent_12 = consistent_match(match_index_12, match_index_21) - consistent_21 = consistent_match(match_index_21, match_index_12) - - # Write the output - def write_output_field(name, data, description): - dataset = phdf5.collective_write(outfile, name, data, comm) - dataset.attrs["Description"] = description - - message("Writing output") - with h5py.File(args.output_file, "w", driver="mpio", comm=comm) as outfile: - # Write input parameters - params = outfile.create_group("Parameters") - for name, value in vars(args).items(): - if value is not None: - params.attrs[name] = value - # Matching from first catalogue to second - write_output_field( - "MatchIndex1to2", - match_index_12, - "For each halo in the first catalogue, index of the matching halo in the second", - ) - write_output_field( - "MatchCount1to2", - count_12, - f"How many of the {args.nr_particles} most bound particles from the halo in the first catalogue are in the matched halo in the second", - ) - write_output_field( - "Consistent1to2", - consistent_12, - "Whether the match from first to second catalogue is consistent with second to first (1) or not (0)", - ) - # Matching from second catalogue to first - write_output_field( - "MatchIndex2to1", - match_index_21, - "For each halo in the second catalogue, index of the matching halo in the first", - ) - write_output_field( - "MatchCount2to1", - count_21, - f"How many of the {args.nr_particles} most bound particles from the halo in the second catalogue are in the matched halo in the first", - ) - write_output_field( - "Consistent2to1", - consistent_21, - "Whether the match from second to first catalogue is consistent with first to second (1) or not (0)", - ) - comm.barrier() - message("Done.") diff --git a/match_vr_halos.py b/match_vr_halos.py deleted file mode 100644 index 9586879f..00000000 --- a/match_vr_halos.py +++ /dev/null @@ -1,460 +0,0 @@ -#!/bin/env python - -import os - -import numpy as np -import h5py - -import virgo.mpi.parallel_sort as psort -import virgo.mpi.parallel_hdf5 as phdf5 - -import lustre -import read_vr - -from mpi4py import MPI - -comm = MPI.COMM_WORLD -comm_rank = comm.Get_rank() -comm_size = comm.Get_size() - -# Maximum number of particle types -NTYPEMAX = 7 - - -def message(s): - if comm_rank == 0: - print(s) - - -def exchange_array(arr, dest, comm): - """ - Carry out an alltoallv on the supplied array, given the MPI rank - to send each element to. - """ - order = np.argsort(dest) - sendbuf = arr[order] - send_count = np.bincount(dest, minlength=comm_size) - send_offset = np.cumsum(send_count) - send_count - recv_count = np.zeros_like(send_count) - comm.Alltoall(send_count, recv_count) - recv_offset = np.cumsum(recv_count) - recv_count - recvbuf = np.ndarray(recv_count.sum(), dtype=arr.dtype) - psort.my_alltoallv( - sendbuf, send_count, send_offset, recvbuf, recv_count, recv_offset, comm=comm - ) - return recvbuf - - -def read_host_index(basename): - """ - Find the host halo's global array index for each halo in a VR output. - Returns -1 for field halos. - """ - - # Read the ID and hostHaloID - cat = read_vr.read_vr_datasets(basename, "properties", ("ID", "hostHaloID")) - vr_id = cat["ID"] - vr_host_id = cat["hostHaloID"] - - # For each halo, find the index of the host halo by matching hostHaloID to - # ID. Field halos have hostHaloID=-1, which will not match to any halo ID. - return psort.parallel_match(vr_host_id, vr_id, comm=comm) - - -def find_matching_halos( - cat1_length, - cat1_offset, - cat1_ids, - cat1_types, - host_index1, - cat2_length, - cat2_offset, - cat2_ids, - cat2_types, - host_index2, - max_nr_particles, - use_type, - field_only, -): - - # Decide range of halos in cat1 which we'll store on each rank: - # This is used to partition the result between MPI ranks. - nr_cat1_tot = comm.allreduce(len(cat1_length)) - nr_cat1_per_rank = nr_cat1_tot // comm_size - if comm_rank < comm_size - 1: - nr_cat1_local = nr_cat1_per_rank - else: - nr_cat1_local = nr_cat1_tot - (comm_size - 1) * nr_cat1_per_rank - - # Find group membership for particles in the first catalogue: - cat1_grnr_in_cat1 = read_vr.vr_group_membership_from_ids( - cat1_length, cat1_offset, cat1_ids - ) - - # Find group membership for particles in the second catalogue - cat2_grnr_in_cat2 = read_vr.vr_group_membership_from_ids( - cat2_length, cat2_offset, cat2_ids - ) - - # Clear group membership for particles of types we're not using in the first catalogue - discard = (use_type[cat1_types] == False) | (cat1_grnr_in_cat1 < 0) - cat1_grnr_in_cat1[discard] = -1 - - # If we're only matching to field halos, then any particles in the second catalogue which - # belong to a halo with hostHaloID != -1 need to have their group membership reset to their - # host halo. - if field_only: - # Find particles in halos in cat2 - in_halo = cat2_grnr_in_cat2 >= 0 - # Fetch host halo array index for each particle in cat2, or -1 if not in a halo - particle_host_index = -np.ones_like(cat2_grnr_in_cat2) - particle_host_index[in_halo] = psort.fetch_elements( - host_index2, cat2_grnr_in_cat2[in_halo], comm=comm - ) - # Where a particle's halo has a host halo, set its group membership to be the host halo - have_host = particle_host_index >= 0 - cat2_grnr_in_cat2[have_host] = particle_host_index[have_host] - - # Discard particles which are in no halo from each catalogue - in_group = cat1_grnr_in_cat1 >= 0 - cat1_ids = cat1_ids[in_group] - cat1_grnr_in_cat1 = cat1_grnr_in_cat1[in_group] - in_group = cat2_grnr_in_cat2 >= 0 - cat2_ids = cat2_ids[in_group] - cat2_grnr_in_cat2 = cat2_grnr_in_cat2[in_group] - - # Now we need to identify the first max_nr_particles remaining particles for each - # halo in catalogue 1. First, find the ranking of each particle within the part of - # its group which is stored on this MPI rank. First particle in a group has rank 0. - unique_grnr, unique_index, unique_count = np.unique( - cat1_grnr_in_cat1, return_index=True, return_counts=True - ) - cat1_rank_in_group = -np.ones_like(cat1_grnr_in_cat1) - for ui, uc in zip(unique_index, unique_count): - cat1_rank_in_group[ui : ui + uc] = np.arange(uc, dtype=int) - assert np.all(cat1_rank_in_group >= 0) - - # Then for the first group on each rank we'll need to add the total number of particles in - # the same group on all lower numbered ranks. Since the particles are sorted by group this - # can only ever be the last group on each lower numbered rank. - if len(unique_grnr) > 0: - # This rank has at least one particle in a group. Store indexes of first and last groups - # and the number of particles from the last group which are stored on this rank. - assert unique_index[0] == 0 - first_grnr = unique_grnr[0] - last_grnr = unique_grnr[-1] - last_grnr_count = unique_count[-1] - else: - # This rank has no particles in groups - first_grnr = -1 - last_grnr = -1 - last_grnr_count = 0 - all_last_grnr = comm.allgather(last_grnr) - all_last_grnr_count = comm.allgather(last_grnr_count) - # Loop over lower numbered ranks - for rank_nr in range(comm_rank): - if first_grnr >= 0 and all_last_grnr[rank_nr] == first_grnr: - cat1_rank_in_group[: unique_count[0]] += all_last_grnr_count[rank_nr] - - # Only keep the first max_nr_particles remaining particles in each group in catalogue 1 - keep = cat1_rank_in_group < max_nr_particles - cat1_ids = cat1_ids[keep] - cat1_grnr_in_cat1 = cat1_grnr_in_cat1[keep] - - # For each particle ID in catalogue 1, try to find the same particle ID in catalogue 2 - ptr = psort.parallel_match(cat1_ids, cat2_ids, comm=comm) - matched = ptr >= 0 - - # For each particle ID in catalogue 1, fetch the group membership of the matching ID in catalogue 2 - cat1_grnr_in_cat2 = -np.ones_like(cat1_grnr_in_cat1) - cat1_grnr_in_cat2[matched] = psort.fetch_elements(cat2_grnr_in_cat2, ptr[matched]) - - # Discard unmatched particles - cat1_grnr_in_cat1 = cat1_grnr_in_cat1[matched] - cat1_grnr_in_cat2 = cat1_grnr_in_cat2[matched] - - # Get sorted, unique (grnr1, grnr2) combinations and counts of how many instances of each we have - assert np.all(cat1_grnr_in_cat1 < 2 ** 32) - assert np.all(cat1_grnr_in_cat1 >= 0) - assert np.all(cat1_grnr_in_cat2 < 2 ** 32) - assert np.all(cat1_grnr_in_cat2 >= 0) - sort_key = (cat1_grnr_in_cat1.astype(np.uint64) << 32) + cat1_grnr_in_cat2.astype( - np.uint64 - ) - unique_value, cat1_count = psort.parallel_unique( - sort_key, comm=comm, return_counts=True, repartition_output=True - ) - cat1_grnr_in_cat1 = (unique_value >> 32).astype( - int - ) # Cast to int because mixing signed and unsigned causes numpy to cast to float! - cat1_grnr_in_cat2 = (unique_value % (1 << 32)).astype(int) - - # Send each (grnr1, grnr2, count) combination to the rank which will store the result for that halo - if nr_cat1_per_rank > 0: - dest = (cat1_grnr_in_cat1 // nr_cat1_per_rank).astype(int) - dest[dest > comm_size - 1] = comm_size - 1 - else: - dest = np.empty_like(cat1_grnr_in_cat1, dtype=int) - dest[:] = comm_size - 1 - recv_grnr_in_cat1 = exchange_array(cat1_grnr_in_cat1, dest, comm) - recv_grnr_in_cat2 = exchange_array(cat1_grnr_in_cat2, dest, comm) - recv_count = exchange_array(cat1_count, dest, comm) - - # Allocate output arrays: - # Each rank has nr_cat1_per_rank halos with any extras on the last rank - first_in_cat1 = comm_rank * nr_cat1_per_rank - result_grnr_in_cat2 = -np.ones( - nr_cat1_local, dtype=int - ) # For each halo in cat1, will store index of match in cat2 - result_count = np.zeros( - nr_cat1_local, dtype=int - ) # Will store number of matching particles - - # Update output arrays using the received data. - for recv_nr in range(len(recv_grnr_in_cat1)): - # Compute local array index of halo to update - local_halo_nr = recv_grnr_in_cat1[recv_nr] - first_in_cat1 - assert local_halo_nr >= 0 - assert local_halo_nr < nr_cat1_local - # Check if the received count is higher than the highest so far - if recv_count[recv_nr] > result_count[local_halo_nr]: - # This received combination has the highest count so far - result_grnr_in_cat2[local_halo_nr] = recv_grnr_in_cat2[recv_nr] - result_count[local_halo_nr] = recv_count[recv_nr] - elif recv_count[recv_nr] == result_count[local_halo_nr]: - # In the event of a tie, go for the lowest group number for reproducibility - if recv_grnr_in_cat2[recv_nr] < result_grnr_in_cat2[local_halo_nr]: - result_grnr_in_cat2[local_halo_nr] = recv_grnr_in_cat2[recv_nr] - result_count[local_halo_nr] = recv_count[recv_nr] - - return result_grnr_in_cat2, result_count - - -def consistent_match(match_index_12, match_index_21): - """ - For each halo in catalogue 1, determine if its match in catalogue 2 - points back at it. - - match_index_12 has one entry for each halo in catalogue 1 and - specifies the matching halo in catalogue 2 (or -1 for not match) - - match_index_21 has one entry for each halo in catalogue 2 and - specifies the matching halo in catalogue 1 (or -1 for not match) - - Returns an array with 1 for a match and 0 otherwise. - """ - - # Find the global array indexes of halos stored on this rank - nr_local_halos = len(match_index_12) - local_halo_offset = comm.scan(nr_local_halos) - nr_local_halos - local_halo_index = np.arange( - local_halo_offset, local_halo_offset + nr_local_halos, dtype=int - ) - - # For each halo, find the halo that its match in the other catalogue was matched with - match_back = -np.ones(nr_local_halos, dtype=int) - has_match = match_index_12 >= 0 - match_back[has_match] = psort.fetch_elements( - match_index_21, match_index_12[has_match], comm=comm - ) - - # If we retrieved our own halo index, we have a match - return np.where(match_back == local_halo_index, 1, 0) - - -def get_match_vr_halos_args(comm): - """ - Process command line arguments for halo matching program. - - Returns a dict with the argument values, or None on failure. - """ - - from virgo.mpi.util import MPIArgumentParser - - parser = MPIArgumentParser( - comm, description="Find matching halos between snapshots" - ) - parser.add_argument( - "vr_basename1", - help="Base name of the first VELOCIraptor files, excluding trailing .properties[.N] etc.", - ) - parser.add_argument( - "vr_basename2", - help="Base name of the second VELOCIraptor files, excluding trailing .properties[.N] etc.", - ) - parser.add_argument( - "nr_particles", - metavar="N", - type=int, - help="Number of most bound particles to use.", - ) - parser.add_argument("output_file", help="Output file name") - parser.add_argument( - "--use-types", - nargs="*", - type=int, - help="Only use the specified particle types (integer, 0-6)", - ) - parser.add_argument( - "--to-field-halos-only", - action="store_true", - help="Only match to field halos (with hostHaloID=-1 in VR catalogue)", - ) - args = parser.parse_args() - - return args - - -if __name__ == "__main__": - - # Read command line parameters - args = get_match_vr_halos_args(comm) - - # Ensure output dir exists - if comm_rank == 0: - lustre.ensure_output_dir(args.output_file) - comm.barrier() - - # Read VR lengths, offsets and IDs for the two outputs - ( - length_bound1, - offset_bound1, - ids_bound1, - length_unbound1, - offset_unbound1, - ids_unbound1, - ) = read_vr.read_vr_lengths_and_offsets(args.vr_basename1) - ( - length_bound2, - offset_bound2, - ids_bound2, - length_unbound2, - offset_unbound2, - ids_unbound2, - ) = read_vr.read_vr_lengths_and_offsets(args.vr_basename2) - - # Read in particle types for the two outputs - type_bound1 = read_vr.read_vr_datasets( - args.vr_basename1, "catalog_parttypes", ("Particle_types",) - )["Particle_types"] - type_bound2 = read_vr.read_vr_datasets( - args.vr_basename2, "catalog_parttypes", ("Particle_types",) - )["Particle_types"] - - # Read host halo indexes - host_index1 = read_host_index(args.vr_basename1) - host_index2 = read_host_index(args.vr_basename2) - - # Decide which particle types we want to keep - if args.use_types is not None: - use_type = np.zeros(NTYPEMAX, dtype=bool) - for ut in args.use_types: - use_type[ut] = True - message(f"Using particle type {ut}") - else: - message("Using all particle types") - use_type = np.ones(NTYPEMAX, dtype=bool) - - # For each halo in output 1, find the matching halo in output 2 - message("Matching from first catalogue to second") - match_index_12, count_12 = find_matching_halos( - length_bound1, - offset_bound1, - ids_bound1, - type_bound1, - host_index1, - length_bound2, - offset_bound2, - ids_bound2, - type_bound2, - host_index2, - args.nr_particles, - use_type, - args.to_field_halos_only, - ) - total_nr_halos = comm.allreduce(len(match_index_12)) - total_nr_matched = comm.allreduce(np.sum(match_index_12 >= 0)) - message(f" Matched {total_nr_matched} of {total_nr_halos} halos") - - # For each halo in output 2, find the matching halo in output 1 - message("Matching from second catalogue to first") - match_index_21, count_21 = find_matching_halos( - length_bound2, - offset_bound2, - ids_bound2, - type_bound2, - host_index2, - length_bound1, - offset_bound1, - ids_bound1, - type_bound1, - host_index1, - args.nr_particles, - use_type, - args.to_field_halos_only, - ) - total_nr_halos = comm.allreduce(len(match_index_21)) - total_nr_matched = comm.allreduce(np.sum(match_index_21 >= 0)) - message(f" Matched {total_nr_matched} of {total_nr_halos} halos") - - # Check for consistent matches in both directions - message("Checking for consistent matches") - consistent_12 = consistent_match(match_index_12, match_index_21) - consistent_21 = consistent_match(match_index_21, match_index_12) - - # Write the output - def write_output_field(name, data, description): - dataset = phdf5.collective_write(outfile, name, data, comm) - dataset.attrs["Description"] = description - - message("Writing output") - with h5py.File(args.output_file, "w", driver="mpio", comm=comm) as outfile: - # Write input parameters - params = outfile.create_group("Parameters") - for name, value in vars(args).items(): - if value is not None: - params.attrs[name] = value - # Matching from first catalogue to second - write_output_field( - "BoundParticleNr1", - length_bound1, - "Number of bound particles in each halo in the first catalogue", - ) - write_output_field( - "MatchIndex1to2", - match_index_12, - "For each halo in the first catalogue, index of the matching halo in the second", - ) - write_output_field( - "MatchCount1to2", - count_12, - f"How many of the {args.nr_particles} most bound particles from the halo in the first catalogue are in the matched halo in the second", - ) - write_output_field( - "Consistent1to2", - consistent_12, - "Whether the match from first to second catalogue is consistent with second to first (1) or not (0)", - ) - # Matching from second catalogue to first - write_output_field( - "BoundParticleNr2", - length_bound2, - "Number of bound particles in each halo in the second catalogue", - ) - write_output_field( - "MatchIndex2to1", - match_index_21, - "For each halo in the second catalogue, index of the matching halo in the first", - ) - write_output_field( - "MatchCount2to1", - count_21, - f"How many of the {args.nr_particles} most bound particles from the halo in the second catalogue are in the matched halo in the first", - ) - write_output_field( - "Consistent2to1", - consistent_21, - "Whether the match from second to first catalogue is consistent with first to second (1) or not (0)", - ) - comm.barrier() - message("Done.") diff --git a/misc/README.md b/misc/README.md new file mode 100644 index 00000000..76831203 --- /dev/null +++ b/misc/README.md @@ -0,0 +1,76 @@ +## Retrieving the evolution of subhaloes for selected properties + +This repository contains a script (`get_evolution_HBT_tracks.py`) used to generate +a HDF5 file that contains the property evolution for a subset of selected subhalos. +Using the script is recommended when looking at the evolution of more than ~1000 +subhaloes at a time, as above this limit h5py fancy indexing is much slower than +loading the whole catalogue directly. + +The script works by opening in parallel each of the SOAP catalogue files available in +the provided base directory, finding the target `TrackIDs` and getting the values of each +property of interest. After opening every available SOAP catalogue, the collected +data is saved to an HDF5 file. + +It only supports SOAP catalogues generated using HBT-HERONS or HBT+. + +### Running + +The MPI routines used to load the SOAP catalogues in parallel use the [VirgoDC](https://github.com/jchelly/VirgoDC) +package. If using COSMA, you can run `./scripts/cosma_python_env.sh` from the SOAP base folder +to install the required packages in a virtual enviroment. + +The following options need to be provided when running the scripts: + + - `SOAP_basedir`: Path to the directory containing the SOAP catalogues for the run, which are assumed to be named as `SOAP_basedir/halo_properties_*.hdf5`. + - `output_file`: Name of the HDF5 file where the property evolution will be saved for each subhalo. + - `-tracks` or `-t`: Path to a text file containing the `TrackId` of the subhaloes of interest. Each row should only contain a single `TrackId` value. + - `-properties` or `-p`: Path to a text file containing the SOAP properties to save. Each row should only specify a single property, which should be the absolute path to the dataset within the SOAP catalogue (e.g. `ExclusiveSphere/50kpc/StellarMass`). + +To run in MPI (recommended), you can do: + +```bash +mpirun -np -- python3 -m mpi4py get_evolution_HBT_tracks.py -tracks -properties +``` +### Output + +The HDF5 file saved by the script has a similar structure to SOAP catalogues, +but with a few important differences. Here are the main highlights: + + - There is no unit information. Additionally, quantities can be in comoving or physical units depending on what SOAP decided to use and save. Triple check what SOAP does. + - The name of each property dataset matches the name in the original SOAP catalogues. + - Each row in a given property dataset corresponds to a different TrackId, with the column being the value of the corresponding property at different output times. + - An additional dataset indicating the output time of SOAP catalogues (`Redshift`) is always provided. + +## Matching halos between outputs + +This repository also contains a script to find halos which contain the same +particle IDs between two outputs. It can be used to find the same halos between +different snapshots or between hydro and dark matter only simulations. + +For each halo in the first output we find the N most bound particle IDs and +determine which halo in the second output contains the largest number of these +IDs. This matching process is then repeated in the opposite direction and we +check for cases were we have consistent matches in both directions. + +### Matching to field halos only + +The `--centrals-only` flag can be used to match central halos +between outputs. If it is set we follow the +first `nr_particles` most bound particles from each central subhalo as usual, but when +locating them in the other output any particles in satellite subhalos +are treated as belonging to the host halo. + +In this mode central subhalos in one catalogue will only ever be matched to central +subhalos in the other catalogue. Satellite subhalos are not matched. + +### Output + +The output is a HDF5 file with the following datasets: + + * `MatchIndex1to2` - for each halo in the first catalogue, index of the matching halo in the second + * `MatchCount1to2` - how many of the most bound particles from the halo in the first catalogue are in the matched halo in the second + * `Consistent1to2` - whether the match from first to second catalogue is consistent with second to first (1) or not (0) + +There are corresponding datasets with `1` and `2` reversed with information about matching in the opposite direction. + + diff --git a/misc/calculate_fof_radii.py b/misc/calculate_fof_radii.py new file mode 100644 index 00000000..a1aca2c3 --- /dev/null +++ b/misc/calculate_fof_radii.py @@ -0,0 +1,456 @@ +#!/bin/env python + +""" +calculate_fof_radii.py + +This script calculates the maximum particle radii for each FOF. +Usage: + + mpirun -- python misc/calculate_fof_radii.py \ + --snap-basename=SNAPSHOT \ + --fof-basename=FOF \ + --output-basename=OUTPUT + +where SNAPSHOT is the basename of the snapshot files (the snapshot +name without the .{file_nr}.hdf5 suffix), FOF is the basename of the +fof catalogues, and OUTPUT is the basename of the output fof catalogues. + +For a full list of optional arguments run: + python misc/calculate_fof_radii.py -h +""" + +import argparse +import os +import time + +from mpi4py import MPI + +comm = MPI.COMM_WORLD +comm_rank = comm.Get_rank() +comm_size = comm.Get_size() + +import h5py +import numpy as np +import unyt + +import virgo.mpi.parallel_sort as psort +import virgo.mpi.parallel_hdf5 as phdf5 +from virgo.mpi.gather_array import gather_array + +# Parse arguments +parser = argparse.ArgumentParser( + description=("Script to calculate extent of FoF groups.") +) +parser.add_argument( + "--snap-basename", + type=str, + required=True, + help=( + "The basename for the snapshot files (the snapshot " + "name without the .{file_nr}.hdf5 suffix)" + ), +) +parser.add_argument( + "--fof-basename", + type=str, + required=True, + help="The basename for the input FoF files", +) +parser.add_argument( + "--output-basename", + type=str, + required=True, + help="The basename for the output files", +) +parser.add_argument( + "--null-fof-id", + type=int, + required=False, + default=2147483647, + help="The FOFGroupIDs of particles not in a FOF group", +) +parser.add_argument( + "--copy-datasets", + action="store_true", + help="Copy datasets from input FoF files (otherwise a link is created)", +) +parser.add_argument( + "--n-test", + type=int, + required=False, + default=0, + help="The number of FOFs to check. If -1 all objects will be checked", +) +parser.add_argument( + "--single-fof-file", + action="store_true", + help="If there is a single FoF file even though there are distributed snapshots", +) +parser.add_argument( + "--recalculate-sizes", + action="store_true", + help="Whether to calculate the number of particles in each FOF group", +) +parser.add_argument( + "--recalculate-centres", + action="store_true", + help="Whether to calculate the CoM of each FOF group", +) +parser.add_argument( + "--reset-ids", + action="store_true", + help=( + "Whether to set the group ids in the FOF catalgoues to 1...N ", + "(This is required for some runs as there was a bug where the incorrect", + "IDs were output)", + ), +) + +args = parser.parse_args() +snap_filename = args.snap_basename + ".{file_nr}.hdf5" +if args.single_fof_file: + fof_filename = args.fof_basename + ".hdf5" + output_filename = args.output_basename + ".hdf5" +else: + fof_filename = args.fof_basename + ".{file_nr}.hdf5" + output_filename = args.output_basename + ".{file_nr}.hdf5" +os.makedirs(os.path.dirname(output_filename), exist_ok=True) + +if comm_rank == 0: + start_time = time.time() + with h5py.File(fof_filename.format(file_nr=0), "r") as file: + fof_header = dict(file["Header"].attrs) + unit_attrs = { + "Radii": dict(file["Groups/Centres"].attrs), + "Centres": dict(file["Groups/Centres"].attrs), + "Sizes": dict(file["Groups/Sizes"].attrs), + "GroupIDs": dict(file["Groups/GroupIDs"].attrs), + } + desc = "Distance to the particle furthest from the centre" + unit_attrs["Radii"]["Description"] = desc + with h5py.File(snap_filename.format(file_nr=0), "r") as file: + snap_header = dict(file["Header"].attrs) + print(f"Running with {comm_size} ranks") + for k, v in vars(args).items(): + print(f" {k}: {v}") + print(f' nr_files: {fof_header["NumFilesPerSnapshot"][0]}') +else: + fof_header = None + snap_header = None + unit_attrs = None +fof_header = comm.bcast(fof_header) +snap_header = comm.bcast(snap_header) +unit_attrs = comm.bcast(unit_attrs) + +boxsize = snap_header["BoxSize"] +ptypes = np.where(snap_header["TotalNumberOfParticles"] != 0)[0] +nr_files = fof_header["NumFilesPerSnapshot"][0] + +# If we are recalculating something we should just a create a fresh file +if args.recalculate_sizes or args.reset_ids or args.recalculate_centres: + assert args.copy_datasets + + +def copy_attrs(src_obj, dst_obj): + for key, val in src_obj.attrs.items(): + dst_obj.attrs[key] = val + + +def copy_object(src_obj, dst_obj, src_filename, prefix="", skip_datasets=False): + copy_attrs(src_obj, dst_obj) + for name, item in src_obj.items(): + if isinstance(item, h5py.Dataset): + if skip_datasets and (item.name != "/Header/PartTypeNames"): + continue + if (item.name == "/Groups/Sizes") and args.recalculate_sizes: + continue + if (item.name == "/Groups/Centres") and args.recalculate_centres: + continue + if (item.name == "/Groups/GroupIDs") and args.reset_ids: + continue + if args.copy_datasets: + src_obj.copy(name, dst_obj) + else: + shape = item.shape + dtype = item.dtype + layout = h5py.VirtualLayout(shape=shape, dtype=dtype) + vsource = h5py.VirtualSource(src_filename, prefix + name, shape=shape) + layout[...] = vsource + dst_obj.create_virtual_dataset(name, layout) + copy_attrs(item, dst_obj[name]) + elif isinstance(item, h5py.Group): + new_group = dst_obj.create_group(name) + copy_object( + item, + new_group, + src_filename, + prefix + name + "/", + skip_datasets=skip_datasets, + ) + + +# Assign files to ranks +files_per_rank = np.zeros(comm_size, dtype=int) +files_per_rank[:] = nr_files // comm_size +remainder = nr_files % comm_size +if remainder > 0: + step = max(nr_files // (remainder + 1), 1) + for i in range(remainder): + files_per_rank[(i * step) % comm_size] += 1 +first_file = np.cumsum(files_per_rank) - files_per_rank +assert sum(files_per_rank) == nr_files + +# Create output files +for i_file in range( + first_file[comm_rank], first_file[comm_rank] + files_per_rank[comm_rank] +): + src_filename = fof_filename.format(file_nr=i_file) + dst_filename = output_filename.format(file_nr=i_file) + with ( + h5py.File(src_filename, "r") as src_file, + h5py.File(dst_filename, "w") as dst_file, + ): + rel_filename = os.path.relpath(src_filename, os.path.dirname(dst_filename)) + copy_object(src_file, dst_file, rel_filename) + +# Load FOF catalogue +fof_file = phdf5.MultiFile( + fof_filename, file_nr_attr=("Header", "NumFilesPerSnapshot"), comm=comm +) +fof_sizes = fof_file.read(f"Groups/Sizes") +fof_centres = fof_file.read(f"Groups/Centres") +fof_group_ids = fof_file.read(f"Groups/GroupIDs") +if args.reset_ids: + n_fof = fof_group_ids.shape[0] + fof_offset = comm.scan(n_fof, op=MPI.SUM) - n_fof + fof_group_ids = np.arange(n_fof, dtype=fof_group_ids.dtype) + fof_offset + 1 + +# Initialise arrays for storing results +fof_radius = np.zeros_like(fof_centres[:, 0]) +total_part_counts = np.zeros_like(fof_sizes) + +# Open snapshot file +snap_file = phdf5.MultiFile( + snap_filename, file_nr_attr=("Header", "NumFilesPerSnapshot"), comm=comm +) + +if args.recalculate_centres: + if comm_rank == 0: + print(f"Recalculating centres") + + # Load the FOF masses + fof_mass = fof_file.read(f"Groups/Masses") + # Wipe the centres value we loaded from the catalogues + fof_centres[:] = 0 + # Initialise an array to store the origin we will use for each FOF group + fof_origin = -1 * np.ones_like(fof_centres) + + for ptype in ptypes: + if comm_rank == 0: + print(f" Processing PartType{ptype}") + + # Load particle positions and their FOF IDs + part_pos = snap_file.read(f"PartType{ptype}/Coordinates") + part_fof_ids = snap_file.read(f"PartType{ptype}/FOFGroupIDs") + mass_field = "DynamicalMasses" if ptype == 5 else "Masses" + part_mass = snap_file.read(f"PartType{ptype}/{mass_field}") + + # Ignore particles which aren't part of a FOF group + mask = part_fof_ids != args.null_fof_id + part_pos = part_pos[mask] + part_fof_ids = part_fof_ids[mask] + part_mass = part_mass[mask] + + if comm_rank == 0: + print(f" Loaded particles") + + # Get the current FOF origin for each particle + idx = psort.parallel_match(part_fof_ids, fof_group_ids, comm=comm) + assert np.all(idx != -1), "FOFs could not be found for some particles" + part_origin = psort.fetch_elements(fof_origin, idx, comm=comm) + + if comm_rank == 0: + print(f" Matched to FOF catalogue") + + # For each FOF that does not yet have an origin we use the max(x, y, z) + # of the particles in this group as the origin + tmp_part_pos = part_pos[part_origin[:, 0] == -1] + tmp_idx = idx[part_origin[:, 0] == -1] + assert np.all(tmp_part_pos >= 0) + psort.reduce_elements( + fof_origin, tmp_part_pos, tmp_idx, op=np.maximum, comm=comm + ) + del tmp_part_pos, tmp_idx + + # Get the current FOF origin for each particle + part_origin = psort.fetch_elements(fof_origin, idx, comm=comm) + assert np.all(part_origin >= 0) + + # Centre the particles + shift = (boxsize[None, :] / 2) - part_origin + part_pos = ((part_pos + shift) % boxsize[None, :]) - (boxsize[None, :] / 2) + + # Load FOF mass of each particle + part_fof_mass = psort.fetch_elements(fof_mass, idx, comm=comm) + # Add CoM contribution + part_com = part_mass[:, None] * part_pos / part_fof_mass[:, None] + psort.reduce_elements(fof_centres, part_com, idx, op=MPI.SUM, comm=comm) + + # Undo the shift that we introduced when calculating the COM + fof_centres = (fof_centres + fof_origin) % boxsize[None, :] + +for ptype in ptypes: + if comm_rank == 0: + print(f"Processing PartType{ptype}") + + # Load particle positions and their FOF IDs + part_pos = snap_file.read(f"PartType{ptype}/Coordinates") + part_fof_ids = snap_file.read(f"PartType{ptype}/FOFGroupIDs") + + # Ignore particles which aren't part of a FOF group + mask = part_fof_ids != args.null_fof_id + part_pos = part_pos[mask] + part_fof_ids = part_fof_ids[mask] + + if comm_rank == 0: + print(f" Loaded particles") + + # Get the centre of the FOF each particle is part of + idx = psort.parallel_match(part_fof_ids, fof_group_ids, comm=comm) + assert np.all(idx != -1), "FOFs could not be found for some particles" + part_centre = psort.fetch_elements(fof_centres, idx, comm=comm) + + if comm_rank == 0: + print(f" Matched to FOF catalogue") + + # Centre the particles + shift = (boxsize[None, :] / 2) - part_centre + part_pos = ((part_pos + shift) % boxsize[None, :]) - (boxsize[None, :] / 2) + + # Calculate max particle radius for each FOF ID + part_radius = np.sqrt(np.sum(part_pos**2, axis=1)) + psort.reduce_elements(fof_radius, part_radius, idx, op=np.maximum, comm=comm) + + # Count the number of particles found for each FOF + unique_fof_ids, unique_counts = psort.parallel_unique( + part_fof_ids, + return_counts=True, + comm=comm, + ) + + # Keep track of number of particles in each FOF (to compare with "Groups/Sizes") + idx = psort.parallel_match(fof_group_ids, unique_fof_ids, comm=comm) + mask = idx != -1 + total_part_counts[mask] += psort.fetch_elements(unique_counts, idx[mask], comm=comm) + +# Carry out some sanity checks +assert np.all(fof_radius > 0) +output_data = {"Radii": fof_radius} +if args.recalculate_sizes: + output_data["Sizes"] = total_part_counts +else: + msg = "Not all particles found for some FOFs" + assert np.all(total_part_counts == fof_sizes), msg +if args.reset_ids: + output_data["GroupIDs"] = fof_group_ids +if args.recalculate_centres: + output_data["Centres"] = fof_centres + +# Write data to file +elements_per_file = fof_file.get_elements_per_file("Groups/GroupIDs") +fof_file.write( + output_data, + elements_per_file, + filenames=output_filename, + mode="r+", + group="Groups", + attrs=unit_attrs, +) + +if (comm_rank == 0) and (not args.single_fof_file): + + # Create virtual file + dst_filename = args.output_basename + ".hdf5" + with h5py.File(dst_filename, "w") as dst_file: + + # Copy original virtual file, skip datasets + src_filename = args.fof_basename + ".hdf5" + with h5py.File(src_filename, "r") as src_file: + copy_object(src_file, dst_file, src_filename, skip_datasets=True) + nr_groups = dst_file["Header"].attrs["NumGroups_Total"][0] + + # Get number of groups in each chunk file + counts, shapes, dtypes, attrs = [], [], [], {} + for i_file in range(nr_files): + src_filename = output_filename.format(file_nr=i_file) + with h5py.File(src_filename, "r") as src_file: + if i_file == 0: + props = list(src_file["Groups"].keys()) + for prop in props: + shapes.append(src_file[f"Groups/{prop}"].shape) + dtypes.append(src_file[f"Groups/{prop}"].dtype) + attrs[prop] = dict(src_file[f"Groups/{prop}"].attrs) + counts.append(src_file["Header"].attrs["NumGroups_ThisFile"][0]) + + # Create virtual datasets + for prop, shape, dtype in zip(props, shapes, dtypes): + full_shape = (nr_groups, *shape[1:]) + layout = h5py.VirtualLayout(shape=full_shape, dtype=dtype) + + offset = 0 + for i_file in range(nr_files): + src_filename = output_filename.format(file_nr=i_file) + rel_filename = os.path.relpath( + src_filename, os.path.dirname(dst_filename) + ) + count = counts[i_file] + layout[offset : offset + count] = h5py.VirtualSource( + rel_filename, f"Groups/{prop}", shape=(count, *shape[1:]) + ) + offset += count + dset = dst_file.create_virtual_dataset( + f"Groups/{prop}", layout, fillvalue=-999 + ) + for k, v in attrs[prop].items(): + dset.attrs[k] = v + + print("New files generated!") + +if comm_rank == 0: + print(f"Took {int(time.time() - start_time)} seconds") + +if (comm_rank == 0) and (args.n_test != 0): + print("Testing we can load all particles") + import swiftsimio as sw + import tqdm + + # Load the FOF file + fof = sw.load(args.output_basename + ".hdf5") + min_pos = fof.fof_groups.centres - fof.fof_groups.radii[:, None] + max_pos = fof.fof_groups.centres + fof.fof_groups.radii[:, None] + + n_test = args.n_test if args.n_test != -1 else fof.fof_groups.sizes.shape[0] + to_test = np.random.choice( + np.arange(fof.fof_groups.sizes.shape[0]), n_test, replace=False + ) + for i_fof in tqdm.tqdm(to_test): + + # Create mask and load data + snap_filename = args.snap_basename + ".hdf5" + mask = sw.mask(snap_filename) + load_region = [ + [min_pos[i_fof, 0], max_pos[i_fof, 0]], + [min_pos[i_fof, 1], max_pos[i_fof, 1]], + [min_pos[i_fof, 2], max_pos[i_fof, 2]], + ] + mask.constrain_spatial(load_region) + snap = sw.load(snap_filename, mask=mask) + + # Check we loaded all the particles + n_total = 0 + for ptype in ["gas", "dark_matter", "stars", "black_holes"]: + fof_id = fof.fof_groups.group_ids[i_fof].value + part_fof_ids = getattr(snap, ptype).fofgroup_ids.value + n_total += np.sum(part_fof_ids == fof_id) + msg = f"Failed for object {i_fof}" + assert n_total == fof.fof_groups.sizes[i_fof].value, msg diff --git a/check_group_membership.py b/misc/check_group_membership.py similarity index 100% rename from check_group_membership.py rename to misc/check_group_membership.py diff --git a/check_subhalo_ranking.py b/misc/check_subhalo_ranking.py similarity index 100% rename from check_subhalo_ranking.py rename to misc/check_subhalo_ranking.py diff --git a/tests/compare_new_implementation.py b/misc/compare_new_implementation.py similarity index 59% rename from tests/compare_new_implementation.py rename to misc/compare_new_implementation.py index c1fc2ece..e1383462 100644 --- a/tests/compare_new_implementation.py +++ b/misc/compare_new_implementation.py @@ -1,11 +1,12 @@ import h5py import numpy as np -def test_dataset_values(old_file, new_file, cumulative_path = ''): - ''' + +def test_dataset_values(old_file, new_file, cumulative_path=""): + """ Checks if all the datasets common to two SOAP generated HDF5 files contain the same values. - + Param ----- old_file: h5py.File @@ -15,12 +16,14 @@ def test_dataset_values(old_file, new_file, cumulative_path = ''): cumulative_path: str, opt The full path that we are currently looking at. Should be ignored, as it is only used for printing information when we find a value mismatch. - ''' - + """ + # If we have a dataset, test if values agree between old and new files. if not isinstance(old_file, h5py._hl.group.Group): if not (old_file[()] == new_file[()]).all(): - print(f"Value mismatch ({(old_file[()] != new_file[()]).sum()} entries; {(old_file[()] != new_file[()]).sum()/len(old_file[()]) * 100:.5f}% of all) at dataset {cumulative_path[1:]}") + print( + f"Value mismatch ({(old_file[()] != new_file[()]).sum()} entries; {(old_file[()] != new_file[()]).sum()/len(old_file[()]) * 100:.5f}% of all) at dataset {cumulative_path[1:]}" + ) return # If we still haven't reached a dataset, navigate deeper in the file @@ -32,13 +35,16 @@ def test_dataset_values(old_file, new_file, cumulative_path = ''): print("Keys only present in new file: ", new_keys - old_keys) print() - for key in old_keys.intersection(new_keys): # Iterate over the common keys - test_dataset_values(old_file[key],new_file[key], f'{cumulative_path}/{key}') + for key in old_keys.intersection(new_keys): # Iterate over the common keys + test_dataset_values(old_file[key], new_file[key], f"{cumulative_path}/{key}") + if __name__ == "__main__": - old_soap_file_path = '/cosma8/data/dp004/dc-foro1/colibre/low_res_test/halo_finding/velociraptor/soap/colibre_SOAP_halo_properties_0127.hdf5' - new_soap_file_path = '/cosma8/data/dp004/dc-foro1/colibre/low_res_test/halo_finding/velociraptor/soap/SOAP_halo_properties_0127.hdf5' - - with h5py.File(old_soap_file_path) as old_soap_file, \ - h5py.File(new_soap_file_path) as new_soap_file: - test_dataset_values(old_soap_file, new_soap_file) \ No newline at end of file + old_soap_file_path = "/cosma8/data/dp004/dc-foro1/colibre/low_res_test/halo_finding/velociraptor/soap/colibre_SOAP_halo_properties_0127.hdf5" + new_soap_file_path = "/cosma8/data/dp004/dc-foro1/colibre/low_res_test/halo_finding/velociraptor/soap/SOAP_halo_properties_0127.hdf5" + + with ( + h5py.File(old_soap_file_path) as old_soap_file, + h5py.File(new_soap_file_path) as new_soap_file, + ): + test_dataset_values(old_soap_file, new_soap_file) diff --git a/misc/convert_eagle.py b/misc/convert_eagle.py new file mode 100644 index 00000000..6066e2bc --- /dev/null +++ b/misc/convert_eagle.py @@ -0,0 +1,861 @@ +#!/bin/env python + +""" +convert_eagle.py + +This script converts EAGLE GADGET snapshots into SWIFT snapshots. +Usage: + + mpirun -- python convert_eagle.py \ + --snapshot-basename=SNAPSHOT \ + --output-basename=OUTPUT \ + --membership-basename=MEMBERSHIP + +where SNAPSHOT is the EAGLE snapshot (use the particledata_*** files, +since the normal snapshots don't store SubGroupNumber), and OUTPUT & +MEMBERSHIP are the names of the output files. You must run with +the same number of ranks as input files. + +This script has three mains parts: + We first have to determine the units system of the input snapshots, + and specify the units system of the output snapshots. This step + also includes copying over relevant cosmology information. This + is all done by reading the headers of the input snapshots + + The second part is conversion of the datasets in the input snapshot + to the units system of the output snapshot. This is done by + creating a "properties" dictionary, which contains information + about how to convert every property. More information about the + structure of the dictionary can be found below in the code comments. + + The third part is to sort the new snapshots spatially based on + the SWIFT cell structure. This step requires reading the coordinates + and IDs of every particle before we can save all the other properties. + +Since there are many different GADGET snapshot formats this script +is not guaranteed to work for other simulations, but the basic +structure should be the same. + +For EAGLE the information about which subhalo each particle is bound +to can be found in the particledata_*** files. We therefore use these +files for this script rather than the full snapshots. This allows us +to easily output the membership files that are also required to run +SOAP. If you wanted to use the full snapshots you could obtain the +the SubGroup number by matching with the particledata_*** files based +on ParticleID. +""" + +import argparse +import collections +import os +import glob + +from mpi4py import MPI + +comm = MPI.COMM_WORLD +comm_rank = comm.Get_rank() +comm_size = comm.Get_size() + +import astropy.cosmology +import h5py +import numpy as np +import unyt + +import virgo.mpi.parallel_sort as psort +import virgo.mpi.parallel_hdf5 as phdf5 + +from SOAP.core import swift_units + +# Parse arguments +parser = argparse.ArgumentParser( + description=( + "Script to convert EAGLE snapshots to SWIFT format. " + "Also outputs membership files directly" + ) +) +parser.add_argument( + "--snap-basename", + type=str, + required=True, + help=( + "The basename for the snapshot files (the snapshot " + "name without the .{file_nr}.hdf5 suffix)" + ), +) +parser.add_argument( + "--subfind-basename", + type=str, + required=True, + help=("The basename for the subfind files"), +) +parser.add_argument( + "--output-basename", + type=str, + required=True, + help="The basename for the output files", +) +parser.add_argument( + "--membership-basename", + type=str, + required=True, + help="The basename for the membership files", +) +args = parser.parse_args() +snap_filename = args.snap_basename + ".{file_nr}.hdf5" +subfind_filename = args.subfind_basename + ".{file_nr}.hdf5" +output_filename = args.output_basename + ".{file_nr}.hdf5" +membership_filename = args.membership_basename + ".{file_nr}.hdf5" +os.makedirs(os.path.dirname(output_filename), exist_ok=True) +os.makedirs(os.path.dirname(membership_filename), exist_ok=True) + +if comm_rank == 0: + print("Extracting basic information from EAGLE snapshot header") + with h5py.File(snap_filename.format(file_nr=0), "r") as infile: + n_file = infile["Header"].attrs["NumFilesPerSnapshot"] + h = infile["Header"].attrs["HubbleParam"] + a = infile["Header"].attrs["ExpansionFactor"] + box_size_cmpc = infile["Header"].attrs["BoxSize"] / h + + ptypes = [] + nr_types = infile["Header"].attrs["NumPart_Total"].shape[0] + nr_parts = infile["Header"].attrs["NumPart_Total"] + nr_parts_hw = infile["Header"].attrs["NumPart_Total_HighWord"] + for i in range(nr_types): + if nr_parts[i] > 0 or nr_parts_hw[i] > 0: + ptypes.append(i) + +else: + n_file, ptypes, a, h, box_size_cmpc = None, None, None, None, None +n_file = comm.bcast(n_file) +ptypes = comm.bcast(ptypes) +a = comm.bcast(a) +h = comm.bcast(h) +box_size_cmpc = comm.bcast(box_size_cmpc) + +assert comm_size == n_file + +# Specify the unit system of the output SWIFT snapshot +if comm_rank == 0: + units_header = { + "Unit current in cgs (U_I)": np.array([1.0]), + "Unit length in cgs (U_L)": np.array([3.08567758e24]), + "Unit mass in cgs (U_M)": np.array([1.98841e43]), + "Unit temperature in cgs (U_T)": np.array([1.0]), + "Unit time in cgs (U_t)": np.array([3.08567758e19]), + } + const_cgs_header = { + "T_CMB_0": np.array([2.7255]), + "newton_G": np.array([6.6743e-08]), + "parsec": np.array([3.08567758e18]), + "solar_mass": np.array([1.98841e33]), + "speed_light_c": np.array([2.99792458e10]), + } + const_internal_header = { + "T_CMB_0": np.array([2.7255]), + "newton_G": np.array([43.00917552]), + "parsec": np.array([1.0e-06]), + "solar_mass": np.array([1.0e-10]), + "speed_light_c": np.array([299792.458]), + } + + # Pretend these are in an HDF5 file so we can create a unyt + # registry using an existing SOAP function + MockGroup = collections.namedtuple("MockGroup", ["attrs"]) + mock_swift_snap = { + "Units": MockGroup(attrs=units_header), + "InternalCodeUnits": MockGroup(attrs=units_header), + "PhysicalConstants/CGS": MockGroup(attrs=const_cgs_header), + "Cosmology": MockGroup( + attrs={ + "Scale-factor": [a], + "h": [h], + } + ), + } + reg = swift_units.unit_registry_from_snapshot(mock_swift_snap) + +else: + reg = None +reg = comm.bcast(reg) + +# Extract information required for headers of output SWIFT snapshot +if comm_rank == 0: + with h5py.File(snap_filename.format(file_nr=0), "r") as infile: + runtime = infile["RuntimePars"] + parameters_header = { + "Gravity:comoving_DM_softening": runtime.attrs["SofteningHalo"], + "Gravity:max_physical_DM_softening": runtime.attrs["SofteningHaloMaxPhys"], + "Gravity:comoving_baryon_softening": runtime.attrs["SofteningGas"], + "Gravity:max_physical_baryon_softening": runtime.attrs[ + "SofteningGasMaxPhys" + ], + "EAGLE:InitAbundance_Hydrogen": runtime.attrs["InitAbundance_Hydrogen"], + "EAGLE:InitAbundance_Helium": runtime.attrs["InitAbundance_Helium"], + "EAGLE:EOS_Jeans_GammaEffective": runtime.attrs["EOS_Jeans_GammaEffective"], + "EAGLE:EOS_Jeans_TempNorm_K": runtime.attrs["EOS_Jeans_TempNorm_K"], + } + + # Check units are indeed what we are assuming below, since HO, + # BoxSize, etc must match with SWIFT internal units + snap_L_in_cm = (1 * unyt.Mpc).to("cm").value + assert np.isclose(units_header["Unit length in cgs (U_L)"][0], snap_L_in_cm) + snap_M_in_g = (10**10 * unyt.Msun).to("g").value + assert np.isclose(units_header["Unit mass in cgs (U_M)"][0], snap_M_in_g) + snap_V = (1 * unyt.km / unyt.s).to("cm/s").value + snap_t_in_s = snap_L_in_cm / snap_V + assert np.isclose(units_header["Unit time in cgs (U_t)"][0], snap_t_in_s) + + # Cosmology + header = infile["Header"] + z = header.attrs["Redshift"] + H = astropy.cosmology.Planck13.H(z).value + G = const_internal_header["newton_G"][0] + critical_density = 3 * (H**2) / (8 * np.pi * G) + cosmology_header = { + "Omega_b": header.attrs["OmegaBaryon"], + "Omega_m": header.attrs["Omega0"], + "Omega_k": 0, + "Omega_nu_0": 0, + "Omega_r": astropy.cosmology.Planck13.Ogamma(z), + "Omega_g": astropy.cosmology.Planck13.Ogamma(z), + "Omega_lambda": header.attrs["OmegaLambda"], + "Redshift": z, + "H0 [internal units]": h * 100, + "H [internal units]": H, + "Critical density [internal units]": critical_density, + "Scale-factor": header.attrs["ExpansionFactor"], + "h": h, + "w": -1, + "w_0": -1, + "w_a": 0, + } + swift_header = { + "BoxSize": [box_size_cmpc, box_size_cmpc, box_size_cmpc], + "NumFilesPerSnapshot": [n_file], + "NumPartTypes": [max(ptypes) + 1], + "Scale-factor": [header.attrs["ExpansionFactor"]], + "Dimension": [3], + "Redshift": [z], + "RunName": snap_filename.encode(), + } + + +# The information in this dictionary is used to convert the datasets in +# the EAGLE files into datasets in the output SWIFT snapshot. The key +# should be the name of the dataset in the EAGLE file. The values are: +# +# 'swift_name' - name of the dataset in output swift file +# 'exponents' - the dimensions of the units of the dataset +# 'a_exponent' - the scale factor exponent if the dataset is stored +# in comoving coordinates +# 'description' - short description of the dataset +# 'conversion_factor' - factor required to convert the values of the +# dataset to the units system of the output +# SWIFT snapshot. Note the resulting units +# should be h-free +# +# In the case of EAGLE snapshots most of this information is available +# in the metadata of the snapshots themselves. We therefore set those +# values as None in the properties dictionary, and read them in directly. +properties = { + "PartType0": { + "Coordinates": { + "swift_name": "Coordinates", + "exponents": {"L": 1, "M": 0, "T": 0, "t": 0}, + "a_exponent": None, + "description": None, + "conversion_factor": None, + }, + "Velocity": { + "swift_name": "Velocities", + "exponents": {"L": 1, "M": 0, "T": 0, "t": -1}, + "a_exponent": None, + "description": None, + "conversion_factor": None, + }, + "Mass": { + "swift_name": "Masses", + "exponents": {"L": 0, "M": 1, "T": 0, "t": 0}, + "a_exponent": None, + "description": None, + "conversion_factor": None, + }, + "GroupNumber": { + "swift_name": "FOFGroupIDs", + "exponents": {"L": 0, "M": 0, "T": 0, "t": 0}, + "a_exponent": None, + "description": None, + "conversion_factor": None, + }, + "ParticleIDs": { + "swift_name": "ParticleIDs", + "exponents": {"L": 0, "M": 0, "T": 0, "t": 0}, + "a_exponent": None, + "description": None, + "conversion_factor": None, + }, + "Density": { + "swift_name": "Densities", + "exponents": {"L": -3, "M": 1, "T": 0, "t": 0}, + "a_exponent": None, + "description": None, + "conversion_factor": None, + }, + "InternalEnergy": { + "swift_name": "InternalEnergies", + "exponents": {"L": 2, "M": 0, "T": 0, "t": -2}, + "a_exponent": None, + "description": None, + "conversion_factor": None, + }, + "IronMassFracFromSNIa": { + "swift_name": "IronMassFractionsFromSNIa", + "exponents": {"L": 0, "M": 0, "T": 0, "t": 0}, + "a_exponent": None, + "description": None, + "conversion_factor": None, + }, + "Metallicity": { + "swift_name": "MetalMassFractions", + "exponents": {"L": 0, "M": 0, "T": 0, "t": 0}, + "a_exponent": None, + "description": None, + "conversion_factor": None, + }, + "StarFormationRate": { + "swift_name": "StarFormationRates", + "exponents": {"L": 0, "M": 1, "T": 0, "t": -1}, + "a_exponent": None, + "description": None, + "conversion_factor": None, + }, + "Temperature": { + "swift_name": "Temperatures", + "exponents": {"L": 0, "M": 0, "T": 1, "t": 0}, + "a_exponent": None, + "description": None, + "conversion_factor": None, + }, + # Note this is handled differently to normal properties. + # EAGLE stores each element in its own group, SWIFT expects + # all elements to be in a 2D array + "ElementMassFractions": { + "swift_name": "ElementMassFractions", + "exponents": None, + "a_exponent": None, + "description": None, + "conversion_factor": None, + }, + }, + "PartType1": { + "Coordinates": { + "swift_name": "Coordinates", + "exponents": {"L": 1, "M": 0, "T": 0, "t": 0}, + "a_exponent": None, + "description": None, + "conversion_factor": None, + }, + "Velocity": { + "swift_name": "Velocities", + "exponents": {"L": 1, "M": 0, "T": 0, "t": -1}, + "a_exponent": None, + "description": None, + "conversion_factor": None, + }, + # Note this is handled differently to normal properties. + # All particles have the same mass in EAGLE, so there + # is no mass dataset for dark matter. + "Mass": { + "swift_name": "Masses", + "exponents": {"L": 0, "M": 1, "T": 0, "t": 0}, + "a_exponent": 0, + "description": "Particle mass", + "conversion_factor": None, + }, + "GroupNumber": { + "swift_name": "FOFGroupIDs", + "exponents": {"L": 0, "M": 0, "T": 0, "t": 0}, + "a_exponent": None, + "description": None, + "conversion_factor": None, + }, + "ParticleIDs": { + "swift_name": "ParticleIDs", + "exponents": {"L": 0, "M": 0, "T": 0, "t": 0}, + "a_exponent": None, + "description": None, + "conversion_factor": None, + }, + }, + "PartType4": { + "Coordinates": { + "swift_name": "Coordinates", + "exponents": {"L": 1, "M": 0, "T": 0, "t": 0}, + "a_exponent": None, + "description": None, + "conversion_factor": None, + }, + "Velocity": { + "swift_name": "Velocities", + "exponents": {"L": 1, "M": 0, "T": 0, "t": -1}, + "a_exponent": None, + "description": None, + "conversion_factor": None, + }, + "Mass": { + "swift_name": "Masses", + "exponents": {"L": 0, "M": 1, "T": 0, "t": 0}, + "a_exponent": None, + "description": None, + "conversion_factor": None, + }, + "GroupNumber": { + "swift_name": "FOFGroupIDs", + "exponents": {"L": 0, "M": 0, "T": 0, "t": 0}, + "a_exponent": None, + "description": None, + "conversion_factor": None, + }, + "ParticleIDs": { + "swift_name": "ParticleIDs", + "exponents": {"L": 0, "M": 0, "T": 0, "t": 0}, + "a_exponent": None, + "description": None, + "conversion_factor": None, + }, + "BirthDensity": { + "swift_name": "BirthDensities", + "exponents": {"L": -3, "M": 1, "T": 0, "t": 0}, + "a_exponent": None, + "description": None, + "conversion_factor": None, + }, + "InitialMass": { + "swift_name": "InitialMasses", + "exponents": {"L": 0, "M": 1, "T": 0, "t": 0}, + "a_exponent": None, + "description": None, + "conversion_factor": None, + }, + "IronMassFracFromSNIa": { + "swift_name": "IronMassFractionsFromSNIa", + "exponents": {"L": 0, "M": 0, "T": 0, "t": 0}, + "a_exponent": None, + "description": None, + "conversion_factor": None, + }, + "Metallicity": { + "swift_name": "MetalMassFractions", + "exponents": {"L": 0, "M": 0, "T": 0, "t": 0}, + "a_exponent": None, + "description": None, + "conversion_factor": None, + }, + "StellarFormationTime": { + "swift_name": "BirthScaleFactors", + "exponents": {"L": 0, "M": 0, "T": 0, "t": 0}, + "a_exponent": None, + "description": None, + "conversion_factor": None, + }, + # Note this is handled differently to normal properties. + # EAGLE stores each element in its own group, SWIFT expects + # all elements to be in a 2D array + "ElementMassFractions": { + "swift_name": "ElementMassFractions", + "exponents": None, + "a_exponent": None, + "description": None, + "conversion_factor": None, + }, + }, + "PartType5": { + "Coordinates": { + "swift_name": "Coordinates", + "exponents": {"L": 1, "M": 0, "T": 0, "t": 0}, + "a_exponent": None, + "description": None, + "conversion_factor": None, + }, + "Velocity": { + "swift_name": "Velocities", + "exponents": {"L": 1, "M": 0, "T": 0, "t": -1}, + "a_exponent": None, + "description": None, + "conversion_factor": None, + }, + "BH_Mass": { + "swift_name": "SubgridMasses", + "exponents": {"L": 0, "M": 1, "T": 0, "t": 0}, + "a_exponent": None, + "description": None, + "conversion_factor": None, + }, + "Mass": { + "swift_name": "DynamicalMasses", + "exponents": {"L": 0, "M": 1, "T": 0, "t": 0}, + "a_exponent": None, + "description": None, + "conversion_factor": None, + }, + "GroupNumber": { + "swift_name": "FOFGroupIDs", + "exponents": {"L": 0, "M": 0, "T": 0, "t": 0}, + "a_exponent": None, + "description": None, + "conversion_factor": None, + }, + "ParticleIDs": { + "swift_name": "ParticleIDs", + "exponents": {"L": 0, "M": 0, "T": 0, "t": 0}, + "a_exponent": None, + "description": None, + "conversion_factor": None, + }, + "BH_Mdot": { + "swift_name": "AccretionRates", + "exponents": {"L": 0, "M": 1, "T": 0, "t": -1}, + "a_exponent": None, + "description": None, + "conversion_factor": None, + }, + }, +} +if comm_rank == 0: + print("Calculating conversion factor for each property using dataset metadata") + with h5py.File(snap_filename.format(file_nr=0), "r") as infile: + for ptype in ptypes: + for k in infile[f"PartType{ptype}"]: + if k not in properties.get(f"PartType{ptype}", {}): + continue + a_exponent = infile[f"PartType{ptype}/{k}"].attrs["aexp-scale-exponent"] + properties[f"PartType{ptype}"][k]["a_exponent"] = a_exponent + + description = infile[f"PartType{ptype}/{k}"].attrs["VarDescription"] + properties[f"PartType{ptype}"][k]["description"] = description + + # Calculate the conversion factor to (h-free) SWIFT internal units + cgs_factor = infile[f"PartType{ptype}/{k}"].attrs["CGSConversionFactor"] + exponents = properties[f"PartType{ptype}"][k]["exponents"] + swift_cgs_factor = 1 + swift_cgs_factor *= (1 * unyt.Unit("snap_length", registry=reg)).to( + "cm" + ) ** exponents["L"] + swift_cgs_factor *= (1 * unyt.Unit("snap_mass", registry=reg)).to( + "g" + ) ** exponents["M"] + swift_cgs_factor *= (1 * unyt.Unit("snap_time", registry=reg)).to( + "s" + ) ** exponents["t"] + swift_cgs_factor *= ( + 1 * unyt.Unit("snap_temperature", registry=reg) + ).to("K") ** exponents["T"] + conversion_factor = cgs_factor / swift_cgs_factor + + h_exponent = infile[f"PartType{ptype}/{k}"].attrs["h-scale-exponent"] + if h_exponent != 0: + conversion_factor *= h**h_exponent + + properties[f"PartType{ptype}"][k][ + "conversion_factor" + ] = conversion_factor + + # Load DM mass from mass table + if "Mass" in properties.get(f"PartType1", {}): + dm_mass = infile["Header"].attrs["MassTable"][1] / h + properties["PartType1"]["Mass"]["conversion_factor"] = dm_mass + + # Get list of elements for ElementMassFractions + if "ElementMassFractions" in properties.get(f"PartType0", {}): + elements = sorted(infile["PartType0/ElementAbundance"].keys()) + else: + elements = None + if "ElementMassFractions" in properties.get(f"PartType4", {}): + star_elements = sorted(infile["PartType4/ElementAbundance"].keys()) + if elements is not None: + for i in range(len(elements)): + assert elements[i] == star_elements[i] + elements = star_elements +properties = comm.bcast(properties) + + +# Define cell structure required for SWIFT output +n_cell = 16 +cell_size_cmpc = box_size_cmpc / n_cell +cell_counts = {} +cell_offsets = {} +cell_files = {} +idx = np.arange(n_cell) +cell_indices = np.meshgrid(idx, idx, idx, indexing="ij") +cell_indices = np.stack(cell_indices, axis=-1).reshape(-1, 3) +cell_centres = cell_indices * cell_size_cmpc +cell_centres += cell_size_cmpc / 2 + +# Load and process the snapshot files using multiple ranks +snap_file = phdf5.MultiFile( + snap_filename, file_nr_attr=("Header", "NumFilesPerSnapshot"), comm=comm +) +# Load the SubFind catalogue and create an array that links the GroupNumber +# and SubGroupNumber of a subhalo to its index within the subhalo catalogue +# (for creating the membership files) +subfind_file = phdf5.MultiFile( + subfind_filename, file_nr_attr=("Header", "NumFilesPerSnapshot"), comm=comm +) +subfind_id = subfind_file.read("Subhalo/GroupNumber").astype(np.int64) +subfind_id <<= 32 +subfind_id += subfind_file.read("Subhalo/SubGroupNumber").astype(np.int64) + +create_output_file = True +create_membership_file = True +for ptype in ptypes: + + if comm_rank == 0: + print(f"Sorting PartType{ptype} arrays") + # We sort the particles spatially (based on what cell they are in), + # and then by particle ID + pos = snap_file.read(f"PartType{ptype}/Coordinates") / h + particle_id = snap_file.read(f"PartType{ptype}/ParticleIDs").astype(np.int64) + + cell_indices = (pos // cell_size_cmpc).astype(np.int64) + assert np.min(cell_indices) >= 0 + assert np.max(cell_indices) < n_cell + sort_hash_dtype = [("cell_index", np.int64), ("particle_id", np.int64)] + sort_hash = np.zeros(cell_indices.shape[0], dtype=sort_hash_dtype) + sort_hash["cell_index"] += cell_indices[:, 0] * n_cell**2 + sort_hash["cell_index"] += cell_indices[:, 1] * n_cell + sort_hash["cell_index"] += cell_indices[:, 2] + sort_hash["particle_id"] = particle_id + order = psort.parallel_sort(sort_hash, return_index=True, comm=comm) + + # Calculate count of particles in each cell + local_cell_counts = np.bincount( + sort_hash["cell_index"], minlength=n_cell**3 + ).astype("int64") + cell_counts[ptype] = comm.allreduce(local_cell_counts) + + # Calculate how to partition particles across files + cells_per_file = np.zeros(n_file, dtype=int) + cells_per_file[:] = cell_counts[ptype].shape[0] // n_file + remainder = cell_counts[ptype].shape[0] % n_file + cells_per_file[:remainder] += 1 + assert np.sum(cells_per_file) == n_cell**3 + i_cell = np.cumsum(cells_per_file) - 1 + elements_per_file = np.cumsum(cell_counts[ptype])[i_cell] + elements_per_file[1:] -= elements_per_file[:-1] + assert np.sum(elements_per_file) == np.sum(cell_counts[ptype]) + + # Calculate offsets of the first particle in each cell + cell_files[ptype] = np.repeat(np.arange(n_file), cells_per_file) + absolute_offset = np.cumsum(cell_counts[ptype]) - cell_counts[ptype] + file_offset = (np.cumsum(elements_per_file) - elements_per_file)[cell_files[ptype]] + cell_offsets[ptype] = absolute_offset - file_offset + + # Loop though the datasets, sort them spatially, convert their units, + # and write them to the output file + for prop, prop_info in properties[f"PartType{ptype}"].items(): + if prop_info["conversion_factor"] is None: + # The property is missing from snapshots, or we + # would have calculated the value of 'conversion_factor' + continue + + if comm_rank == 0: + print(f"Converting PartType{ptype}/{prop}") + + # DM particles all have the same mass, so are not saved in the snapshots + if (ptype == 1) and (prop == "Mass"): + arr = np.ones(pos.shape[0]) + else: + # Load data from file and sort according to cell structure + arr = snap_file.read(f"PartType{ptype}/{prop}") + arr = psort.fetch_elements(arr, order, comm=comm) + + # Convert to SWIFT internal units + arr *= prop_info["conversion_factor"] + + # Create the units metadata for the SWIFT snapshot + exponents = prop_info["exponents"] + a_exponent = prop_info["a_exponent"] + unit = 1 + unit *= unyt.Unit("snap_length", registry=reg) ** exponents["L"] + unit *= unyt.Unit("snap_mass", registry=reg) ** exponents["M"] + unit *= unyt.Unit("snap_time", registry=reg) ** exponents["t"] + unit *= unyt.Unit("snap_temperature", registry=reg) ** exponents["T"] + unit *= unyt.Unit("a", registry=reg) ** a_exponent + unit_attrs = swift_units.attributes_from_units(unit.units, False, a_exponent) + + # Add some extra metadata + attrs = { + "original_name": prop, + "Description": prop_info["description"], + } + attrs.update(unit_attrs) + + # Write to the output file + arr = psort.repartition(arr, elements_per_file, comm=comm) + if create_output_file: + mode = "w" + create_output_file = False + else: + mode = "r+" + snap_file.write( + {prop_info["swift_name"]: arr}, + elements_per_file, + filenames=output_filename, + mode=mode, + group=f"PartType{ptype}", + attrs={prop_info["swift_name"]: attrs}, + ) + + # Handle ElementMassFractions as a special case + if "ElementMassFractions" in properties[f"PartType{ptype}"]: + if comm_rank == 0: + print(f"Converting PartType{ptype}/ElementMassFractions") + else: + elements = None + elements = comm.bcast(elements) + + # Read in the array for each element + arr = np.zeros((pos.shape[0], len(elements))) + for i_element, element in enumerate(elements): + element_arr = snap_file.read(f"PartType{ptype}/ElementAbundance/{element}") + element_arr = psort.fetch_elements(element_arr, order, comm=comm) + arr[:, i_element] = element_arr + + # Create attributes + exponents = {"L": 0, "M": 0, "T": 0, "t": 0} + a_exponent = 0 + unit = 1 + unit *= unyt.Unit("snap_length", registry=reg) ** exponents["L"] + unit *= unyt.Unit("snap_mass", registry=reg) ** exponents["M"] + unit *= unyt.Unit("snap_time", registry=reg) ** exponents["t"] + unit *= unyt.Unit("snap_temperature", registry=reg) ** exponents["T"] + unit *= unyt.Unit("a", registry=reg) ** a_exponent + unit_attrs = swift_units.attributes_from_units(unit.units, False, a_exponent) + attrs = { + "original_name": "ElementAbundance", + "Description": "Fraction of mass in each element", + } + attrs.update(unit_attrs) + + # Write to the output file + arr = psort.repartition(arr, elements_per_file, comm=comm) + if create_output_file: + mode = "w" + create_output_file = False + else: + mode = "r+" + snap_file.write( + {"ElementMassFractions": arr}, + elements_per_file, + filenames=output_filename, + mode=mode, + group=f"PartType{ptype}", + attrs={"ElementMassFractions": attrs}, + ) + + # Create a subhalo id for each particle by combining the + # group number and subgroup number + sub_group = snap_file.read(f"PartType{ptype}/SubGroupNumber") + subhalo = snap_file.read(f"PartType{ptype}/GroupNumber").astype(np.int64) + subhalo <<= 32 + subhalo += sub_group.astype(np.int64) + # Indicate unbound particles with -1 + bound = sub_group != 1073741824 + subhalo[np.logical_not(bound)] = -1 + # Get SubFind index of bound particles + subhalo[bound] = psort.parallel_match(subhalo[bound], subfind_id, comm=comm) + assert np.all(subhalo[bound] != -1) + + # Sort, add units, and write to file (same as for other properties) + subhalo = psort.fetch_elements(subhalo, order, comm=comm) + units = unyt.Unit("dimensionless", registry=reg) + unit_attrs = swift_units.attributes_from_units(units, False, 0) + attrs = { + "Description": ( + "Unique identifier of the subhalo this particle is " + "bound to. This is a combination of the GroupNumber and" + "the SubGroupNumber. -1 if the particle is not bound" + ) + } + attrs.update(unit_attrs) + subhalo = psort.repartition(subhalo, elements_per_file, comm=comm) + if create_membership_file: + mode = "w" + create_membership_file = False + else: + mode = "r+" + snap_file.write( + {"GroupNr_bound": subhalo}, + elements_per_file, + filenames=membership_filename, + mode=mode, + group=f"PartType{ptype}", + attrs={"GroupNr_bound": attrs}, + ) + +# Add headers to the snapshots +if comm_rank == 0: + print("Writing output snapshot headers") + for i_file in range(n_file): + with h5py.File(output_filename.format(file_nr=i_file), "r+") as outfile: + header = outfile.create_group("Header") + for name, value in swift_header.items(): + header.attrs[name] = value + n_part = np.zeros(max(ptypes) + 1) + for ptype in ptypes: + n_part[ptype] = outfile[f"PartType{ptype}/Coordinates"].shape[0] + header.attrs["NumPart_ThisFile"] = n_part + header.attrs["ThisFile"] = [i_file] + header.attrs["NumPart_Total"] = nr_parts + header.attrs["NumPart_Total_HighWord"] = nr_parts_hw + + cosmo = outfile.create_group("Cosmology") + for name, value in cosmology_header.items(): + cosmo.attrs[name] = [value] + + units = outfile.create_group("Units") + for name, value in units_header.items(): + units.attrs[name] = value + units = outfile.create_group("InternalCodeUnits") + for name, value in units_header.items(): + units.attrs[name] = value + + const = outfile.create_group("PhysicalConstants") + const_cgs = const.create_group("CGS") + for name, value in const_cgs_header.items(): + const_cgs.attrs[name] = value + const_internal = const.create_group("InternalUnits") + for name, value in const_internal_header.items(): + const_internal.attrs[name] = value + + params = outfile.create_group("Parameters") + for name, value in parameters_header.items(): + params.attrs[name] = value + + cells = outfile.create_group("Cells") + cells_metadata = cells.create_group("Meta-data") + cells_metadata.attrs["dimension"] = np.array([n_cell, n_cell, n_cell]) + cells_metadata.attrs["nr_cells"] = [n_cell**3] + cells_metadata.attrs["size"] = np.array( + [cell_size_cmpc, cell_size_cmpc, cell_size_cmpc] + ) + cells.create_dataset("Centres", data=cell_centres) + for ptype in ptypes: + cells.create_dataset(f"Counts/PartType{ptype}", data=cell_counts[ptype]) + cells.create_dataset(f"Files/PartType{ptype}", data=cell_files[ptype]) + cells.create_dataset( + f"OffsetsInFile/PartType{ptype}", data=cell_offsets[ptype] + ) + + # Store the order used for ElementMassFractions + if elements is not None: + subgrid_scheme = outfile.create_group("SubgridScheme") + named_columns = subgrid_scheme.create_group("NamedColumns") + encoded_elements = [element.encode() for element in elements] + named_columns.create_dataset( + "ElementMassFractions", + data=encoded_elements, + ) + +if comm_rank == 0: + print("Done!") diff --git a/misc/get_evolution_HBT_tracks.py b/misc/get_evolution_HBT_tracks.py new file mode 100644 index 00000000..6e9172fe --- /dev/null +++ b/misc/get_evolution_HBT_tracks.py @@ -0,0 +1,203 @@ +#!/bin/env python + +from mpi4py import MPI + +comm = MPI.COMM_WORLD +comm_rank = comm.Get_rank() +comm_size = comm.Get_size() + +import h5py +import numpy as np +from glob import glob + +import virgo.mpi.util +import virgo.mpi.parallel_hdf5 as phdf5 +import virgo.mpi.parallel_sort as psort + + +def get_SOAP_property_evolution(SOAP_paths, TrackIDs_to_follow, properties): + """ + Loads the SOAP catalogues in parallel and retrieves for each catalogue + the specified properties for the provided TrackIDs. + + Parameters + ---------- + SOAP_paths: list of str + List of sorted strings where each one provides the path to a single SOAP + catalogue. + TrackIDs_to_follow: np.ndarray + TrackId of the subhaloes we are interested in. + properties: list of str + The properties we want to retrieve for each fo the provided TrackIDs. + + Returns + ------- + redshift_evolution: np.ndarray + The redshifts at which each SOAP catalogue was saved. + property_evolution: dict of np.ndarray + Dictionary where each entry corresponds to the evolution of the requested + properties for the TrackIDs that are in the local MPI rank. + """ + + # We balance the number of Tracks across the tasks we have + TrackIDs_to_follow = psort.parallel_unique( + TrackIDs_to_follow, repartition_output=True + ) + + # Final dictionary with the data we will save. + property_evolution = dict([(property, []) for property in properties]) + + # Redshifts at which we loaded the SOAP catalogues. + redshift_evolution = -np.ones(len(SOAP_paths)) + + for i, path in enumerate(SOAP_paths): + + if comm_rank == 0: + print(f"Reading SOAP catalogue: {path}", end=" --- ") + + with h5py.File(path, "r") as SOAP_catalogue: + redshift_evolution[i] = SOAP_catalogue["Header"].attrs["Redshift"][0] + + mf = phdf5.MultiFile( + [path], + comm=comm, + ) + data = {} + for property in properties: + data[property] = mf.read(property) + del mf + + # Each rank will find the entry with the TrackID they are supposed to use. + order = psort.parallel_match( + TrackIDs_to_follow, data["InputHalos/HBTplus/TrackId"] + ) + + # Get the properties we want + for property in properties: + property_evolution[property].append( + [psort.fetch_elements(data[property], order, comm=comm)] + ) + + if comm_rank == 0: + print("DONE") + + return redshift_evolution, property_evolution + + +def save_evolution(redshift_evolution, property_evolution, output_file): + """ + Saves the property evolution of the subhaloes in a specified HDF5 file path. + + Parameters + ---------- + redshift_evolution: np.ndarray + The redshifts at which each SOAP catalogue was saved. + property_evolution: dict of np.ndarray + Dictionary where each entry corresponds to the evolution of the requested + properties for the TrackIDs that are in the local MPI rank. + output_file: str + Location where to save the HDF5 containing the evolution of TrackIDs. + """ + + if comm_rank == 0: + print(f"Saving output", end=" --- ") + + # We create a single array for each output property + with h5py.File(output_file, "w", driver="mpio", comm=comm) as outfile: + + for property in property_evolution.keys(): + data_to_save = np.concatenate(property_evolution[property]).T + + names = property.split("/") + group_name = "/".join(names[:-1]) + dset_name = names[-1] + + if group_name in outfile: + group = outfile[group_name] + else: + group = outfile.create_group(group_name) + + phdf5.collective_write(group, dset_name, data_to_save, comm) + + # Save the redshift for future use + dset = outfile.create_dataset("Redshift", data=redshift_evolution) + + comm.Barrier() + if comm_rank == 0: + print("DONE") + + +def get_track_evolution(SOAP_basedir, output_file, track_path, property_path): + """ + Loads in parallel the SOAP properties for each catalogue in the provided + base directory, retrieves the specified properties for each TrackID of interest. + Saves the property evolution in a HDF5 file. + + Parameters + ---------- + SOAP_basedir: str + Path to the directory containing all SOAP catalogues. + output_file: str + Location the HDF5 containing the evolution of TrackIDs is saved. + track_path: np.ndarray + Path to a text file containing in each row a different TrackId, which we + use to flag which subhaloes we are interested in. + property_path: np.ndarray + Path to a text file containing in each row a different SOAP property, which we + use to flag which properties we are interested in following. + """ + + # Get all paths by default + SOAP_paths = sorted(glob(f"{SOAP_basedir}/halo_properties_*.hdf5")) + + # Load which subhaloes and properties we are interested in. + tracks = np.loadtxt(track_path, int) + properties = np.loadtxt(property_path, str) + + if comm_rank == 0: + print( + f"Getting the evolution of {len(properties)} properties for {len(tracks)} TrackIDs. There are {len(SOAP_paths)} SOAP files available in the specified directory." + ) + + # Get the evolution + redshift_evolution, property_evolution = get_SOAP_property_evolution( + SOAP_paths, tracks, properties + ) + + comm.Barrier() + + # Save in a separate file for future use + save_evolution(redshift_evolution, property_evolution, output_file) + + +if __name__ == "__main__": + + from virgo.mpi.util import MPIArgumentParser + + parser = MPIArgumentParser( + comm, + description="Obtain the evolution of the specified SOAP properties for the provided TrackIds.", + ) + parser.add_argument( + "SOAP_basedir", type=str, help="Root directory of the the SOAP catalogues." + ) + parser.add_argument( + "output_file", type=str, help="File in which to write the output." + ) + parser.add_argument( + "-t", + "--tracks", + type=str, + dest="track_path", + help="Path to a text file containing in each row a TrackId whose evolution are interested in tracking.", + ) + parser.add_argument( + "-p", + "--properties", + type=str, + dest="property_path", + help="Path to a text file containing in each row a SOAP property that we are interested in tracking.", + ) + args = parser.parse_args() + + get_track_evolution(**vars(args)) diff --git a/misc/hdecompose_hydrogen_fractions.py b/misc/hdecompose_hydrogen_fractions.py new file mode 100644 index 00000000..92b55eac --- /dev/null +++ b/misc/hdecompose_hydrogen_fractions.py @@ -0,0 +1,210 @@ +#!/bin/env python + +""" +approximate_hydrogen_fractions.py + +This script calculates HI and H2 species fractions using the +Hdecompose package (https://github.com/kyleaoman/Hdecompose). +Usage: + + mpirun -- python approximate_hydrogen_fractions.py \ + --snapshot-basename=SNAPSHOT \ + --output-basename=OUTPUT + +where SNAPSHOT is the basename of the input Swift snapshots and +OUTPUT is basename of the output files. +""" + +import argparse +import collections +import os +import glob + +from mpi4py import MPI + +comm = MPI.COMM_WORLD +comm_rank = comm.Get_rank() +comm_size = comm.Get_size() + +import astropy.units +import h5py +import numpy as np +import unyt + +import virgo.mpi.parallel_sort as psort +import virgo.mpi.parallel_hdf5 as phdf5 + +from SOAP.core import swift_units + +# TODO: Install Hdecompose from github, pypi version is out of date +from Hdecompose.RahmatiEtal2013 import neutral_frac as calculate_neutral_frac +from Hdecompose.BlitzRosolowsky2006 import molecular_frac as calculate_molecular_frac + +# Parse arguments +parser = argparse.ArgumentParser( + description=("Script to estimate HI and H2 species fractions.") +) +parser.add_argument( + "--snap-basename", + type=str, + required=True, + help=( + "The basename for the snapshot files (the snapshot " + "name without the .{file_nr}.hdf5 suffix)" + ), +) +parser.add_argument( + "--output-basename", + type=str, + required=True, + help="The basename for the output files", +) +args = parser.parse_args() +snap_filename = args.snap_basename + ".{file_nr}.hdf5" +output_filename = args.output_basename + ".{file_nr}.hdf5" +os.makedirs(os.path.dirname(output_filename), exist_ok=True) + +# List of gas properties we require from the input snapshots +property_list = [ + "Densities", + "StarFormationRates", + "ElementMassFractions", + "Temperatures", +] + +if comm_rank == 0: + print("Reading in run parameters and units") + params = {} + units = {} + with h5py.File(snap_filename.format(file_nr=0), "r") as file: + params["z"] = file["Cosmology"].attrs["Redshift"] + params["fH"] = file["Parameters"].attrs["EAGLE:InitAbundance_Hydrogen"] + params["fHe"] = file["Parameters"].attrs["EAGLE:InitAbundance_Helium"] + params["gamma"] = file["Parameters"].attrs["EAGLE:EOS_Jeans_GammaEffective"] + params["T0"] = ( + file["Parameters"].attrs["EAGLE:EOS_Jeans_TempNorm_K"] * astropy.units.K + ) + + reg = swift_units.unit_registry_from_snapshot(file) + for prop in property_list: + attrs = file[f"PartType0/{prop}"].attrs + units[prop] = swift_units.units_from_attributes(attrs, reg) + elements = [ + e.decode() + for e in file["SubgridScheme/NamedColumns/ElementMassFractions"][:] + ] + i_H = elements.index("Hydrogen") + + n_file = file["Header"].attrs["NumFilesPerSnapshot"][0] +else: + params = None + units = None + i_H = None +params = comm.bcast(params) +units = comm.bcast(units) +i_H = comm.bcast(i_H) + +if comm_rank == 0: + print("Loading data") + +# Load raw arrays from file +snap_file = phdf5.MultiFile( + snap_filename, file_nr_attr=("Header", "NumFilesPerSnapshot") +) +raw_properties = snap_file.read(property_list, "PartType0") + +# Convert to astropy arrays (physical not comoving) +densities = ( + (raw_properties["Densities"] * units["Densities"]).to("g/cm**3").to_astropy() +) +temperatures = ( + (raw_properties["Temperatures"] * units["Temperatures"]).to("K").to_astropy() +) +# SFR units don't matter, we just need to know if they are nonzero +sfr = raw_properties["StarFormationRates"] +Habundance = raw_properties["ElementMassFractions"][:, i_H] + +# Free up some memory +del raw_properties + +if comm_rank == 0: + print("Calculating neutral fraction") +mu = 1 / (params["fH"] + 0.25 * params["fHe"]) +neutral_frac = calculate_neutral_frac( + params["z"], + densities * Habundance / (mu * astropy.constants.m_p), + temperatures, + onlyA1=True, + noCol=False, + onlyCol=False, + SSH_Thresh=False, + local=False, + EAGLE_corrections=True, + TNG_corrections=False, + SFR=sfr, + mu=mu, + gamma=params["gamma"], + fH=params["fH"], + Habundance=Habundance, + T0=params["T0"], + rho=densities, +) + +if comm_rank == 0: + print("Calculating molecular fraction") +molecular_mass_frac = calculate_molecular_frac( + temperatures, + densities, + EAGLE_corrections=True, + SFR=sfr, + mu=mu, + gamma=params["gamma"], + fH=params["fH"], + T0=params["T0"], +) + +if comm_rank == 0: + print("Writing output") +species_names = ["HI", "H2"] +species_fractions = np.zeros((neutral_frac.shape[0], 2)) +species_fractions[:, 0] = (1.0 - molecular_mass_frac) * neutral_frac +species_fractions[:, 1] = molecular_mass_frac / 2 +attrs = { + "SpeciesFractions": { + "Description": "The fraction of species i in terms of its number density relative to hydrogen, i.e. n_i / n_H_tot.", + "Conversion factor to CGS (not including cosmological corrections)": [1.0], + "Conversion factor to physical CGS (including cosmological corrections)": [1.0], + "U_I exponent": [0.0], + "U_L exponent": [0.0], + "U_M exponent": [0.0], + "U_t exponent": [0.0], + "U_T exponent": [0.0], + "a-scale exponent": [0.0], + "h-scale exponent": [0.0], + "Property can be converted to comoving": [1], + "Value stored as physical": [0], + } +} +elements_per_file = snap_file.get_elements_per_file("ParticleIDs", group="PartType0") +snap_file.write( + {"SpeciesFractions": species_fractions}, + elements_per_file, + filenames=output_filename, + mode="w", + group="PartType0", + attrs=attrs, +) +comm.barrier() + +# Write the NamedColumns using rank 0 +if comm_rank == 0: + for i_file in range(n_file): + with h5py.File(output_filename.format(file_nr=i_file), "a") as file: + subgrid_scheme = file.create_group("SubgridScheme") + named_columns = subgrid_scheme.create_group("NamedColumns") + encoded_species = [species.encode() for species in species_names] + named_columns.create_dataset( + "SpeciesFractions", + data=encoded_species, + ) + print("Done!") diff --git a/misc/load_symmetric_matrix.py b/misc/load_symmetric_matrix.py new file mode 100644 index 00000000..a8ab8556 --- /dev/null +++ b/misc/load_symmetric_matrix.py @@ -0,0 +1,127 @@ +import numpy as np +import swiftsimio as sw + + +def build_matrix(flattened_matrix): + """ + Creates a (ndim,ndim) representation of a symmetric matrix, given + a flattened SOAP representation of shape (ndim * (ndim + 1) / 2). + + flat_matrix: swiftsimio.cosmo_array + One or more flattened representations of a (ndim x ndim) matrix, where + the first ndim columns are diagonal elements and the rest are + off-diagonal. For example, the flattened representation of a single + 2D array would be: + + np.array([11, 22, 12]) + + where the values indicate the array location in the (2,2) matrix. + + Returns + ------- + matrix: swiftsimio.cosmo_array + The (ndim, ndim) representations of the provided matricies. For the + above 2D example, it would correspond to: + + np.array([[11,12],[12,22]]) + """ + + # Guess number of dimensions based on the first flattened representation. + for ndim in range(1, 5): + if ndim * (ndim + 1) / 2 == len(flattened_matrix[0]): + break + + # We check if we found a solution above. If not, the input may be incorrect + if ndim * (ndim + 1) / 2 != len(flattened_matrix[0]): + print("Could not find number of dimensions based on input. Exiting.") + return + + number_of_matricies, size_of_matricies = flattened_matrix[:, :ndim].shape + + # We create the output cosmo array with correct dtype and units + matrix = sw.cosmo_array( + np.ones((number_of_matricies, ndim, ndim)), + units=flattened_matrix.units, + cosmo_factor=flattened_matrix.cosmo_factor, + comoving=flattened_matrix.comoving, + ) + + # Handle diagonals + matrix_index = np.arange(number_of_matricies).T[:, None] + row_idx, col_idx = np.tril_indices(ndim) + + # Identify combinations for diagonals + diagonal = row_idx == col_idx + off_diagonal = ~diagonal + + # Fill in values: diag, lower and upper triangles. + matrix[matrix_index, row_idx[diagonal], col_idx[diagonal]] = flattened_matrix[ + :, :ndim + ] + matrix[matrix_index, row_idx[off_diagonal], col_idx[off_diagonal]] = ( + flattened_matrix[:, ndim:] + ) + matrix[matrix_index, col_idx[off_diagonal], row_idx[off_diagonal]] = ( + flattened_matrix[:, ndim:] + ) + + return matrix + + +if __name__ == "__main__": + + # How many random matricies we generate + number_test_matricies = 100 + + # Test implementation in relevant dimensions + for ndim in [2, 3]: + + # Number elements in flattened array for the chosen + # dimensions + entries = int(ndim * (ndim + 1) / 2) + + # Generate random tests for 2D, currently in interval [0,1) + random_flattened_matrix = sw.cosmo_array( + np.random.random((number_test_matricies, entries)), + units=sw.units.unyt.Mpc, + comoving=False, + cosmo_factor=sw.cosmo_factor(sw.a**1, scale_factor=1), + ) + + # Make the (ndim, ndim) representation + reconstructed_matrix = build_matrix(random_flattened_matrix) + + # Test if symmetric + assert (reconstructed_matrix.swapaxes(1, 2) == reconstructed_matrix).all() + + # Test diagonal elements + assert ( + np.diagonal(reconstructed_matrix, axis1=1, axis2=2) + == random_flattened_matrix[:, :ndim] + ).all() + + # Test off-diagonal elements. We vstack to retrieve off-diagonal elements for + # all matricies in one go without using the same method I use + # in the function we are testing. NOTE: if I do not use .value, the hstack fails... + flattened_off_diagonal = [] + for i in range(ndim - 1): + flattened_off_diagonal.append(reconstructed_matrix[:, 1 + i :, i].value) + + flattened_off_diagonal = np.hstack(flattened_off_diagonal) + assert (flattened_off_diagonal == random_flattened_matrix[:, ndim:].value).all() + + # Test cosmo array properties + assert reconstructed_matrix.units == random_flattened_matrix.units + assert reconstructed_matrix.comoving == random_flattened_matrix.comoving + assert reconstructed_matrix.cosmo_factor == random_flattened_matrix.cosmo_factor + + # Print the first example + print(f"Dimension number {ndim} suceeded.") + + print("Original flattened array: ") + print(random_flattened_matrix[0]) + + print("Reconstructed array: ") + print(reconstructed_matrix[0]) + + print() diff --git a/misc/match_group_membership.py b/misc/match_group_membership.py new file mode 100644 index 00000000..7424a9ba --- /dev/null +++ b/misc/match_group_membership.py @@ -0,0 +1,506 @@ +#!/bin/env python + +""" +match_group_membership.py + +This script matches halos between different simulations run from the +same initial conditions. + +Usage: + + mpirun -- python -u misc/match_group_membership \ + --snap-basename1 SNAP_BASENAME1 \ + --snap-basename2 SNAP_BASENAME2 \ + --membership-basename1 MEMBERSHIP_BASENAME1 \ + --membership-basename2 MEMBERSHIP_BASENAME2 \ + --catalogue-filename1 CATALOGUE_FILENAME1 \ + --catalogue-filename2 CATALOGUE_FILENAME2 \ + --output-filename OUTPUT_FILENAME + +Run "python misc/match_group_membership.py -h" for a discription +of the optional arguments. + +""" + +import argparse +import os + +from mpi4py import MPI + +comm = MPI.COMM_WORLD +comm_rank = comm.Get_rank() +comm_size = comm.Get_size() + +import h5py +import numpy as np + +import virgo.mpi.parallel_sort as psort +import virgo.mpi.parallel_hdf5 as phdf5 +from virgo.mpi.gather_array import gather_array + + +def load_particle_data(snap_basename, membership_basename, ptypes, match_fof, comm): + """ + Load the particle IDs and halo membership for the particle types + we will use to match. Removes unbound particles. + """ + + # Load particle IDs + snap_filename = snap_basename + ".{file_nr}.hdf5" + file = phdf5.MultiFile( + snap_filename, file_nr_attr=("Header", "NumFilesPerSnapshot"), comm=comm + ) + particle_ids = [] + for ptype in ptypes: + particle_ids.append(file.read(f"PartType{ptype}/ParticleIDs")) + particle_ids = np.concatenate(particle_ids) + + # Membership files don't have a header, so create a list of filenames + n_file = len(file.filenames) + membership_filenames = [f"{membership_basename}.{i}.hdf5" for i in range(n_file)] + # Load membership information + file = phdf5.MultiFile( + membership_filenames, file_nr_attr=("Header", "NumFilesPerSnapshot"), comm=comm + ) + halo_catalogue_idx, rank_bound = [], [] + for ptype in ptypes: + if match_fof: + halo_catalogue_idx.append(file.read(f"PartType{ptype}/FOFGroupIDs")) + # Use particle IDs to decide which particles to keep as most bound, + # which should be nearly random + rank_bound.append(file.read(f"PartType{ptype}/ParticleIDs")) + else: + halo_catalogue_idx.append(file.read(f"PartType{ptype}/GroupNr_bound")) + rank_bound.append(file.read(f"PartType{ptype}/Rank_bound")) + halo_catalogue_idx = np.concatenate(halo_catalogue_idx) + rank_bound = np.concatenate(rank_bound) + + # Check the two files are partitioned the same way + assert particle_ids.shape == halo_catalogue_idx.shape + + # Remove any particles which are not bound to a subhalo + mask = halo_catalogue_idx != -1 + particle_ids = particle_ids[mask] + halo_catalogue_idx = halo_catalogue_idx[mask] + rank_bound = rank_bound[mask] + + return { + "particle_ids": particle_ids, + "halo_catalogue_idx": halo_catalogue_idx, + "rank_bound": rank_bound, + } + + +def load_catalogue(catalogue_filename, match_fof, comm): + """ + Loads the required fields from a halo catalogue (SOAP/FOF) + """ + # Load particle IDs + file = phdf5.MultiFile( + catalogue_filename, file_nr_attr=("Header", "NumFilesPerSnapshot"), comm=comm + ) + if match_fof: + return { + "halo_catalogue_idx": file.read(f"Groups/GroupIDs"), + } + else: + return { + "halo_catalogue_idx": file.read(f"InputHalos/HaloCatalogueIndex"), + "host_halo_idx": file.read(f"SOAP/HostHaloIndex"), + "is_central": file.read(f"InputHalos/IsCentral") == 1, + } + + +def match_sim( + particle_ids, + particle_halo_ids, + rank_bound, + particle_ids_to_match, + particle_halo_ids_to_match, + catalogue, + catalogue_to_match, +): + """ + Input: + - particle_ids, particle_halo_ids, and rank_bound from simulation 1 + - particle_ids_to_match, and particle_halo_ids_to_match from simulation 2 + - catalogue from simulation 1 + - catalogue_to_match from simulation 2 + + Returns halo_ids, matched_halo_ids, n_match + """ + + if (not args.match_fof) and (not args.match_satellites): + # Only keep particles in simulation 1 which are bound to a central + idx = psort.parallel_match( + particle_halo_ids, catalogue["halo_catalogue_idx"], comm=comm + ) + assert np.all(idx >= 0), "Some subhalos could not be found" + particle_is_cen = psort.fetch_elements(catalogue["is_central"], idx, comm=comm) + particle_ids = particle_ids[particle_is_cen] + particle_halo_ids = particle_halo_ids[particle_is_cen] + rank_bound = rank_bound[particle_is_cen] + + # Replace satellite halo_ids of particles in simulation 2 + # with their host halo_id + idx = psort.parallel_match( + particle_halo_ids_to_match, + catalogue_to_match["halo_catalogue_idx"], + comm=comm, + ) + is_sat = np.logical_not( + psort.fetch_elements(catalogue_to_match["is_central"], idx, comm=comm) + ) + host_halo_idx = psort.fetch_elements( + catalogue_to_match["host_halo_idx"], idx[is_sat], comm=comm + ) + host_halo_catalogue_idx = psort.fetch_elements( + catalogue_to_match["halo_catalogue_idx"], host_halo_idx, comm=comm + ) + particle_halo_ids_to_match[is_sat] = host_halo_catalogue_idx + + # Sort particles + sort_hash_dtype = [ + ("halo_ids", particle_halo_ids.dtype), + ("rank_bound", rank_bound.dtype), + ] + sort_hash = np.zeros(particle_halo_ids.shape[0], dtype=sort_hash_dtype) + sort_hash["halo_ids"] = particle_halo_ids + sort_hash["rank_bound"] = rank_bound + order = psort.parallel_sort(sort_hash, return_index=True, comm=comm) + # We don't require rank_bound after this point, so don't sort it + particle_ids = psort.fetch_elements(particle_ids, order, comm=comm) + particle_halo_ids = psort.fetch_elements(particle_halo_ids, order, comm=comm) + + # Count the number of particles for each subhalo + unique_halo_ids, unique_counts = psort.parallel_unique( + particle_halo_ids, + return_counts=True, + comm=comm, + ) + + # Determine how to partition the particles, so no subhalo spans rank + gathered_counts = gather_array(unique_counts) + gathered_halo_ids = gather_array(unique_halo_ids) + if comm_rank == 0: + argsort = np.argsort(gathered_halo_ids) + gathered_counts = gathered_counts[argsort] + + n_part_target = np.sum(gathered_counts) / comm_size + cumsum = np.cumsum(gathered_counts) + ranks = np.floor(cumsum / n_part_target).astype(np.int64) + ranks = np.clip(ranks, 0, comm_size - 1) + n_part_per_rank = np.bincount( + ranks, weights=gathered_counts, minlength=comm_size + ).astype(np.int64) + assert np.sum(n_part_per_rank) == np.sum(gathered_counts) + else: + n_part_per_rank = None + n_part_per_rank = comm.bcast(n_part_per_rank) + + # Repartition data + particle_ids = psort.repartition(particle_ids, n_part_per_rank, comm=comm) + particle_halo_ids = psort.repartition(particle_halo_ids, n_part_per_rank, comm=comm) + + # Only keep the first {args.nr_particles} particles for each subhalo + # We can't just do a cut on rank_bound since we might be missing some ptypes + # Skip this step for ranks which have no particles + if (particle_ids.shape[0] != 0) and (args.nr_particles != -1): + # Count how many particles we have for each subhalo + unique, counts = np.unique(particle_halo_ids, return_counts=True) + argsort = np.argsort(unique) + counts = counts[argsort] + + # Calculate a running sum + cumsum = np.cumsum(counts) + n_part_before_group_i = np.concatenate([np.array([0]), cumsum[:-1]]) + + # Calculate the position of each particle within its group + group_position = np.arange(particle_ids.shape[0]) + group_position -= np.repeat(n_part_before_group_i, counts) + + # Remove unneeded particles + mask = group_position < args.nr_particles + particle_ids = particle_ids[mask] + particle_halo_ids = particle_halo_ids[mask] + + # Identify which subhalo each particle is bound to within simulation 2 + idx = psort.parallel_match(particle_ids, particle_ids_to_match, comm=comm) + particle_matched_halo_ids = psort.fetch_elements( + particle_halo_ids_to_match, idx, comm=comm + ) + + # Combine (halo_id, matched_halo_id) into a single ID + combined_ids = particle_halo_ids.astype(np.int64) + combined_ids <<= 32 + combined_ids += particle_matched_halo_ids.astype(np.int64) + + # Carry out a count on the combined ID + combined_ids, combined_counts = np.unique(combined_ids, return_counts=True) + + # Extract original halo ids + matched_halo_ids = combined_ids.astype(np.int32) + combined_ids -= matched_halo_ids + combined_ids >>= 32 + halo_ids = combined_ids.astype(np.int32) + + # Sort first based on halo_ids, then by count, then by matched_halo_ids + idx = np.lexsort((matched_halo_ids, -combined_counts, halo_ids)) + matched_halo_ids = matched_halo_ids[idx] + halo_ids = halo_ids[idx] + combined_counts = combined_counts[idx] + + # Use np.unique to find the first instance of each halo_id + halo_ids, idx = np.unique(halo_ids, return_index=True) + matched_halo_ids = matched_halo_ids[idx] + combined_counts = combined_counts[idx] + + # Get the catalogue index of each object in matched_halo_ids + matched_catalogue_idx = psort.parallel_match( + matched_halo_ids, catalogue_to_match["halo_catalogue_idx"], comm=comm + ) + + # Create arrays we will return. These have the same length as + # the arrays in the input catalogue + match_index = -1 * np.ones_like(catalogue["halo_catalogue_idx"]) + match_count = np.zeros_like(catalogue["halo_catalogue_idx"]) + + # Retrieve the values we require, skipping halos which don't have a match + idx = psort.parallel_match(catalogue["halo_catalogue_idx"], halo_ids) + match_index[idx != -1] = psort.fetch_elements( + matched_catalogue_idx, idx[idx != -1], comm=comm + ) + match_count[idx != -1] = psort.fetch_elements( + combined_counts, idx[idx != -1], comm=comm + ) + + return match_index, match_count + + +def consistent_match(match_index_12, match_index_21): + """ + For each halo in catalogue 1, determine if its match in catalogue 2 + points back at it. + + match_index_12 has one entry for each halo in catalogue 1 and + specifies the matching halo in catalogue 2 (or -1 for not match) + + match_index_21 has one entry for each halo in catalogue 2 and + specifies the matching halo in catalogue 1 (or -1 for not match) + + Returns an array with 1 for a match and 0 otherwise. + """ + + # Find the global array indexes of halos stored on this rank + nr_local_halos = len(match_index_12) + local_halo_offset = comm.scan(nr_local_halos) - nr_local_halos + local_halo_index = np.arange( + local_halo_offset, local_halo_offset + nr_local_halos, dtype=np.int32 + ) + + # For each halo, find the halo that its match in the other + # catalogue was matched with + match_back = -np.ones(nr_local_halos, dtype=np.int32) + has_match = match_index_12 >= 0 + match_back[has_match] = psort.fetch_elements( + match_index_21, match_index_12[has_match], comm=comm + ) + + # If we retrieved our own halo index, we have a match + consistent_matches = match_back == local_halo_index + + return consistent_matches.astype(np.int32) + + +def mpi_print(string, comm_rank): + if comm_rank == 0: + print(string) + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser( + description=( + "Script to match halos across runs by comparing particles" + "Default is to match SOAP catalgoues, but can also match" + "FoF catalgoues" + ), + ) + parser.add_argument( + "--snap-basename1", + type=str, + required=True, + help=( + "The basename of the snapshot files (the snapshot " + "name without the .{file_nr}.hdf5 suffix) for simulation 1" + ), + ) + parser.add_argument( + "--snap-basename2", + type=str, + required=True, + help="The basename of the snapshot files for simulation 2", + ) + parser.add_argument( + "--membership-basename1", + type=str, + required=True, + help="The basename of the membership files for simulation 1", + ) + parser.add_argument( + "--membership-basename2", + type=str, + required=True, + help="The basename of the membership files for simulation 2", + ) + parser.add_argument( + "--catalogue-filename1", + type=str, + required=True, + help="The filename of the catalogue for simulation 1", + ) + parser.add_argument( + "--catalogue-filename2", + type=str, + required=True, + help="The filename of the catalogue for simulation 2", + ) + parser.add_argument( + "--output-filename", + type=str, + required=True, + help="The filename of the output file", + ) + parser.add_argument( + "--ptypes", + type=int, + required=False, + nargs="+", + default=[1], + help="Particle types to use for the matching. Defaults to [1]", + ) + parser.add_argument( + "--nr-particles", + type=int, + required=False, + default=50, + help=( + "Number of particles to use when matching. Defaults to 50. " + "Pass -1 to use all particles" + ), + ) + parser.add_argument( + "--match-satellites", + action="store_true", + help="Attempt to match satellites subhalos as well as centrals", + ) + parser.add_argument( + "--match-fof", + action="store_true", + help=( + "Whether to match FoF catalogues instead of SOAP catalogues. " + "If using this option then pass the snapshots as both the " + "--snap-basename and the --membership-basename. Pass the FoF " + "catalogues as --catalogue-filename" + ), + ) + + args = parser.parse_args() + + # Log the arguments + if comm_rank == 0: + for k, v in vars(args).items(): + print(f" {k}: {v}") + + if args.match_fof: + assert not args.match_satellites + + mpi_print("Loading data from simulation 1", comm_rank) + data_1 = load_particle_data( + args.snap_basename1, + args.membership_basename1, + args.ptypes, + args.match_fof, + comm, + ) + catalogue_1 = load_catalogue(args.catalogue_filename1, args.match_fof, comm) + + mpi_print("Loading data from simulation 2", comm_rank) + data_2 = load_particle_data( + args.snap_basename2, + args.membership_basename2, + args.ptypes, + args.match_fof, + comm, + ) + catalogue_2 = load_catalogue(args.catalogue_filename2, args.match_fof, comm) + + mpi_print("Removing particles which are not bound in both snapshots", comm_rank) + idx = psort.parallel_match( + data_1["particle_ids"], data_2["particle_ids"], comm=comm + ) + for dset in data_1: + data_1[dset] = data_1[dset][idx != -1] + + idx = psort.parallel_match( + data_2["particle_ids"], data_1["particle_ids"], comm=comm + ) + for dset in data_2: + data_2[dset] = data_2[dset][idx != -1] + + mpi_print("Matching simulation 1 to simulation 2", comm_rank) + match_index_12, match_count_12 = match_sim( + data_1["particle_ids"], + data_1["halo_catalogue_idx"], + data_1["rank_bound"], + data_2["particle_ids"], + data_2["halo_catalogue_idx"], + catalogue_1, + catalogue_2, + ) + + mpi_print("Matching simulation 2 to simulation 1", comm_rank) + match_index_21, match_count_21 = match_sim( + data_2["particle_ids"], + data_2["halo_catalogue_idx"], + data_2["rank_bound"], + data_1["particle_ids"], + data_1["halo_catalogue_idx"], + catalogue_2, + catalogue_1, + ) + + mpi_print("Checking matches for consistency", comm_rank) + consistent_12 = consistent_match(match_index_12, match_index_21) + consistent_21 = consistent_match(match_index_21, match_index_12) + + mpi_print("Writing output", comm_rank) + if comm_rank == 0: + os.makedirs(os.path.dirname(args.output_filename), exist_ok=True) + with h5py.File(args.output_filename, "w") as file: + header = file.create_group("Header") + for k, v in [ + ("snap-basename1", args.snap_basename1), + ("snap-basename2", args.snap_basename2), + ("membership-basename1", args.membership_basename1), + ("membership-basename2", args.membership_basename2), + ("catalogue-filename1", args.catalogue_filename1), + ("catalogue-filename2", args.catalogue_filename2), + ("output-filename", args.output_filename), + ("ptypes", args.ptypes), + ("nr-particles", args.nr_particles), + ("match-satellites", args.match_satellites), + ("match-fof", args.match_fof), + ]: + header.attrs[k] = v + comm.barrier() + + with h5py.File(args.output_filename, "r+", driver="mpio", comm=comm) as file: + phdf5.collective_write(file, "MatchIndex1to2", match_index_12, comm=comm) + phdf5.collective_write(file, "MatchCount1to2", match_count_12, comm=comm) + phdf5.collective_write(file, "Consistent1to2", consistent_12, comm=comm) + phdf5.collective_write(file, "MatchIndex2to1", match_index_21, comm=comm) + phdf5.collective_write(file, "MatchCount2to1", match_count_21, comm=comm) + phdf5.collective_write(file, "Consistent2to1", consistent_21, comm=comm) + + mpi_print("Done!", comm_rank) diff --git a/misc/plot_time.py b/misc/plot_time.py new file mode 100644 index 00000000..77f92d7b --- /dev/null +++ b/misc/plot_time.py @@ -0,0 +1,283 @@ +import argparse +import os +import sys + +import h5py +import matplotlib.pyplot as plt +import numpy as np + +parser = argparse.ArgumentParser( + description="Generates plots to identify where SOAP spent its time." +) +parser.add_argument( + "soap_catalogue", type=str, help="Filepath to SOAP catalogue to analyse" +) +parser.add_argument( + "-o", + "--output", + type=str, + default="timings", + help="Name of output directory to place plots within", +) +args = parser.parse_args() + +filename = args.soap_catalogue +output_dir = args.output +print(f"Output directory for plots: {output_dir}") +os.makedirs(output_dir, exist_ok=True) + +with h5py.File(filename, "r") as file: + # Extract some subhalo properties + n_bound = file["InputHalos/NumberOfBoundParticles"][:] + is_central = file["InputHalos/IsCentral"][:] == 1 + host_idx = file["SOAP/HostHaloIndex"][:] + host_n_bound = n_bound.copy() + host_n_bound[~is_central] = n_bound[host_idx[~is_central]] + + # Load the number of loops required to process this halo + n_loop = file["InputHalos/n_loop"][:] + n_process = file["InputHalos/n_process"][:] + # Load the total processing time for each subhalo + process_time = file["InputHalos/process_time"][:] + + # Load the processing time for each halo type + # (e.g. time spend on SO/200_crit) + calc_time = 0 * process_time + halo_props = file["Parameters"].attrs["calculations"].tolist() + subhalo_types = file["Header"].attrs["SubhaloTypes"].tolist() + halo_prop_times = {} + for k in file["InputHalos"].keys(): + if "_time" in k: + halo_prop_times[k] = file[f"InputHalos/{k}"][:] + if "_final_time" in k: + calc_time += file[f"InputHalos/{k}"][:] + +# Defining limits used by multiple plots +xlim_lo = np.min(0.9 * n_bound) +xlim_hi = np.max(1.1 * n_bound) +ylim_lo = np.min(0.9 * process_time) +ylim_hi = np.max(1.1 * process_time) + +####################### + +print("Plotting total time to process each subhalo") + + +def plot_time_vs_nbound(cprop, cbar_label, plot_name): + """ + Plot the total time taken to process a subhalo + against its Nbound. Color the points according + to cprop. + """ + fig, ax = plt.subplots() + sc = ax.scatter(n_bound, process_time, s=1, c=cprop) + fig.colorbar(sc, ax=ax, label=cbar_label) + ax.set_xlabel("Subhalo $N_{bound}$") + ax.set_ylabel("Time to process [s]") + ax.set_xscale("log") + ax.set_yscale("log") + ax.set_xlim(xlim_lo, xlim_hi) + ax.set_ylim(ylim_lo, ylim_hi) + plt.savefig(f"{output_dir}/ProcessTime_{plot_name}.png", dpi=200) + plt.close() + + +plot_time_vs_nbound( + calc_time / process_time, "Fraction of time spent doing calculation", "FracCalc" +) +plot_time_vs_nbound(is_central, "Is central?", "IsCentral") +plot_time_vs_nbound(n_loop, "Number of density loops", "Nloop") +plot_time_vs_nbound(n_process, "Number of times the subhalo was processed", "Nprocess") +plot_time_vs_nbound(np.log10(host_n_bound), "log(Host $N_{bound}$)", "HostNbound") + +####################### + +print("Plotting total processing time taken by each time-mass bin") + +fig, ax = plt.subplots(1) +weights = process_time / np.sum(process_time) +n_bound_bins = 10 ** np.linspace(np.log10(xlim_lo), np.log10(xlim_hi), 15) +process_time_bins = 10 ** np.linspace(np.log10(ylim_lo), np.log10(ylim_hi), 15) +h = ax.hist2d( + n_bound, + process_time, + weights=weights, + bins=[n_bound_bins, process_time_bins], + cmap="Reds", + cmin=1e-5, +) +fig.colorbar(h[3], ax=ax, label="Fraction of total time") +ax.set_xlabel("$N_{bound}$") +ax.set_ylabel("Time to process [s]") +ax.set_xscale("log") +ax.set_yscale("log") +plt.savefig(f"{output_dir}/SubhaloMassTimeFraction.png", dpi=200) +plt.close() + +####################### + +print("Plotting time taken for each halo type, splitting results into mass bins") + +n_bin = 4 + +n_bound_bins = 10 ** np.linspace(np.log10(xlim_lo), np.log10(xlim_hi), n_bin + 1) +fig, axs = plt.subplots(n_bin, sharex=True) +fig.subplots_adjust(hspace=0) +for i in range(n_bin): + mask = (n_bound_bins[i] < n_bound) & (n_bound <= n_bound_bins[i + 1]) + + final_fracs, total_fracs = [], [] + for halo_prop in halo_props: + final_time = halo_prop_times[halo_prop + "_final_time"][mask] + final_frac = np.mean(final_time / process_time[mask]) + final_fracs.append(final_frac) + + total_time = halo_prop_times[halo_prop + "_total_time"][mask] + total_frac = np.mean(total_time / process_time[mask]) + total_fracs.append(total_frac) + + x = np.arange(len(halo_props)) + axs[i].bar(x, total_fracs) + axs[i].bar(x, final_fracs) + label = f"$10^{{{np.log10(n_bound_bins[i]):.2g}}} < N < 10^{{{np.log10(n_bound_bins[i+1]):.2g}}}$" + axs[i].set_ylabel(label, fontsize=8) +axs[n_bin - 1].set_xticks(x) +axs[n_bin - 1].set_xticklabels(labels=halo_props, rotation=45, ha="right") +fig.text( + -0.05, + 0.5, + "Fraction of time spent on halo type", + va="center", + rotation="vertical", + fontsize=14, +) +plt.savefig(f"{output_dir}/HaloTypeTimeFraction.png", dpi=400, bbox_inches="tight") +plt.close() + +####################### + +print( + "Plotting the time taken on each property for each halo type, splitting results into mass bins" +) +# This essentially loads the entire SOAP file, so takes a while to run + +n_bin = 4 + + +# This loop also saves the total time spent on calculations, +# which is needed for the next plot +def get_internal_name(subhalo_type): + """ + Helper function to convert HDF5 group names to internal SOAP names + """ + if subhalo_type == "BoundSubhalo": + return "bound_subhalo" + if "ExclusiveSphere" in subhalo_type: + r = subhalo_type.split("/")[1].replace("kpc", "") + return f"exclusive_sphere_{r}kpc" + if "InclusiveSphere" in subhalo_type: + r = subhalo_type.split("/")[1].replace("kpc", "") + return f"inclusive_sphere_{r}kpc" + if "ProjectedAperture" in subhalo_type: + r = subhalo_type.split("/")[1].replace("kpc", "") + return f"projected_aperture_{r}kpc" + if "SO/" in subhalo_type: + return f"SO_{subhalo_type.split('/')[1]}" + return None + + +total_prop_calculation_time = {} + +n_bound_bins = 10 ** np.linspace(np.log10(xlim_lo), np.log10(xlim_hi), n_bin + 1) +with h5py.File(filename, "r") as file: + for group_name in subhalo_types: + # We don't store times for individual projections + if ("proj" in group_name) or (group_name == "InputHalos"): + continue + + prop_times = {i: [] for i in range(n_bin)} + prop_names = [] + + prop_calculation_time = 0 * halo_prop_times["bound_subhalo_final_time"] + for k in file[group_name]: + if "_time" not in k: + continue + + prop_names.append(k.replace("_time", "")) + arr = file[f"{group_name}/{k}"][:] + # TODO: Remove, was previously outputting times as shape (n, 1) + arr = arr.reshape(-1) + for i_bin in range(n_bin): + mask = (n_bound_bins[i_bin] < n_bound) & ( + n_bound <= n_bound_bins[i_bin + 1] + ) + prop_times[i_bin].append(np.sum(arr[mask])) + prop_calculation_time[mask] += arr[mask] + + if len(prop_names) == 0: + continue + print(f"Plotting {group_name}") + internal_name = get_internal_name(group_name) + total_prop_calculation_time[internal_name] = prop_calculation_time + + # ProjectedAperture times are store in the top level group, but the properties + # themselves are stored in the individual projections. This means we have + # less keys, and so need to take that into account when setting the figsize. + if "ProjectedAperture" in group_name: + figsize = (0.4 * len(file[group_name].keys()), 4.8) + else: + figsize = (0.2 * len(file[group_name].keys()), 4.8) + fig, axs = plt.subplots(n_bin, sharex=True, figsize=figsize) + fig.subplots_adjust(hspace=0) + + x = np.arange(len(prop_names)) + for i_bin in range(n_bin): + axs[i_bin].bar(x, prop_times[i_bin]) + label = f"$10^{{{np.log10(n_bound_bins[i_bin]):.2g}}} < N < 10^{{{np.log10(n_bound_bins[i_bin+1]):.2g}}}$" + axs[i_bin].set_ylabel(label, fontsize=8) + axs[n_bin - 1].set_xticks(x) + axs[n_bin - 1].set_xticklabels(labels=prop_names, rotation=45, ha="right") + # fig.text(0, 0.5, 'Time spent on property', va='center', rotation='vertical', fontsize=14) + plot_name = f"PropertyTime_{group_name.replace('/', '_')}" + plt.savefig(f"{output_dir}/{plot_name}.png", dpi=400, bbox_inches="tight") + plt.close() + +####################### + +print("Plotting the time taken to set up for each halo type") + +n_bin = 4 + +n_bound_bins = 10 ** np.linspace(np.log10(xlim_lo), np.log10(xlim_hi), n_bin + 1) +fig, axs = plt.subplots(n_bin, sharex=True) +fig.subplots_adjust(hspace=0) +for i in range(n_bin): + mask = (n_bound_bins[i] < n_bound) & (n_bound <= n_bound_bins[i + 1]) + + fracs = [] + for halo_prop in halo_props: + final_time = halo_prop_times[halo_prop + "_final_time"][mask] + sum_calc_time = total_prop_calculation_time[halo_prop][mask] + frac = 1 - np.mean(sum_calc_time / final_time) + fracs.append(frac) + + x = np.arange(len(halo_props)) + axs[i].bar(x, fracs) + label = f"$10^{{{np.log10(n_bound_bins[i]):.2g}}} < N < 10^{{{np.log10(n_bound_bins[i+1]):.2g}}}$" + axs[i].set_ylabel(label, fontsize=8) +axs[n_bin - 1].set_xticks(x) +axs[n_bin - 1].set_xticklabels(labels=halo_props, rotation=45, ha="right") +fig.text( + -0.05, + 0.5, + "Fraction of time spent setting up", + va="center", + rotation="vertical", + fontsize=14, +) +plt.savefig(f"{output_dir}/HaloTypeSetupTime.png", dpi=400, bbox_inches="tight") +plt.close() + +####################### + +print("Done!") diff --git a/recalculate_xrays.py b/misc/recalculate_xrays.py similarity index 93% rename from recalculate_xrays.py rename to misc/recalculate_xrays.py index 8155704a..0e3661e4 100644 --- a/recalculate_xrays.py +++ b/misc/recalculate_xrays.py @@ -1,12 +1,13 @@ -import numpy as np +import os + import h5py +from mpi4py import MPI +import numpy as np import virgo.mpi.parallel_hdf5 as phdf5 import virgo.mpi.parallel_sort as psort from virgo.util.partial_formatter import PartialFormatter -from mpi4py import MPI -import lustre -import swift_units +from SOAP.core import swift_units import xray_calculator comm = MPI.COMM_WORLD @@ -31,7 +32,23 @@ def recalculate_xrays(snap_file, output_filename, units, xray_calculator): if comm_rank == 0: print("Recalculating xrays") - idx_he, idx_T, idx_n, t_z, d_z, t_T, d_T, t_nH, d_nH, t_He, d_He, abundance_to_solar, joint_mask, volumes, data_n = xray_calculator.find_indices( + ( + idx_he, + idx_T, + idx_n, + t_z, + d_z, + t_T, + d_T, + t_nH, + d_nH, + t_He, + d_He, + abundance_to_solar, + joint_mask, + volumes, + data_n, + ) = xray_calculator.find_indices( data["Densities"], data["Temperatures"], data["SmoothedElementMassFractions"], @@ -164,6 +181,7 @@ def recalculate_xrays(snap_file, output_filename, units, xray_calculator): if __name__ == "__main__": import datetime + start = datetime.datetime.now() # Read parameters from command line and config file @@ -197,7 +215,11 @@ def recalculate_xrays(snap_file, output_filename, units, xray_calculator): # Ensure output dir exists if comm_rank == 0: - lustre.ensure_output_dir(output_filename) + try: + os.makedirs(os.path.dirname(output_filename), exist_ok=True) + except OSError as e: + print(f"Error creating output directory: {e}") + comm.Abort(1) comm.barrier() # Load tables on rank 0 diff --git a/reorder_swift_fof.py b/misc/reorder_swift_fof.py similarity index 100% rename from reorder_swift_fof.py rename to misc/reorder_swift_fof.py diff --git a/xray_calculator.py b/misc/xray_calculator.py similarity index 98% rename from xray_calculator.py rename to misc/xray_calculator.py index ed81835c..3cf31af5 100644 --- a/xray_calculator.py +++ b/misc/xray_calculator.py @@ -231,11 +231,9 @@ def find_indices( """ redshift = self.z_now scale_factor = 1 / (1 + redshift) - data_n = np.log10( - element_mass_fractions[:, 0] * densities.to(g * cm ** -3) / mp - ) + data_n = np.log10(element_mass_fractions[:, 0] * densities.to(g * cm**-3) / mp) data_T = np.log10(temperatures) - volumes = (masses.astype(np.float64) / densities.astype(np.float64)).to(cm ** 3) + volumes = (masses.astype(np.float64) / densities.astype(np.float64)).to(cm**3) # Create density mask, round to avoid numerical errors density_mask = (data_n >= np.round(self.density_bins.min(), 1)) & ( @@ -393,6 +391,6 @@ def interpolate_X_Ray( ) if "energies" in observing_types[0]: - return luminosities * erg * s ** -1 + return luminosities * erg * s**-1 elif "photon" in observing_types[0]: - return luminosities * s ** -1 + return luminosities * s**-1 diff --git a/parameter_files/COLIBRE_HYBRID.yml b/parameter_files/COLIBRE_HYBRID.yml index 74adecd5..a8de3712 100644 --- a/parameter_files/COLIBRE_HYBRID.yml +++ b/parameter_files/COLIBRE_HYBRID.yml @@ -1,19 +1,21 @@ # Values in this section are substituted into the other sections Parameters: - sim_dir: - output_dir: - scratch_dir: + sim_dir: /cosma8/data/dp004/jlvc76/COLIBRE/ScienceRuns + output_dir: /snap8/scratch/dp004/dc-mcgi1/COLIBRE/soap_aug_04 + scratch_dir: /snap8/scratch/dp004/dc-mcgi1/COLIBRE/soap_aug_04 # Location of the Swift snapshots: Snapshots: # Use {snap_nr:04d} for the snapshot number and {file_nr} for the file number. - filename: "{sim_dir}/{sim_name}/colibre_{snap_nr:04d}/colibre_{snap_nr:04d}.{file_nr}.hdf5" + filename: "{sim_dir}/{sim_name}/snapshots/colibre_{snap_nr:04d}/colibre_{snap_nr:04d}.{file_nr}.hdf5" # Which halo finder we're using, and base name for halo finder output files HaloFinder: type: HBTplus - filename: "{sim_dir}/{sim_name}/HBTplus/{snap_nr:03d}/SubSnap_{snap_nr:03d}" - fof_filename: "{sim_dir}/{sim_name}/fof_output_{snap_nr:04d}/fof_output_{snap_nr:04d}.{file_nr}.hdf5" + filename: "/snap8/scratch/dp004/dc-mcgi1/COLIBRE/sort_hbt/{sim_name}/HBT-HERONS/OrderedSubSnap_{snap_nr:03d}.hdf5" + fof_filename: "{sim_dir}/{sim_name}/fof/fof_output_{snap_nr:04d}/fof_output_{snap_nr:04d}.{file_nr}.hdf5" + fof_radius_filename: "{sim_dir}/{sim_name}/fof_radii/fof_output_{snap_nr:04d}/fof_output_{snap_nr:04d}.{file_nr}.hdf5" + read_potential_energies: true #type: VR #filename: "{sim_dir}/halo_{snap_nr:04d}" #type: Subfind @@ -31,13 +33,11 @@ HaloProperties: ApertureProperties: properties: - AngularMomentumBaryons: true - AngularMomentumDarkMatter: true - AngularMomentumGas: true - AngularMomentumStars: true - AtomicHydrogenMass: - snapshot: true - snipshot: false + AngularMomentumBaryons: general + AngularMomentumDarkMatter: general + AngularMomentumGas: general + AngularMomentumStars: general + AtomicHydrogenMass: true BlackHolesDynamicalMass: true BlackHolesLastEventScalefactor: true BlackHolesSubgridMass: true @@ -46,7 +46,8 @@ ApertureProperties: CentreOfMass: true CentreOfMassVelocity: true DarkMatterMass: true - DarkMatterVelocityDispersionMatrix: true + DarkMatterCentreOfMass: true + DarkMatterVelocityDispersionMatrix: general DiffuseCarbonMass: snapshot: true snipshot: false @@ -62,8 +63,13 @@ ApertureProperties: DiffuseSiliconMass: snapshot: true snipshot: false - DiscToTotalGasMassFraction: true - DiscToTotalStellarMassFraction: true + DiscToTotalGasMassFraction: general + DiscToTotalStellarMassFraction: general + DiscToTotalLuminosityRatioLuminosityWeighted: false + DiscToTotalMassRatioLuminosityWeighted: false + KappaCorotStarsLuminosityWeighted: false + AngularMomentumStarsLuminosityWeighted: false + DustMass: true DustGraphiteMass: snapshot: true snipshot: false @@ -82,6 +88,9 @@ ApertureProperties: DustLargeGrainMassInColdDenseGas: snapshot: true snipshot: false + DustLargeGrainMassSFRWeighted: + snapshot: true + snipshot: false DustLargeGrainMassInMolecularGas: snapshot: true snipshot: false @@ -106,6 +115,9 @@ ApertureProperties: DustSmallGrainMassInMolecularGas: snapshot: true snipshot: false + DustSmallGrainMassSFRWeighted: + snapshot: true + snipshot: false GasMass: true GasMassFractionInIron: snapshot: true @@ -115,25 +127,24 @@ ApertureProperties: snapshot: true snipshot: false GasMassInColdDenseGas: true - GasMassInMetals: true GasTemperature: true - GasTemperatureWithoutRecentAGNHeating: true - GasVelocityDispersionMatrix: true - HalfMassRadiusBaryons: true - HalfMassRadiusDarkMatter: true - HalfMassRadiusGas: true + GasTemperatureWithoutRecentAGNHeating: false + GasVelocityDispersionMatrix: general + HalfMassRadiusBaryons: general + HalfMassRadiusDarkMatter: general + HalfMassRadiusGas: general + HalfMassRadiusDust: general + HalfMassRadiusAtomicHydrogen: true + HalfMassRadiusMolecularHydrogen: true HalfMassRadiusStars: true - HeliumMass: - snapshot: true - snipshot: false - HydrogenMass: - snapshot: true - snipshot: false - KappaCorotBaryons: true - KappaCorotGas: true - KappaCorotStars: true - KineticEnergyGas: true - KineticEnergyStars: true + HalfLightRadiusStars: false + HeliumMass: true + HydrogenMass: true + KappaCorotBaryons: general + KappaCorotGas: general + KappaCorotStars: general + KineticEnergyGas: general + KineticEnergyStars: general LinearMassWeightedDiffuseOxygenOverHydrogenOfGas: snapshot: true snipshot: false @@ -143,18 +154,10 @@ ApertureProperties: LinearMassWeightedOxygenOverHydrogenOfGas: snapshot: true snipshot: false - LinearMassWeightedDiffuseCarbonOverOxygenOfGas: - snapshot: true - snipshot: false - LinearMassWeightedNitrogenOverOxygenOfGas: - snapshot: true - snipshot: false - LinearMassWeightedCarbonOverOxygenOfGas: - snapshot: true - snipshot: false - LinearMassWeightedDiffuseNitrogenOverOxygenOfGas: - snapshot: true - snipshot: false + LinearMassWeightedDiffuseCarbonOverOxygenOfGas: false + LinearMassWeightedNitrogenOverOxygenOfGas: false + LinearMassWeightedCarbonOverOxygenOfGas: false + LinearMassWeightedDiffuseNitrogenOverOxygenOfGas: false LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfAtomicGasHighLimit: snapshot: true snipshot: false @@ -178,23 +181,13 @@ ApertureProperties: LogarithmicMassWeightedMagnesiumOverHydrogenOfStarsHighLimit: true LogarithmicMassWeightedIronOverHydrogenOfStarsLowLimit: true LogarithmicMassWeightedMagnesiumOverHydrogenOfStarsLowLimit: true - LogarithmicMassWeightedDiffuseNitrogenOverOxygenOfGasHighLimit: - snapshot: true - snipshot: false - LogarithmicMassWeightedDiffuseCarbonOverOxygenOfGasHighLimit: - snapshot: true - snipshot: false - LogarithmicMassWeightedDiffuseCarbonOverOxygenOfGasLowLimit: - snapshot: true - snipshot: false - LogarithmicMassWeightedDiffuseNitrogenOverOxygenOfGasLowLimit: - snapshot: true - snipshot: false + LogarithmicMassWeightedDiffuseNitrogenOverOxygenOfGasHighLimit: false + LogarithmicMassWeightedDiffuseCarbonOverOxygenOfGasHighLimit: false + LogarithmicMassWeightedDiffuseCarbonOverOxygenOfGasLowLimit: false + LogarithmicMassWeightedDiffuseNitrogenOverOxygenOfGasLowLimit: false LuminosityWeightedMeanStellarAge: true MassWeightedMeanStellarAge: true - MolecularHydrogenMass: - snapshot: true - snipshot: false + MolecularHydrogenMass: true MostMassiveBlackHoleAccretionRate: true MostMassiveBlackHoleAveragedAccretionRate: true MostMassiveBlackHoleID: true @@ -220,23 +213,6 @@ ApertureProperties: NumberOfDarkMatterParticles: true NumberOfGasParticles: true NumberOfStarParticles: true - SpinParameter: true - TotalInertiaTensor: true - GasInertiaTensor: true - DarkMatterInertiaTensor: true - StellarInertiaTensor: true - TotalInertiaTensorReduced: false - GasInertiaTensorReduced: false - DarkMatterInertiaTensorReduced: false - StellarInertiaTensorReduced: false - TotalInertiaTensorNoniterative: false - GasInertiaTensorNoniterative: false - DarkMatterInertiaTensorNoniterative: false - StellarInertiaTensorNoniterative: false - TotalInertiaTensorReducedNoniterative: false - GasInertiaTensorReducedNoniterative: false - DarkMatterInertiaTensorReducedNoniterative: false - StellarInertiaTensorReducedNoniterative: false StarFormationRate: true AveragedStarFormationRate: snapshot: true @@ -251,10 +227,7 @@ ApertureProperties: StarFormingGasMassFractionInOxygen: snapshot: true snipshot: false - StarFormingGasMassInMetals: - snapshot: true - snipshot: false - StellarCentreOfMass: false + StellarCentreOfMass: true StellarInitialMass: true StellarLuminosity: true StellarMass: true @@ -262,15 +235,30 @@ ApertureProperties: StellarMassFractionInMagnesium: true StellarMassFractionInMetals: true StellarMassFractionInOxygen: true - StellarMassInMetals: true - StellarVelocityDispersionMatrix: true + StellarVelocityDispersionMatrix: general TotalMass: true TotalSNIaRate: true GasMassInColdDenseDiffuseMetals: snapshot: true snipshot: false - StellarCentreOfMassVelocity: false + StellarCentreOfMassVelocity: true + MaximumCircularVelocity: true + MaximumCircularVelocityRadius: true + StellarInertiaTensor: false + StellarInertiaTensorLuminosityWeighted: false + StellarInertiaTensorNoniterative: false + StellarInertiaTensorNoniterativeLuminosityWeighted: false + StellarInertiaTensorReduced: false + StellarInertiaTensorReducedLuminosityWeighted: false + StellarInertiaTensorReducedNoniterative: false + StellarInertiaTensorReducedNoniterativeLuminosityWeighted: false variations: + exclusive_100_pc: + inclusive: false + radius_in_kpc: 0.1 + exclusive_300_pc: + inclusive: false + radius_in_kpc: 0.3 exclusive_1_kpc: inclusive: false radius_in_kpc: 1.0 @@ -289,53 +277,47 @@ ApertureProperties: exclusive_100_kpc: inclusive: false radius_in_kpc: 100.0 - #exclusive_300kpc: - #inclusive: false - #radius_in_kpc: 300.0 - #filter: general - #exclusive_1000_kpc: - #inclusive: false - #radius_in_kpc: 1000.0 - #filter: general - #exclusive_3000kpc: - #inclusive: false - #radius_in_kpc: 3000.0 - #filter: general + exclusive_twice_stellar_half_mass: + inclusive: false + property: BoundSubhalo/HalfMassRadiusStars + radius_multiple: 2.0 + inclusive_100_pc: + inclusive: true + radius_in_kpc: 0.1 + inclusive_300_pc: + inclusive: true + radius_in_kpc: 0.3 inclusive_1_kpc: inclusive: true radius_in_kpc: 1.0 + skip_gt_enclose_radius: true inclusive_3_kpc: inclusive: true radius_in_kpc: 3.0 + skip_gt_enclose_radius: true inclusive_10_kpc: inclusive: true radius_in_kpc: 10.0 + skip_gt_enclose_radius: true inclusive_30_kpc: inclusive: true radius_in_kpc: 30.0 + skip_gt_enclose_radius: true inclusive_50_kpc: inclusive: true radius_in_kpc: 50.0 + skip_gt_enclose_radius: true inclusive_100_kpc: inclusive: true radius_in_kpc: 100.0 - #inclusive_300kpc: - #inclusive: true - #radius_in_kpc: 300.0 - #filter: general - #inclusive_1000_kpc: - #inclusive: true - #radius_in_kpc: 1000.0 - #filter: general - #inclusive_3000kpc: - #inclusive: true - #radius_in_kpc: 3000.0 - #filter: general + skip_gt_enclose_radius: true + inclusive_twice_stellar_half_mass: + inclusive: true + property: BoundSubhalo/HalfMassRadiusStars + radius_multiple: 2.0 ProjectedApertureProperties: properties: - AtomicHydrogenMass: - snapshot: true - snipshot: false + AtomicHydrogenMass: true BlackHolesDynamicalMass: true BlackHolesLastEventScalefactor: true BlackHolesSubgridMass: true @@ -344,23 +326,22 @@ ProjectedApertureProperties: CentreOfMass: true CentreOfMassVelocity: true DarkMatterMass: true - DarkMatterProjectedVelocityDispersion: true + DarkMatterProjectedVelocityDispersion: general + DustMass: true GasMass: true GasMassFractionInMetals: true - GasProjectedVelocityDispersion: true - HalfMassRadiusBaryons: true - HalfMassRadiusDarkMatter: true - HalfMassRadiusGas: true + GasProjectedVelocityDispersion: general + HalfMassRadiusBaryons: general + HalfMassRadiusDarkMatter: general + HalfMassRadiusGas: general + HalfMassRadiusDust: general + HalfMassRadiusAtomicHydrogen: true + HalfMassRadiusMolecularHydrogen: true HalfMassRadiusStars: true - HeliumMass: - snapshot: true - snipshot: false - HydrogenMass: - snapshot: true - snipshot: false - MolecularHydrogenMass: - snapshot: true - snipshot: false + HalfLightRadiusStars: false + HeliumMass: true + HydrogenMass: true + MolecularHydrogenMass: true MostMassiveBlackHoleAccretionRate: true MostMassiveBlackHoleAveragedAccretionRate: true MostMassiveBlackHoleID: true @@ -386,23 +367,28 @@ ProjectedApertureProperties: NumberOfDarkMatterParticles: true NumberOfGasParticles: true NumberOfStarParticles: true - ProjectedTotalInertiaTensor: true + ProjectedTotalInertiaTensor: false ProjectedTotalInertiaTensorReduced: false - ProjectedTotalInertiaTensorNoniterative: false - ProjectedTotalInertiaTensorReducedNoniterative: false - ProjectedGasInertiaTensor: true + ProjectedTotalInertiaTensorNoniterative: general + ProjectedTotalInertiaTensorReducedNoniterative: general + ProjectedGasInertiaTensor: false ProjectedGasInertiaTensorReduced: false - ProjectedGasInertiaTensorNoniterative: false - ProjectedGasInertiaTensorReducedNoniterative: false - ProjectedStellarInertiaTensor: true + ProjectedGasInertiaTensorNoniterative: general + ProjectedGasInertiaTensorReducedNoniterative: general + ProjectedStellarInertiaTensor: false ProjectedStellarInertiaTensorReduced: false - ProjectedStellarInertiaTensorNoniterative: false - ProjectedStellarInertiaTensorReducedNoniterative: false + ProjectedStellarInertiaTensorNoniterative: general + ProjectedStellarInertiaTensorReducedNoniterative: general + ProjectedStellarInertiaTensorLuminosityWeighted: false + ProjectedStellarInertiaTensorReducedLuminosityWeighted: false + ProjectedStellarInertiaTensorNoniterativeLuminosityWeighted: false + ProjectedStellarInertiaTensorReducedNoniterativeLuminosityWeighted: false StarFormationRate: true AveragedStarFormationRate: snapshot: true snipshot: false StarFormingGasMassFractionInMetals: true + StellarCentreOfMass: true StellarInitialMass: true StellarLuminosity: true StellarMass: true @@ -410,9 +396,13 @@ ProjectedApertureProperties: StellarMassFractionInMagnesium: true StellarMassFractionInMetals: true StellarMassFractionInOxygen: true - StellarProjectedVelocityDispersion: true + StellarProjectedVelocityDispersion: general TotalMass: true variations: + 100_pc: + radius_in_kpc: 0.1 + 300_pc: + radius_in_kpc: 0.3 1_kpc: radius_in_kpc: 1.0 3_kpc: @@ -425,12 +415,16 @@ ProjectedApertureProperties: radius_in_kpc: 50.0 100_kpc: radius_in_kpc: 100.0 + twice_stellar_half_mass: + property: BoundSubhalo/HalfMassRadiusStars + radius_multiple: 2.0 SOProperties: properties: - AngularMomentumBaryons: true - AngularMomentumDarkMatter: true - AngularMomentumGas: true - AngularMomentumStars: true + AngularMomentumBaryons: general + AngularMomentumDarkMatter: general + AngularMomentumGas: general + AngularMomentumStars: general + AngularMomentumStarsLuminosityWeighted: false BlackHolesDynamicalMass: true BlackHolesLastEventScalefactor: true BlackHolesSubgridMass: true @@ -441,16 +435,16 @@ SOProperties: ComptonY: snapshot: true snipshot: false - ComptonYWithoutRecentAGNHeating: - snapshot: true - snipshot: false + ComptonYWithoutRecentAGNHeating: false Concentration: true ConcentrationUnsoftened: true - DarkMatterConcentration: true - DarkMatterConcentrationUnsoftened: true + DarkMatterConcentration: general + DarkMatterConcentrationUnsoftened: general DarkMatterMass: true - DiscToTotalGasMassFraction: true - DiscToTotalStellarMassFraction: true + DiscToTotalGasMassFraction: general + DiscToTotalStellarMassFraction: general + DiscToTotalLuminosityRatioLuminosityWeighted: false + DiscToTotalMassRatioLuminosityWeighted: false DopplerB: false GasCentreOfMass: true GasCentreOfMassVelocity: true @@ -462,22 +456,21 @@ SOProperties: GasMassFractionInOxygen: snapshot: true snipshot: false - GasMassInMetals: true GasComptonYTemperature: false - GasComptonYTemperatureCoreExcision: false GasComptonYTemperatureWithoutRecentAGNHeating: false + GasComptonYTemperatureCoreExcision: false GasComptonYTemperatureWithoutRecentAGNHeatingCoreExcision: false GasTemperature: true - GasTemperatureCoreExcision: false - GasTemperatureWithoutCoolGas: true - GasTemperatureWithoutCoolGasAndRecentAGNHeating: true - GasTemperatureWithoutRecentAGNHeating: true - GasTemperatureWithoutCoolGasAndRecentAGNHeatingCoreExcision: false + GasTemperatureWithoutCoolGas: false + GasTemperatureWithoutCoolGasAndRecentAGNHeating: false + GasTemperatureWithoutRecentAGNHeating: false GasTemperatureWithoutCoolGasCoreExcision: false + GasTemperatureCoreExcision: true + GasTemperatureWithoutCoolGasAndRecentAGNHeatingCoreExcision: false GasTemperatureWithoutRecentAGNHeatingCoreExcision: false HotGasMass: true - KineticEnergyGas: true - KineticEnergyStars: true + KineticEnergyGas: general + KineticEnergyStars: general MassFractionSatellites: true MassFractionExternal: true MostMassiveBlackHoleAccretionRate: true @@ -509,24 +502,25 @@ SOProperties: NumberOfStarParticles: true RawNeutrinoMass: false SORadius: true - SpinParameter: true - TotalInertiaTensor: true - GasInertiaTensor: true - DarkMatterInertiaTensor: true - StellarInertiaTensor: true + SpinParameter: general + TotalInertiaTensor: false + GasInertiaTensor: false + DarkMatterInertiaTensor: false + StellarInertiaTensor: false TotalInertiaTensorReduced: false GasInertiaTensorReduced: false DarkMatterInertiaTensorReduced: false StellarInertiaTensorReduced: false - TotalInertiaTensorNoniterative: false - GasInertiaTensorNoniterative: false - DarkMatterInertiaTensorNoniterative: false - StellarInertiaTensorNoniterative: false - TotalInertiaTensorReducedNoniterative: false - GasInertiaTensorReducedNoniterative: false - DarkMatterInertiaTensorReducedNoniterative: false - StellarInertiaTensorReducedNoniterative: false + TotalInertiaTensorNoniterative: general + GasInertiaTensorNoniterative: general + DarkMatterInertiaTensorNoniterative: general + StellarInertiaTensorNoniterative: general + TotalInertiaTensorReducedNoniterative: general + GasInertiaTensorReducedNoniterative: general + DarkMatterInertiaTensorReducedNoniterative: general + StellarInertiaTensorReducedNoniterative: general MaximumCircularVelocity: true + MaximumCircularVelocityRadius: true StarFormationRate: true AveragedStarFormationRate: snapshot: true @@ -540,99 +534,83 @@ SOProperties: StellarMassFractionInIron: true StellarMassFractionInMetals: true StellarMassFractionInOxygen: true - StellarMassInMetals: true ThermalEnergyGas: true TotalMass: true XRayLuminosity: snapshot: true snipshot: false - XRayLuminosityCoreExcision: false - XRayLuminosityWithoutRecentAGNHeating: + XRayLuminosityWithoutRecentAGNHeating: false + XRayLuminosityCoreExcision: + snapshot: true + snipshot: false + XRayLuminosityNoSat: + snapshot: true + snipshot: false + XRayLuminosityCoreExcisionNoSat: snapshot: true snipshot: false XRayLuminosityWithoutRecentAGNHeatingCoreExcision: false XRayLuminosityInRestframe: false - XRayLuminosityInRestframeCoreExcision: false XRayLuminosityInRestframeWithoutRecentAGNHeating: false + XRayLuminosityInRestframeCoreExcision: false XRayLuminosityInRestframeWithoutRecentAGNHeatingCoreExcision: false XRayPhotonLuminosity: snapshot: true snipshot: false - XRayPhotonLuminosityCoreExcision: false - XRayPhotonLuminosityWithoutRecentAGNHeating: + XRayPhotonLuminosityWithoutRecentAGNHeating: false + XRayPhotonLuminosityCoreExcision: snapshot: true snipshot: false XRayPhotonLuminosityWithoutRecentAGNHeatingCoreExcision: false XRayPhotonLuminosityInRestframe: false - XRayPhotonLuminosityInRestframeCoreExcision: false XRayPhotonLuminosityInRestframeWithoutRecentAGNHeating: false + XRayPhotonLuminosityInRestframeCoreExcision: false XRayPhotonLuminosityInRestframeWithoutRecentAGNHeatingCoreExcision: false SpectroscopicLikeTemperature: true - SpectroscopicLikeTemperatureCoreExcision: false - SpectroscopicLikeTemperatureWithoutRecentAGNHeating: true + SpectroscopicLikeTemperatureWithoutRecentAGNHeating: false + SpectroscopicLikeTemperatureCoreExcision: true SpectroscopicLikeTemperatureWithoutRecentAGNHeatingCoreExcision: false - DarkMatterMassFlowRate: true - ColdGasMassFlowRate: true - CoolGasMassFlowRate: true - WarmGasMassFlowRate: true - HotGasMassFlowRate: true - HIMassFlowRate: - snapshot: true - snipshot: false - H2MassFlowRate: - snapshot: true - snipshot: false - MetalMassFlowRate: true - StellarMassFlowRate: true - ColdGasEnergyFlowRate: true - CoolGasEnergyFlowRate: true - WarmGasEnergyFlowRate: true - HotGasEnergyFlowRate: true - ColdGasMomentumFlowRate: true - CoolGasMomentumFlowRate: true - WarmGasMomentumFlowRate: true - HotGasMomentumFlowRate: true + DarkMatterMassFlowRate: general + ColdGasMassFlowRate: general + CoolGasMassFlowRate: general + WarmGasMassFlowRate: general + HotGasMassFlowRate: general + HIMassFlowRate: general + H2MassFlowRate: general + MetalMassFlowRate: general + StellarMassFlowRate: general + ColdGasEnergyFlowRate: general + CoolGasEnergyFlowRate: general + WarmGasEnergyFlowRate: general + HotGasEnergyFlowRate: general + ColdGasMomentumFlowRate: general + CoolGasMomentumFlowRate: general + WarmGasMomentumFlowRate: general + HotGasMomentumFlowRate: general variations: 200_crit: type: crit value: 200.0 - #50_crit: - #type: crit - #value: 50.0 - #filter: general - #100_crit: - #type: crit - #value: 100.0 - #filter: general + core_excision_fraction: 0.15 200_mean: type: mean value: 200.0 + core_excision_fraction: 0.15 500_crit: type: crit value: 500.0 - #5xR500_crit: - #type: crit - #value: 500.0 - #radius_multiple: 5.0 - #filter: general - #1000_crit: - #type: crit - #value: 1000.0 - #filter: general - #2500_crit: - #type: crit - #value: 2500.0 - #filter: general + core_excision_fraction: 0.15 BN98: type: BN98 value: 0.0 filter: general + core_excision_fraction: 0.15 SubhaloProperties: properties: - AngularMomentumBaryons: true - AngularMomentumDarkMatter: true - AngularMomentumGas: true - AngularMomentumStars: true + AngularMomentumBaryons: general + AngularMomentumDarkMatter: general + AngularMomentumGas: general + AngularMomentumStars: general BlackHolesDynamicalMass: true BlackHolesLastEventScalefactor: true BlackHolesSubgridMass: true @@ -641,26 +619,35 @@ SubhaloProperties: CentreOfMass: true CentreOfMassVelocity: true DarkMatterMass: true - DarkMatterVelocityDispersionMatrix: true - DiscToTotalGasMassFraction: true - DiscToTotalStellarMassFraction: true + DarkMatterVelocityDispersionMatrix: general + DiscToTotalGasMassFraction: general + DiscToTotalStellarMassFraction: general + DustMass: true GasMass: true GasMassFractionInMetals: true - GasMassInMetals: true GasTemperature: true GasTemperatureWithoutCoolGas: true GasTemperatureWithoutCoolGasAndRecentAGNHeating: true GasTemperatureWithoutRecentAGNHeating: true - GasVelocityDispersionMatrix: true - HalfMassRadiusBaryons: true - HalfMassRadiusDarkMatter: true - HalfMassRadiusGas: true + GasVelocityDispersionMatrix: general + HalfMassRadiusBaryons: general + HalfMassRadiusDarkMatter: general + HalfMassRadiusGas: general + HalfMassRadiusDust: general HalfMassRadiusStars: true + HalfLightRadiusStars: false HalfMassRadiusTotal: true EncloseRadius: true - KappaCorotBaryons: true - KappaCorotGas: true - KappaCorotStars: true + KappaCorotBaryons: general + KappaCorotGas: general + KappaCorotStars: general + KineticEnergyTotal: true + PotentialEnergyTotal: true + ThermalEnergyGas: true + DiscToTotalLuminosityRatioLuminosityWeighted: false + DiscToTotalMassRatioLuminosityWeighted: false + KappaCorotStarsLuminosityWeighted: false + AngularMomentumStarsLuminosityWeighted: false LastSupernovaEventMaximumGasDensity: snapshot: true snipshot: false @@ -705,47 +692,55 @@ SubhaloProperties: NumberOfDarkMatterParticles: true NumberOfGasParticles: true NumberOfStarParticles: true - SpinParameter: true - TotalInertiaTensor: true - GasInertiaTensor: true - DarkMatterInertiaTensor: true - StellarInertiaTensor: true + SpinParameter: general + TotalInertiaTensor: false + GasInertiaTensor: false + DarkMatterInertiaTensor: false + StellarInertiaTensor: false + StellarInertiaTensorLuminosityWeighted: false TotalInertiaTensorReduced: false GasInertiaTensorReduced: false DarkMatterInertiaTensorReduced: false StellarInertiaTensorReduced: false - TotalInertiaTensorNoniterative: false - GasInertiaTensorNoniterative: false - DarkMatterInertiaTensorNoniterative: false - StellarInertiaTensorNoniterative: false - TotalInertiaTensorReducedNoniterative: false - GasInertiaTensorReducedNoniterative: false - DarkMatterInertiaTensorReducedNoniterative: false - StellarInertiaTensorReducedNoniterative: false + StellarInertiaTensorReducedLuminosityWeighted: false + TotalInertiaTensorNoniterative: general + GasInertiaTensorNoniterative: general + DarkMatterInertiaTensorNoniterative: general + StellarInertiaTensorNoniterative: general + StellarInertiaTensorNoniterativeLuminosityWeighted: false + TotalInertiaTensorReducedNoniterative: general + GasInertiaTensorReducedNoniterative: general + DarkMatterInertiaTensorReducedNoniterative: general + StellarInertiaTensorReducedNoniterative: general + StellarInertiaTensorReducedNoniterativeLuminosityWeighted: false StarFormationRate: true AveragedStarFormationRate: snapshot: true snipshot: false StarFormingGasMass: true StarFormingGasMassFractionInMetals: true + StellarCentreOfMass: true StellarInitialMass: true StellarLuminosity: true StellarMass: true StellarMassFractionInMetals: true - StellarMassInMetals: true - StellarVelocityDispersionMatrix: true + StellarVelocityDispersionMatrix: general TotalMass: true - variations: - Bound: - bound_only: true + StellarRotationalVelocity: general + StellarCylindricalVelocityDispersion: general + StellarCylindricalVelocityDispersionVertical: false + StellarCylindricalVelocityDispersionDiscPlane: false aliases: PartType0/LastSNIIKineticFeedbackDensities: PartType0/DensitiesAtLastSupernovaEvent PartType0/LastSNIIThermalFeedbackDensities: PartType0/DensitiesAtLastSupernovaEvent - PartType0/MetalMassFractionsDiffuse: PartType0/MetalMassFractions + snipshot: + PartType0/SpeciesFractions: PartType0/ReducedSpeciesFractions + PartType0/ElementMassFractions: PartType0/ReducedElementMassFractions + PartType0/LastSNIIKineticFeedbackDensities: PartType0/DensitiesAtLastSupernovaEvent + PartType0/LastSNIIThermalFeedbackDensities: PartType0/DensitiesAtLastSupernovaEvent filters: - # TODO: The current plan is to set these all to 100 for the production runs, but I don't want to break the pipeline general: - limit: 0 + limit: 100 properties: - BoundSubhalo/NumberOfGasParticles - BoundSubhalo/NumberOfDarkMatterParticles @@ -777,8 +772,8 @@ defined_constants: C_O_sun: 0.549 Mg_H_sun: 3.98e-5 calculations: - min_read_radius_cmpc: 0.5 calculate_missing_properties: true + strict_halo_copy: false recently_heated_gas_filter: delta_time_myr: 15 use_AGN_delta_T: false diff --git a/parameter_files/COLIBRE_THERMAL.yml b/parameter_files/COLIBRE_THERMAL.yml index 7c8a171e..8b2ddb4e 100644 --- a/parameter_files/COLIBRE_THERMAL.yml +++ b/parameter_files/COLIBRE_THERMAL.yml @@ -1,19 +1,21 @@ # Values in this section are substituted into the other sections Parameters: - sim_dir: - output_dir: - scratch_dir: + sim_dir: /cosma8/data/dp004/jlvc76/COLIBRE/ScienceRuns + output_dir: /snap8/scratch/dp004/dc-mcgi1/COLIBRE/soap_aug_04 + scratch_dir: /snap8/scratch/dp004/dc-mcgi1/COLIBRE/soap_aug_04 # Location of the Swift snapshots: Snapshots: # Use {snap_nr:04d} for the snapshot number and {file_nr} for the file number. - filename: "{sim_dir}/{sim_name}/colibre_{snap_nr:04d}/colibre_{snap_nr:04d}.{file_nr}.hdf5" + filename: "{sim_dir}/{sim_name}/snapshots/colibre_{snap_nr:04d}/colibre_{snap_nr:04d}.{file_nr}.hdf5" # Which halo finder we're using, and base name for halo finder output files HaloFinder: type: HBTplus - filename: "{sim_dir}/{sim_name}/HBTplus/{snap_nr:03d}/SubSnap_{snap_nr:03d}" - fof_filename: "{sim_dir}/{sim_name}/fof_output_{snap_nr:04d}/fof_output_{snap_nr:04d}.{file_nr}.hdf5" + filename: "/snap8/scratch/dp004/dc-mcgi1/COLIBRE/sort_hbt/{sim_name}/HBT-HERONS/OrderedSubSnap_{snap_nr:03d}.hdf5" + fof_filename: "{sim_dir}/{sim_name}/fof/fof_output_{snap_nr:04d}/fof_output_{snap_nr:04d}.{file_nr}.hdf5" + fof_radius_filename: "{sim_dir}/{sim_name}/fof_radii/fof_output_{snap_nr:04d}/fof_output_{snap_nr:04d}.{file_nr}.hdf5" + read_potential_energies: true #type: VR #filename: "{sim_dir}/halo_{snap_nr:04d}" #type: Subfind @@ -31,13 +33,11 @@ HaloProperties: ApertureProperties: properties: - AngularMomentumBaryons: true - AngularMomentumDarkMatter: true - AngularMomentumGas: true - AngularMomentumStars: true - AtomicHydrogenMass: - snapshot: true - snipshot: false + AngularMomentumBaryons: general + AngularMomentumDarkMatter: general + AngularMomentumGas: general + AngularMomentumStars: general + AtomicHydrogenMass: true BlackHolesDynamicalMass: true BlackHolesLastEventScalefactor: true BlackHolesSubgridMass: true @@ -46,7 +46,8 @@ ApertureProperties: CentreOfMass: true CentreOfMassVelocity: true DarkMatterMass: true - DarkMatterVelocityDispersionMatrix: true + DarkMatterCentreOfMass: true + DarkMatterVelocityDispersionMatrix: general DiffuseCarbonMass: snapshot: true snipshot: false @@ -62,8 +63,13 @@ ApertureProperties: DiffuseSiliconMass: snapshot: true snipshot: false - DiscToTotalGasMassFraction: true - DiscToTotalStellarMassFraction: true + DiscToTotalGasMassFraction: general + DiscToTotalStellarMassFraction: general + DiscToTotalLuminosityRatioLuminosityWeighted: false + DiscToTotalMassRatioLuminosityWeighted: false + KappaCorotStarsLuminosityWeighted: false + AngularMomentumStarsLuminosityWeighted: false + DustMass: true DustGraphiteMass: snapshot: true snipshot: false @@ -85,6 +91,9 @@ ApertureProperties: DustLargeGrainMassInMolecularGas: snapshot: true snipshot: false + DustLargeGrainMassSFRWeighted: + snapshot: true + snipshot: false DustSilicatesMass: snapshot: true snipshot: false @@ -106,6 +115,9 @@ ApertureProperties: DustSmallGrainMassInMolecularGas: snapshot: true snipshot: false + DustSmallGrainMassSFRWeighted: + snapshot: true + snipshot: false GasMass: true GasMassFractionInIron: snapshot: true @@ -115,25 +127,24 @@ ApertureProperties: snapshot: true snipshot: false GasMassInColdDenseGas: true - GasMassInMetals: true GasTemperature: true - GasTemperatureWithoutRecentAGNHeating: true - GasVelocityDispersionMatrix: true - HalfMassRadiusBaryons: true - HalfMassRadiusDarkMatter: true - HalfMassRadiusGas: true + GasTemperatureWithoutRecentAGNHeating: false + GasVelocityDispersionMatrix: general + HalfMassRadiusBaryons: general + HalfMassRadiusDarkMatter: general + HalfMassRadiusGas: general + HalfMassRadiusDust: general + HalfMassRadiusAtomicHydrogen: true + HalfMassRadiusMolecularHydrogen: true HalfMassRadiusStars: true - HeliumMass: - snapshot: true - snipshot: false - HydrogenMass: - snapshot: true - snipshot: false - KappaCorotBaryons: true - KappaCorotGas: true - KappaCorotStars: true - KineticEnergyGas: true - KineticEnergyStars: true + HalfLightRadiusStars: false + HeliumMass: true + HydrogenMass: true + KappaCorotBaryons: general + KappaCorotGas: general + KappaCorotStars: general + KineticEnergyGas: general + KineticEnergyStars: general LinearMassWeightedDiffuseOxygenOverHydrogenOfGas: snapshot: true snipshot: false @@ -143,18 +154,10 @@ ApertureProperties: LinearMassWeightedOxygenOverHydrogenOfGas: snapshot: true snipshot: false - LinearMassWeightedDiffuseCarbonOverOxygenOfGas: - snapshot: true - snipshot: false - LinearMassWeightedNitrogenOverOxygenOfGas: - snapshot: true - snipshot: false - LinearMassWeightedCarbonOverOxygenOfGas: - snapshot: true - snipshot: false - LinearMassWeightedDiffuseNitrogenOverOxygenOfGas: - snapshot: true - snipshot: false + LinearMassWeightedDiffuseCarbonOverOxygenOfGas: false + LinearMassWeightedNitrogenOverOxygenOfGas: false + LinearMassWeightedCarbonOverOxygenOfGas: false + LinearMassWeightedDiffuseNitrogenOverOxygenOfGas: false LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfAtomicGasHighLimit: snapshot: true snipshot: false @@ -178,23 +181,13 @@ ApertureProperties: LogarithmicMassWeightedMagnesiumOverHydrogenOfStarsHighLimit: true LogarithmicMassWeightedIronOverHydrogenOfStarsLowLimit: true LogarithmicMassWeightedMagnesiumOverHydrogenOfStarsLowLimit: true - LogarithmicMassWeightedDiffuseNitrogenOverOxygenOfGasHighLimit: - snapshot: true - snipshot: false - LogarithmicMassWeightedDiffuseCarbonOverOxygenOfGasHighLimit: - snapshot: true - snipshot: false - LogarithmicMassWeightedDiffuseCarbonOverOxygenOfGasLowLimit: - snapshot: true - snipshot: false - LogarithmicMassWeightedDiffuseNitrogenOverOxygenOfGasLowLimit: - snapshot: true - snipshot: false + LogarithmicMassWeightedDiffuseNitrogenOverOxygenOfGasHighLimit: false + LogarithmicMassWeightedDiffuseCarbonOverOxygenOfGasHighLimit: false + LogarithmicMassWeightedDiffuseCarbonOverOxygenOfGasLowLimit: false + LogarithmicMassWeightedDiffuseNitrogenOverOxygenOfGasLowLimit: false LuminosityWeightedMeanStellarAge: true MassWeightedMeanStellarAge: true - MolecularHydrogenMass: - snapshot: true - snipshot: false + MolecularHydrogenMass: true MostMassiveBlackHoleAccretionRate: true MostMassiveBlackHoleAveragedAccretionRate: true MostMassiveBlackHoleID: true @@ -220,23 +213,6 @@ ApertureProperties: NumberOfDarkMatterParticles: true NumberOfGasParticles: true NumberOfStarParticles: true - SpinParameter: true - TotalInertiaTensor: true - GasInertiaTensor: true - DarkMatterInertiaTensor: true - StellarInertiaTensor: true - TotalInertiaTensorReduced: false - GasInertiaTensorReduced: false - DarkMatterInertiaTensorReduced: false - StellarInertiaTensorReduced: false - TotalInertiaTensorNoniterative: false - GasInertiaTensorNoniterative: false - DarkMatterInertiaTensorNoniterative: false - StellarInertiaTensorNoniterative: false - TotalInertiaTensorReducedNoniterative: false - GasInertiaTensorReducedNoniterative: false - DarkMatterInertiaTensorReducedNoniterative: false - StellarInertiaTensorReducedNoniterative: false StarFormationRate: true AveragedStarFormationRate: snapshot: true @@ -251,10 +227,7 @@ ApertureProperties: StarFormingGasMassFractionInOxygen: snapshot: true snipshot: false - StarFormingGasMassInMetals: - snapshot: true - snipshot: false - StellarCentreOfMass: false + StellarCentreOfMass: true StellarInitialMass: true StellarLuminosity: true StellarMass: true @@ -262,15 +235,30 @@ ApertureProperties: StellarMassFractionInMagnesium: true StellarMassFractionInMetals: true StellarMassFractionInOxygen: true - StellarMassInMetals: true - StellarVelocityDispersionMatrix: true + StellarVelocityDispersionMatrix: general TotalMass: true TotalSNIaRate: true GasMassInColdDenseDiffuseMetals: snapshot: true snipshot: false - StellarCentreOfMassVelocity: false + StellarCentreOfMassVelocity: true + MaximumCircularVelocity: true + MaximumCircularVelocityRadius: true + StellarInertiaTensor: false + StellarInertiaTensorLuminosityWeighted: false + StellarInertiaTensorNoniterative: false + StellarInertiaTensorNoniterativeLuminosityWeighted: false + StellarInertiaTensorReduced: false + StellarInertiaTensorReducedLuminosityWeighted: false + StellarInertiaTensorReducedNoniterative: false + StellarInertiaTensorReducedNoniterativeLuminosityWeighted: false variations: + exclusive_100_pc: + inclusive: false + radius_in_kpc: 0.1 + exclusive_300_pc: + inclusive: false + radius_in_kpc: 0.3 exclusive_1_kpc: inclusive: false radius_in_kpc: 1.0 @@ -289,53 +277,47 @@ ApertureProperties: exclusive_100_kpc: inclusive: false radius_in_kpc: 100.0 - #exclusive_300kpc: - #inclusive: false - #radius_in_kpc: 300.0 - #filter: general - #exclusive_1000_kpc: - #inclusive: false - #radius_in_kpc: 1000.0 - #filter: general - #exclusive_3000kpc: - #inclusive: false - #radius_in_kpc: 3000.0 - #filter: general + exclusive_twice_stellar_half_mass: + inclusive: false + property: BoundSubhalo/HalfMassRadiusStars + radius_multiple: 2.0 + inclusive_100_pc: + inclusive: true + radius_in_kpc: 0.1 + inclusive_300_pc: + inclusive: true + radius_in_kpc: 0.3 inclusive_1_kpc: inclusive: true radius_in_kpc: 1.0 + skip_gt_enclose_radius: true inclusive_3_kpc: inclusive: true radius_in_kpc: 3.0 + skip_gt_enclose_radius: true inclusive_10_kpc: inclusive: true radius_in_kpc: 10.0 + skip_gt_enclose_radius: true inclusive_30_kpc: inclusive: true radius_in_kpc: 30.0 + skip_gt_enclose_radius: true inclusive_50_kpc: inclusive: true radius_in_kpc: 50.0 + skip_gt_enclose_radius: true inclusive_100_kpc: inclusive: true radius_in_kpc: 100.0 - #inclusive_300kpc: - #inclusive: true - #radius_in_kpc: 300.0 - #filter: general - #inclusive_1000_kpc: - #inclusive: true - #radius_in_kpc: 1000.0 - #filter: general - #inclusive_3000kpc: - #inclusive: true - #radius_in_kpc: 3000.0 - #filter: general + skip_gt_enclose_radius: true + inclusive_twice_stellar_half_mass: + inclusive: true + property: BoundSubhalo/HalfMassRadiusStars + radius_multiple: 2.0 ProjectedApertureProperties: properties: - AtomicHydrogenMass: - snapshot: true - snipshot: false + AtomicHydrogenMass: true BlackHolesDynamicalMass: true BlackHolesLastEventScalefactor: true BlackHolesSubgridMass: true @@ -344,23 +326,22 @@ ProjectedApertureProperties: CentreOfMass: true CentreOfMassVelocity: true DarkMatterMass: true - DarkMatterProjectedVelocityDispersion: true + DarkMatterProjectedVelocityDispersion: general + DustMass: true GasMass: true GasMassFractionInMetals: true - GasProjectedVelocityDispersion: true - HalfMassRadiusBaryons: true - HalfMassRadiusDarkMatter: true - HalfMassRadiusGas: true + GasProjectedVelocityDispersion: general + HalfMassRadiusBaryons: general + HalfMassRadiusDarkMatter: general + HalfMassRadiusGas: general + HalfMassRadiusDust: general + HalfMassRadiusAtomicHydrogen: true + HalfMassRadiusMolecularHydrogen: true HalfMassRadiusStars: true - HeliumMass: - snapshot: true - snipshot: false - HydrogenMass: - snapshot: true - snipshot: false - MolecularHydrogenMass: - snapshot: true - snipshot: false + HalfLightRadiusStars: false + HeliumMass: true + HydrogenMass: true + MolecularHydrogenMass: true MostMassiveBlackHoleAccretionRate: true MostMassiveBlackHoleAveragedAccretionRate: true MostMassiveBlackHoleID: true @@ -386,23 +367,28 @@ ProjectedApertureProperties: NumberOfDarkMatterParticles: true NumberOfGasParticles: true NumberOfStarParticles: true - ProjectedTotalInertiaTensor: true + ProjectedTotalInertiaTensor: false ProjectedTotalInertiaTensorReduced: false - ProjectedTotalInertiaTensorNoniterative: false - ProjectedTotalInertiaTensorReducedNoniterative: false - ProjectedGasInertiaTensor: true + ProjectedTotalInertiaTensorNoniterative: general + ProjectedTotalInertiaTensorReducedNoniterative: general + ProjectedGasInertiaTensor: false ProjectedGasInertiaTensorReduced: false - ProjectedGasInertiaTensorNoniterative: false - ProjectedGasInertiaTensorReducedNoniterative: false - ProjectedStellarInertiaTensor: true + ProjectedGasInertiaTensorNoniterative: general + ProjectedGasInertiaTensorReducedNoniterative: general + ProjectedStellarInertiaTensor: false ProjectedStellarInertiaTensorReduced: false - ProjectedStellarInertiaTensorNoniterative: false - ProjectedStellarInertiaTensorReducedNoniterative: false + ProjectedStellarInertiaTensorNoniterative: general + ProjectedStellarInertiaTensorReducedNoniterative: general + ProjectedStellarInertiaTensorLuminosityWeighted: false + ProjectedStellarInertiaTensorReducedLuminosityWeighted: false + ProjectedStellarInertiaTensorNoniterativeLuminosityWeighted: false + ProjectedStellarInertiaTensorReducedNoniterativeLuminosityWeighted: false StarFormationRate: true AveragedStarFormationRate: snapshot: true snipshot: false StarFormingGasMassFractionInMetals: true + StellarCentreOfMass: true StellarInitialMass: true StellarLuminosity: true StellarMass: true @@ -410,9 +396,13 @@ ProjectedApertureProperties: StellarMassFractionInMagnesium: true StellarMassFractionInMetals: true StellarMassFractionInOxygen: true - StellarProjectedVelocityDispersion: true + StellarProjectedVelocityDispersion: general TotalMass: true variations: + 100_pc: + radius_in_kpc: 0.1 + 300_pc: + radius_in_kpc: 0.3 1_kpc: radius_in_kpc: 1.0 3_kpc: @@ -425,12 +415,16 @@ ProjectedApertureProperties: radius_in_kpc: 50.0 100_kpc: radius_in_kpc: 100.0 + twice_stellar_half_mass: + property: BoundSubhalo/HalfMassRadiusStars + radius_multiple: 2.0 SOProperties: properties: - AngularMomentumBaryons: true - AngularMomentumDarkMatter: true - AngularMomentumGas: true - AngularMomentumStars: true + AngularMomentumBaryons: general + AngularMomentumDarkMatter: general + AngularMomentumGas: general + AngularMomentumStars: general + AngularMomentumStarsLuminosityWeighted: false BlackHolesDynamicalMass: true BlackHolesLastEventScalefactor: true BlackHolesSubgridMass: true @@ -441,16 +435,16 @@ SOProperties: ComptonY: snapshot: true snipshot: false - ComptonYWithoutRecentAGNHeating: - snapshot: true - snipshot: false + ComptonYWithoutRecentAGNHeating: false Concentration: true ConcentrationUnsoftened: true - DarkMatterConcentration: true - DarkMatterConcentrationUnsoftened: true + DarkMatterConcentration: general + DarkMatterConcentrationUnsoftened: general DarkMatterMass: true - DiscToTotalGasMassFraction: true - DiscToTotalStellarMassFraction: true + DiscToTotalGasMassFraction: general + DiscToTotalStellarMassFraction: general + DiscToTotalLuminosityRatioLuminosityWeighted: false + DiscToTotalMassRatioLuminosityWeighted: false DopplerB: false GasCentreOfMass: true GasCentreOfMassVelocity: true @@ -462,22 +456,21 @@ SOProperties: GasMassFractionInOxygen: snapshot: true snipshot: false - GasMassInMetals: true GasComptonYTemperature: false - GasComptonYTemperatureCoreExcision: false GasComptonYTemperatureWithoutRecentAGNHeating: false + GasComptonYTemperatureCoreExcision: false GasComptonYTemperatureWithoutRecentAGNHeatingCoreExcision: false GasTemperature: true - GasTemperatureCoreExcision: false - GasTemperatureWithoutCoolGas: true - GasTemperatureWithoutCoolGasAndRecentAGNHeating: true - GasTemperatureWithoutRecentAGNHeating: true - GasTemperatureWithoutCoolGasAndRecentAGNHeatingCoreExcision: false + GasTemperatureWithoutCoolGas: false + GasTemperatureWithoutCoolGasAndRecentAGNHeating: false + GasTemperatureWithoutRecentAGNHeating: false GasTemperatureWithoutCoolGasCoreExcision: false + GasTemperatureCoreExcision: true + GasTemperatureWithoutCoolGasAndRecentAGNHeatingCoreExcision: false GasTemperatureWithoutRecentAGNHeatingCoreExcision: false HotGasMass: true - KineticEnergyGas: true - KineticEnergyStars: true + KineticEnergyGas: general + KineticEnergyStars: general MassFractionSatellites: true MassFractionExternal: true MostMassiveBlackHoleAccretionRate: true @@ -509,24 +502,25 @@ SOProperties: NumberOfStarParticles: true RawNeutrinoMass: false SORadius: true - SpinParameter: true - TotalInertiaTensor: true - GasInertiaTensor: true - DarkMatterInertiaTensor: true - StellarInertiaTensor: true + SpinParameter: general + TotalInertiaTensor: false + GasInertiaTensor: false + DarkMatterInertiaTensor: false + StellarInertiaTensor: false TotalInertiaTensorReduced: false GasInertiaTensorReduced: false DarkMatterInertiaTensorReduced: false StellarInertiaTensorReduced: false - TotalInertiaTensorNoniterative: false - GasInertiaTensorNoniterative: false - DarkMatterInertiaTensorNoniterative: false - StellarInertiaTensorNoniterative: false - TotalInertiaTensorReducedNoniterative: false - GasInertiaTensorReducedNoniterative: false - DarkMatterInertiaTensorReducedNoniterative: false - StellarInertiaTensorReducedNoniterative: false + TotalInertiaTensorNoniterative: general + GasInertiaTensorNoniterative: general + DarkMatterInertiaTensorNoniterative: general + StellarInertiaTensorNoniterative: general + TotalInertiaTensorReducedNoniterative: general + GasInertiaTensorReducedNoniterative: general + DarkMatterInertiaTensorReducedNoniterative: general + StellarInertiaTensorReducedNoniterative: general MaximumCircularVelocity: true + MaximumCircularVelocityRadius: true StarFormationRate: true AveragedStarFormationRate: snapshot: true @@ -540,99 +534,83 @@ SOProperties: StellarMassFractionInIron: true StellarMassFractionInMetals: true StellarMassFractionInOxygen: true - StellarMassInMetals: true ThermalEnergyGas: true TotalMass: true XRayLuminosity: snapshot: true snipshot: false - XRayLuminosityCoreExcision: false - XRayLuminosityWithoutRecentAGNHeating: + XRayLuminosityWithoutRecentAGNHeating: false + XRayLuminosityCoreExcision: + snapshot: true + snipshot: false + XRayLuminosityNoSat: + snapshot: true + snipshot: false + XRayLuminosityCoreExcisionNoSat: snapshot: true snipshot: false XRayLuminosityWithoutRecentAGNHeatingCoreExcision: false XRayLuminosityInRestframe: false - XRayLuminosityInRestframeCoreExcision: false XRayLuminosityInRestframeWithoutRecentAGNHeating: false + XRayLuminosityInRestframeCoreExcision: false XRayLuminosityInRestframeWithoutRecentAGNHeatingCoreExcision: false XRayPhotonLuminosity: snapshot: true snipshot: false - XRayPhotonLuminosityCoreExcision: false - XRayPhotonLuminosityWithoutRecentAGNHeating: + XRayPhotonLuminosityWithoutRecentAGNHeating: false + XRayPhotonLuminosityCoreExcision: snapshot: true snipshot: false XRayPhotonLuminosityWithoutRecentAGNHeatingCoreExcision: false XRayPhotonLuminosityInRestframe: false - XRayPhotonLuminosityInRestframeCoreExcision: false XRayPhotonLuminosityInRestframeWithoutRecentAGNHeating: false + XRayPhotonLuminosityInRestframeCoreExcision: false XRayPhotonLuminosityInRestframeWithoutRecentAGNHeatingCoreExcision: false SpectroscopicLikeTemperature: true - SpectroscopicLikeTemperatureCoreExcision: false - SpectroscopicLikeTemperatureWithoutRecentAGNHeating: true + SpectroscopicLikeTemperatureWithoutRecentAGNHeating: false + SpectroscopicLikeTemperatureCoreExcision: true SpectroscopicLikeTemperatureWithoutRecentAGNHeatingCoreExcision: false - DarkMatterMassFlowRate: true - ColdGasMassFlowRate: true - CoolGasMassFlowRate: true - WarmGasMassFlowRate: true - HotGasMassFlowRate: true - HIMassFlowRate: - snapshot: true - snipshot: false - H2MassFlowRate: - snapshot: true - snipshot: false - MetalMassFlowRate: true - StellarMassFlowRate: true - ColdGasEnergyFlowRate: true - CoolGasEnergyFlowRate: true - WarmGasEnergyFlowRate: true - HotGasEnergyFlowRate: true - ColdGasMomentumFlowRate: true - CoolGasMomentumFlowRate: true - WarmGasMomentumFlowRate: true - HotGasMomentumFlowRate: true + DarkMatterMassFlowRate: general + ColdGasMassFlowRate: general + CoolGasMassFlowRate: general + WarmGasMassFlowRate: general + HotGasMassFlowRate: general + HIMassFlowRate: general + H2MassFlowRate: general + MetalMassFlowRate: general + StellarMassFlowRate: general + ColdGasEnergyFlowRate: general + CoolGasEnergyFlowRate: general + WarmGasEnergyFlowRate: general + HotGasEnergyFlowRate: general + ColdGasMomentumFlowRate: general + CoolGasMomentumFlowRate: general + WarmGasMomentumFlowRate: general + HotGasMomentumFlowRate: general variations: 200_crit: type: crit value: 200.0 - #50_crit: - #type: crit - #value: 50.0 - #filter: general - #100_crit: - #type: crit - #value: 100.0 - #filter: general + core_excision_fraction: 0.15 200_mean: type: mean value: 200.0 + core_excision_fraction: 0.15 500_crit: type: crit value: 500.0 - #5xR500_crit: - #type: crit - #value: 500.0 - #radius_multiple: 5.0 - #filter: general - #1000_crit: - #type: crit - #value: 1000.0 - #filter: general - #2500_crit: - #type: crit - #value: 2500.0 - #filter: general + core_excision_fraction: 0.15 BN98: type: BN98 value: 0.0 filter: general + core_excision_fraction: 0.15 SubhaloProperties: properties: - AngularMomentumBaryons: true - AngularMomentumDarkMatter: true - AngularMomentumGas: true - AngularMomentumStars: true + AngularMomentumBaryons: general + AngularMomentumDarkMatter: general + AngularMomentumGas: general + AngularMomentumStars: general BlackHolesDynamicalMass: true BlackHolesLastEventScalefactor: true BlackHolesSubgridMass: true @@ -641,26 +619,35 @@ SubhaloProperties: CentreOfMass: true CentreOfMassVelocity: true DarkMatterMass: true - DarkMatterVelocityDispersionMatrix: true - DiscToTotalGasMassFraction: true - DiscToTotalStellarMassFraction: true + DarkMatterVelocityDispersionMatrix: general + DiscToTotalGasMassFraction: general + DiscToTotalStellarMassFraction: general + DustMass: true GasMass: true GasMassFractionInMetals: true - GasMassInMetals: true GasTemperature: true GasTemperatureWithoutCoolGas: true GasTemperatureWithoutCoolGasAndRecentAGNHeating: true GasTemperatureWithoutRecentAGNHeating: true - GasVelocityDispersionMatrix: true - HalfMassRadiusBaryons: true - HalfMassRadiusDarkMatter: true - HalfMassRadiusGas: true + GasVelocityDispersionMatrix: general + HalfMassRadiusBaryons: general + HalfMassRadiusDarkMatter: general + HalfMassRadiusGas: general + HalfMassRadiusDust: general HalfMassRadiusStars: true + HalfLightRadiusStars: false HalfMassRadiusTotal: true EncloseRadius: true - KappaCorotBaryons: true - KappaCorotGas: true - KappaCorotStars: true + KappaCorotBaryons: general + KappaCorotGas: general + KappaCorotStars: general + KineticEnergyTotal: true + PotentialEnergyTotal: true + ThermalEnergyGas: true + DiscToTotalLuminosityRatioLuminosityWeighted: false + DiscToTotalMassRatioLuminosityWeighted: false + KappaCorotStarsLuminosityWeighted: false + AngularMomentumStarsLuminosityWeighted: false LastSupernovaEventMaximumGasDensity: snapshot: true snipshot: false @@ -705,47 +692,55 @@ SubhaloProperties: NumberOfDarkMatterParticles: true NumberOfGasParticles: true NumberOfStarParticles: true - SpinParameter: true - TotalInertiaTensor: true - GasInertiaTensor: true - DarkMatterInertiaTensor: true - StellarInertiaTensor: true + SpinParameter: general + TotalInertiaTensor: false + GasInertiaTensor: false + DarkMatterInertiaTensor: false + StellarInertiaTensor: false + StellarInertiaTensorLuminosityWeighted: false TotalInertiaTensorReduced: false GasInertiaTensorReduced: false DarkMatterInertiaTensorReduced: false StellarInertiaTensorReduced: false - TotalInertiaTensorNoniterative: false - GasInertiaTensorNoniterative: false - DarkMatterInertiaTensorNoniterative: false - StellarInertiaTensorNoniterative: false - TotalInertiaTensorReducedNoniterative: false - GasInertiaTensorReducedNoniterative: false - DarkMatterInertiaTensorReducedNoniterative: false - StellarInertiaTensorReducedNoniterative: false + StellarInertiaTensorReducedLuminosityWeighted: false + TotalInertiaTensorNoniterative: general + GasInertiaTensorNoniterative: general + DarkMatterInertiaTensorNoniterative: general + StellarInertiaTensorNoniterative: general + StellarInertiaTensorNoniterativeLuminosityWeighted: false + TotalInertiaTensorReducedNoniterative: general + GasInertiaTensorReducedNoniterative: general + DarkMatterInertiaTensorReducedNoniterative: general + StellarInertiaTensorReducedNoniterative: general + StellarInertiaTensorReducedNoniterativeLuminosityWeighted: false StarFormationRate: true AveragedStarFormationRate: snapshot: true snipshot: false StarFormingGasMass: true StarFormingGasMassFractionInMetals: true + StellarCentreOfMass: true StellarInitialMass: true StellarLuminosity: true StellarMass: true StellarMassFractionInMetals: true - StellarMassInMetals: true - StellarVelocityDispersionMatrix: true + StellarVelocityDispersionMatrix: general TotalMass: true - variations: - Bound: - bound_only: true + StellarRotationalVelocity: general + StellarCylindricalVelocityDispersion: general + StellarCylindricalVelocityDispersionVertical: false + StellarCylindricalVelocityDispersionDiscPlane: false aliases: PartType0/LastSNIIKineticFeedbackDensities: PartType0/DensitiesAtLastSupernovaEvent PartType0/LastSNIIThermalFeedbackDensities: PartType0/DensitiesAtLastSupernovaEvent - PartType0/MetalMassFractionsDiffuse: PartType0/MetalMassFractions + snipshot: + PartType0/SpeciesFractions: PartType0/ReducedSpeciesFractions + PartType0/ElementMassFractions: PartType0/ReducedElementMassFractions + PartType0/LastSNIIKineticFeedbackDensities: PartType0/DensitiesAtLastSupernovaEvent + PartType0/LastSNIIThermalFeedbackDensities: PartType0/DensitiesAtLastSupernovaEvent filters: - # TODO: The current plan is to set these all to 100 for the production runs, but I don't want to break the pipeline general: - limit: 0 + limit: 100 properties: - BoundSubhalo/NumberOfGasParticles - BoundSubhalo/NumberOfDarkMatterParticles @@ -777,8 +772,8 @@ defined_constants: C_O_sun: 0.549 Mg_H_sun: 3.98e-5 calculations: - min_read_radius_cmpc: 0.5 calculate_missing_properties: true + strict_halo_copy: false recently_heated_gas_filter: delta_time_myr: 15 use_AGN_delta_T: false diff --git a/parameter_files/EAGLE.yml b/parameter_files/EAGLE.yml new file mode 100644 index 00000000..5a1d05d8 --- /dev/null +++ b/parameter_files/EAGLE.yml @@ -0,0 +1,355 @@ +# Values in this section are substituted into the other sections +# The simulation name (box size and resolution) and snapshot will be appended +# to these to get the full name of the input/output files/directories +Parameters: + sim_dir: /snap7/scratch/dp004/dc-mcgi1/SOAP_EAGLE + output_dir: /snap7/scratch/dp004/dc-mcgi1/SOAP_EAGLE + scratch_dir: /snap7/scratch/dp004/dc-mcgi1/SOAP_EAGLE + +# Location of the Swift snapshots: +Snapshots: + # Use {snap_nr:04d} for the snapshot number and {file_nr} for the file number. + filename: "{sim_dir}/{sim_name}/swift_snapshots/swift_{snap_nr:03d}/snap_{snap_nr:03d}.{file_nr}.hdf5" + +# Which halo finder we're using, and base name for halo finder output files +HaloFinder: + type: SubfindEagle + filename: "{sim_dir}/{sim_name}/subfind/groups_{snap_nr:03d}/subfind_tab_{snap_nr:03d}" + +GroupMembership: + # Where to write the group membership files + filename: "{sim_dir}/{sim_name}/SOAP_uncompressed/membership_{snap_nr:03d}/membership_{snap_nr:03d}.{file_nr}.hdf5" + +ExtraInput: + species_frac_filename: "{sim_dir}/{sim_name}/species_fractions/swift_{snap_nr:03d}/snap_{snap_nr:03d}.{file_nr}.hdf5" + +HaloProperties: + # Where to write the halo properties file + filename: "{output_dir}/{sim_name}/SOAP_uncompressed/halo_properties_{snap_nr:04d}.hdf5" + # Where to write temporary chunk output + chunk_dir: "{scratch_dir}/{sim_name}/SOAP-tmp/" + +ApertureProperties: + properties: + AtomicHydrogenMass: true + AngularMomentumBaryons: general + AngularMomentumDarkMatter: general + AngularMomentumGas: general + AngularMomentumStars: general + BlackHolesDynamicalMass: true + BlackHolesSubgridMass: true + CentreOfMass: true + CentreOfMassVelocity: true + DarkMatterMass: true + DarkMatterVelocityDispersionMatrix: general + DiscToTotalGasMassFraction: general + DiscToTotalStellarMassFraction: general + GasMass: true + GasMassFractionInIron: true + GasMassFractionInMetals: true + GasMassFractionInOxygen: true + GasMassInColdDenseGas: true + GasTemperature: true + GasVelocityDispersionMatrix: general + HalfMassRadiusBaryons: general + HalfMassRadiusDarkMatter: general + HalfMassRadiusGas: general + HalfMassRadiusStars: true + HeliumMass: true + HydrogenMass: true + KappaCorotBaryons: general + KappaCorotGas: general + KappaCorotStars: general + KineticEnergyGas: general + KineticEnergyStars: general + LinearMassWeightedIronFromSNIaOverHydrogenOfStars: true + LinearMassWeightedIronOverHydrogenOfStars: true + LinearMassWeightedMagnesiumOverHydrogenOfStars: true + LinearMassWeightedOxygenOverHydrogenOfGas: true + LinearMassWeightedNitrogenOverOxygenOfGas: true + LinearMassWeightedCarbonOverOxygenOfGas: true + LogarithmicMassWeightedIronFromSNIaOverHydrogenOfStarsLowLimit: true + LogarithmicMassWeightedIronOverHydrogenOfStarsHighLimit: true + LogarithmicMassWeightedMagnesiumOverHydrogenOfStarsHighLimit: true + LogarithmicMassWeightedIronOverHydrogenOfStarsLowLimit: true + LogarithmicMassWeightedMagnesiumOverHydrogenOfStarsLowLimit: true + MassWeightedMeanStellarAge: true + MolecularHydrogenMass: true + MostMassiveBlackHoleAccretionRate: true + MostMassiveBlackHoleID: true + MostMassiveBlackHoleMass: true + MostMassiveBlackHolePosition: true + MostMassiveBlackHoleVelocity: true + NumberOfBlackHoleParticles: true + NumberOfDarkMatterParticles: true + NumberOfGasParticles: true + NumberOfStarParticles: true + StarFormationRate: true + StarFormingGasMass: true + StarFormingGasMassFractionInIron: true + StarFormingGasMassFractionInMetals: true + StarFormingGasMassFractionInOxygen: true + StellarCentreOfMass: true + StellarInitialMass: true + StellarMass: true + StellarMassFractionInIron: true + StellarMassFractionInMagnesium: true + StellarMassFractionInMetals: true + StellarMassFractionInOxygen: true + StellarVelocityDispersionMatrix: general + TotalMass: true + StellarCentreOfMassVelocity: true + variations: + exclusive_1_kpc: + inclusive: false + radius_in_kpc: 1.0 + exclusive_3_kpc: + inclusive: false + radius_in_kpc: 3.0 + exclusive_10_kpc: + inclusive: false + radius_in_kpc: 10.0 + exclusive_30_kpc: + inclusive: false + radius_in_kpc: 30.0 + exclusive_50_kpc: + inclusive: false + radius_in_kpc: 50.0 + exclusive_100_kpc: + inclusive: false + radius_in_kpc: 100.0 +ProjectedApertureProperties: + properties: + AtomicHydrogenMass: true + BlackHolesDynamicalMass: true + BlackHolesSubgridMass: true + CentreOfMass: true + CentreOfMassVelocity: true + DarkMatterMass: true + DarkMatterProjectedVelocityDispersion: general + GasMass: true + GasMassFractionInMetals: true + GasProjectedVelocityDispersion: general + HalfMassRadiusBaryons: general + HalfMassRadiusDarkMatter: general + HalfMassRadiusGas: general + HalfMassRadiusStars: true + HeliumMass: true + HydrogenMass: true + MolecularHydrogenMass: true + MostMassiveBlackHoleAccretionRate: true + MostMassiveBlackHoleID: true + MostMassiveBlackHoleMass: true + MostMassiveBlackHolePosition: true + MostMassiveBlackHoleVelocity: true + NumberOfBlackHoleParticles: true + NumberOfDarkMatterParticles: true + NumberOfGasParticles: true + NumberOfStarParticles: true + ProjectedGasInertiaTensorNoniterative: general + ProjectedStellarInertiaTensorNoniterative: general + StarFormationRate: true + StarFormingGasMassFractionInMetals: true + StellarInitialMass: true + StellarMass: true + StellarMassFractionInIron: true + StellarMassFractionInMagnesium: true + StellarMassFractionInMetals: true + StellarMassFractionInOxygen: true + StellarProjectedVelocityDispersion: general + TotalMass: true + variations: + 1_kpc: + radius_in_kpc: 1.0 + 3_kpc: + radius_in_kpc: 3.0 + 10_kpc: + radius_in_kpc: 10.0 + 30_kpc: + radius_in_kpc: 30.0 + 50_kpc: + radius_in_kpc: 50.0 + 100_kpc: + radius_in_kpc: 100.0 +SOProperties: + properties: + AngularMomentumBaryons: general + AngularMomentumDarkMatter: general + AngularMomentumGas: general + AngularMomentumStars: general + BlackHolesDynamicalMass: true + BlackHolesSubgridMass: true + CentreOfMass: true + CentreOfMassVelocity: true + Concentration: true + ConcentrationUnsoftened: true + DarkMatterConcentration: general + DarkMatterConcentrationUnsoftened: general + DarkMatterMass: true + DiscToTotalGasMassFraction: general + DiscToTotalStellarMassFraction: general + GasCentreOfMass: true + GasCentreOfMassVelocity: true + GasMass: true + GasMassFractionInIron: true + GasMassFractionInMetals: true + GasMassFractionInOxygen: true + GasTemperature: true + GasTemperatureWithoutCoolGas: true + HotGasMass: true + KineticEnergyGas: general + KineticEnergyStars: general + MassFractionSatellites: true + MassFractionExternal: true + MostMassiveBlackHoleAccretionRate: true + MostMassiveBlackHoleID: true + MostMassiveBlackHoleMass: true + MostMassiveBlackHolePosition: true + MostMassiveBlackHoleVelocity: true + NumberOfBlackHoleParticles: true + NumberOfDarkMatterParticles: true + NumberOfGasParticles: true + NumberOfStarParticles: true + SORadius: true + SpinParameter: general + TotalInertiaTensorNoniterative: general + GasInertiaTensorNoniterative: general + DarkMatterInertiaTensorNoniterative: general + StellarInertiaTensorNoniterative: general + MaximumCircularVelocity: true + MaximumCircularVelocityRadius: true + StarFormationRate: true + StarFormingGasMassFractionInMetals: true + StellarCentreOfMass: true + StellarCentreOfMassVelocity: true + StellarInitialMass: true + StellarMass: true + StellarMassFractionInIron: true + StellarMassFractionInMetals: true + StellarMassFractionInOxygen: true + TotalMass: true + SpectroscopicLikeTemperature: true + DarkMatterMassFlowRate: general + ColdGasMassFlowRate: general + CoolGasMassFlowRate: general + WarmGasMassFlowRate: general + HotGasMassFlowRate: general + MetalMassFlowRate: general + StellarMassFlowRate: general + ColdGasEnergyFlowRate: general + CoolGasEnergyFlowRate: general + WarmGasEnergyFlowRate: general + HotGasEnergyFlowRate: general + ColdGasMomentumFlowRate: general + CoolGasMomentumFlowRate: general + WarmGasMomentumFlowRate: general + HotGasMomentumFlowRate: general + variations: + 200_crit: + type: crit + value: 200.0 + 500_crit: + type: crit + value: 500.0 + BN98: + type: BN98 + value: 0.0 + filter: general +SubhaloProperties: + properties: + AngularMomentumBaryons: general + AngularMomentumDarkMatter: general + AngularMomentumGas: general + AngularMomentumStars: general + BlackHolesDynamicalMass: true + BlackHolesSubgridMass: true + CentreOfMass: true + CentreOfMassVelocity: true + DarkMatterMass: true + DarkMatterVelocityDispersionMatrix: general + DiscToTotalGasMassFraction: general + DiscToTotalStellarMassFraction: general + GasMass: true + GasMassFractionInMetals: true + GasTemperature: true + GasTemperatureWithoutCoolGas: true + GasVelocityDispersionMatrix: general + HalfMassRadiusBaryons: general + HalfMassRadiusDarkMatter: general + HalfMassRadiusGas: general + HalfMassRadiusStars: true + HalfMassRadiusTotal: true + EncloseRadius: true + KappaCorotBaryons: general + KappaCorotGas: general + KappaCorotStars: general + MassWeightedMeanStellarAge: true + MaximumCircularVelocity: true + MaximumCircularVelocityRadiusUnsoftened: true + MaximumCircularVelocityUnsoftened: true + MaximumDarkMatterCircularVelocity: true + MaximumDarkMatterCircularVelocityRadius: true + MedianStellarBirthDensity: true + MaximumStellarBirthDensity: true + MinimumStellarBirthDensity: true + MostMassiveBlackHoleAccretionRate: true + MostMassiveBlackHoleID: true + MostMassiveBlackHoleMass: true + MostMassiveBlackHolePosition: true + MostMassiveBlackHoleVelocity: true + NumberOfBlackHoleParticles: true + NumberOfDarkMatterParticles: true + NumberOfGasParticles: true + NumberOfStarParticles: true + SpinParameter: general + TotalInertiaTensorNoniterative: general + GasInertiaTensorNoniterative: general + DarkMatterInertiaTensorNoniterative: general + StellarInertiaTensorNoniterative: general + StarFormationRate: true + StarFormingGasMass: true + StarFormingGasMassFractionInMetals: true + StellarInitialMass: true + StellarMass: true + StellarMassFractionInMetals: true + StellarVelocityDispersionMatrix: general + TotalMass: true +filters: + general: + limit: 100 + properties: + - BoundSubhalo/NumberOfGasParticles + - BoundSubhalo/NumberOfDarkMatterParticles + - BoundSubhalo/NumberOfStarParticles + - BoundSubhalo/NumberOfBlackHoleParticles + combine_properties: sum + baryon: + limit: 100 + properties: + - BoundSubhalo/NumberOfGasParticles + - BoundSubhalo/NumberOfStarParticles + combine_properties: sum + dm: + limit: 100 + properties: + - BoundSubhalo/NumberOfDarkMatterParticles + gas: + limit: 100 + properties: + - BoundSubhalo/NumberOfGasParticles + star: + limit: 100 + properties: + - BoundSubhalo/NumberOfStarParticles +defined_constants: + O_H_sun: 4.9e-4 + Fe_H_sun: 3.16e-5 + N_O_sun: 0.138 + C_O_sun: 0.549 + Mg_H_sun: 3.98e-5 +calculations: + calculate_missing_properties: false + min_read_radius_cmpc: 0.5 + cold_dense_gas_filter: + maximum_temperature_K: 3.16e4 + minimum_hydrogen_number_density_cm3: 0.1 diff --git a/parameter_files/FLAMINGO.yml b/parameter_files/FLAMINGO.yml index 7350d2d7..dd214e46 100644 --- a/parameter_files/FLAMINGO.yml +++ b/parameter_files/FLAMINGO.yml @@ -41,135 +41,59 @@ HaloProperties: ApertureProperties: properties: - AngularMomentumBaryons: true - AngularMomentumDarkMatter: true - AngularMomentumGas: true - AngularMomentumStars: true - AtomicHydrogenMass: false - BlackHolesDynamicalMass: true - BlackHolesLastEventScalefactor: true - BlackHolesSubgridMass: true - BlackHolesTotalInjectedThermalEnergy: false - BlackHolesTotalInjectedJetEnergy: false - CentreOfMass: true - CentreOfMassVelocity: true - DarkMatterMass: true - DarkMatterVelocityDispersionMatrix: false - DiffuseCarbonMass: false - DiffuseIronMass: false - DiffuseMagnesiumMass: false - DiffuseOxygenMass: false - DiffuseSiliconMass: false - DiscToTotalGasMassFraction: true - DiscToTotalStellarMassFraction: true - DustGraphiteMass: false - DustGraphiteMassInAtomicGas: false - DustGraphiteMassInColdDenseGas: false - DustGraphiteMassInMolecularGas: false - DustLargeGrainMass: false - DustLargeGrainMassInColdDenseGas: false - DustLargeGrainMassInMolecularGas: false - DustSilicatesMass: false - DustSilicatesMassInAtomicGas: false - DustSilicatesMassInColdDenseGas: false - DustSilicatesMassInMolecularGas: false - DustSmallGrainMass: false - DustSmallGrainMassInColdDenseGas: false - DustSmallGrainMassInMolecularGas: false - GasMassInColdDenseDiffuseMetals: false - GasMass: true - GasMassFractionInIron: true - GasMassFractionInMetals: true - GasMassFractionInOxygen: true - GasMassInColdDenseGas: false - GasMassInMetals: false - GasTemperature: true - GasTemperatureWithoutRecentAGNHeating: true - GasVelocityDispersionMatrix: false - HalfMassRadiusBaryons: true - HalfMassRadiusDarkMatter: true - HalfMassRadiusGas: true - HalfMassRadiusStars: true - HeliumMass: false - HydrogenMass: false - KappaCorotBaryons: true - KappaCorotGas: true - KappaCorotStars: true - KineticEnergyGas: true - KineticEnergyStars: true - LinearMassWeightedDiffuseOxygenOverHydrogenOfGas: false - LinearMassWeightedIronFromSNIaOverHydrogenOfStars: false - LinearMassWeightedIronOverHydrogenOfStars: false - LinearMassWeightedMagnesiumOverHydrogenOfStars: false - LinearMassWeightedOxygenOverHydrogenOfGas: false - LinearMassWeightedCarbonOverOxygenOfGas: false - LinearMassWeightedDiffuseCarbonOverOxygenOfGas: false - LinearMassWeightedNitrogenOverOxygenOfGas: false - LinearMassWeightedDiffuseNitrogenOverOxygenOfGas: false - LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfAtomicGasHighLimit: false - LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfAtomicGasLowLimit: false - LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfGasHighLimit: false - LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfGasLowLimit: false - LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfMolecularGasHighLimit: false - LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfMolecularGasLowLimit: false - LogarithmicMassWeightedIronFromSNIaOverHydrogenOfStarsLowLimit: false - LogarithmicMassWeightedIronOverHydrogenOfStarsHighLimit: false - LogarithmicMassWeightedMagnesiumOverHydrogenOfStarsHighLimit: false - LogarithmicMassWeightedIronOverHydrogenOfStarsLowLimit: false - LogarithmicMassWeightedMagnesiumOverHydrogenOfStarsLowLimit: false - LogarithmicMassWeightedDiffuseCarbonOverOxygenOfGasLowLimit: false - LogarithmicMassWeightedDiffuseNitrogenOverOxygenOfGasLowLimit: false - LogarithmicMassWeightedDiffuseNitrogenOverOxygenOfGasHighLimit: false - LogarithmicMassWeightedDiffuseCarbonOverOxygenOfGasHighLimit: false - LuminosityWeightedMeanStellarAge: true - MassWeightedMeanStellarAge: true - MolecularHydrogenMass: false - MostMassiveBlackHoleAccretionRate: true - MostMassiveBlackHoleAveragedAccretionRate: false - MostMassiveBlackHoleID: true - MostMassiveBlackHoleLastEventScalefactor: true - MostMassiveBlackHoleMass: true - MostMassiveBlackHolePosition: true - MostMassiveBlackHoleVelocity: true - MostMassiveBlackHoleInjectedThermalEnergy: false - MostMassiveBlackHoleNumberOfAGNEvents: false - MostMassiveBlackHoleAccretionMode: false - MostMassiveBlackHoleGWMassLoss: false - MostMassiveBlackHoleInjectedJetEnergyByMode: false - MostMassiveBlackHoleLastJetEventScalefactor: false - MostMassiveBlackHoleNumberOfAGNJetEvents: false - MostMassiveBlackHoleNumberOfMergers: false - MostMassiveBlackHoleRadiatedEnergyByMode: false - MostMassiveBlackHoleTotalAccretedMassesByMode: false - MostMassiveBlackHoleWindEnergyByMode: false - MostMassiveBlackHoleSpin: false - MostMassiveBlackHoleTotalAccretedMass: false - MostMassiveBlackHoleFormationScalefactor: false - NumberOfBlackHoleParticles: true - NumberOfDarkMatterParticles: true - NumberOfGasParticles: true - NumberOfStarParticles: true - SpinParameter: true - StarFormationRate: true - AveragedStarFormationRate: false - StarFormingGasMass: true - StarFormingGasMassFractionInIron: true - StarFormingGasMassFractionInMetals: true - StarFormingGasMassFractionInOxygen: true - StarFormingGasMassInMetals: false - StellarCentreOfMass: true - StellarCentreOfMassVelocity: true - StellarInitialMass: true - StellarLuminosity: true - StellarMass: true - StellarMassFractionInIron: true - StellarMassFractionInMagnesium: false - StellarMassFractionInMetals: true - StellarMassFractionInOxygen: true - StellarMassInMetals: false - StellarVelocityDispersionMatrix: false - TotalMass: true - TotalSNIaRate: false + AngularMomentumBaryons: baryon + AngularMomentumDarkMatter: dm + AngularMomentumGas: gas + AngularMomentumStars: star + BlackHolesDynamicalMass: basic + BlackHolesLastEventScalefactor: general + BlackHolesSubgridMass: basic + CentreOfMass: basic + CentreOfMassVelocity: basic + DarkMatterMass: basic + DiscToTotalGasMassFraction: gas + DiscToTotalStellarMassFraction: star + GasMass: basic + GasMassFractionInIron: general + GasMassFractionInMetals: basic + GasMassFractionInOxygen: general + GasTemperature: general + GasTemperatureWithoutRecentAGNHeating: general + HalfMassRadiusBaryons: baryon + HalfMassRadiusDarkMatter: dm + HalfMassRadiusGas: gas + HalfMassRadiusStars: basic + KappaCorotBaryons: baryon + KappaCorotGas: gas + KappaCorotStars: star + KineticEnergyGas: gas + KineticEnergyStars: star + LuminosityWeightedMeanStellarAge: star + MassWeightedMeanStellarAge: star + MostMassiveBlackHoleAccretionRate: general + MostMassiveBlackHoleID: basic + MostMassiveBlackHoleLastEventScalefactor: general + MostMassiveBlackHoleMass: basic + MostMassiveBlackHolePosition: general + MostMassiveBlackHoleVelocity: general + NumberOfBlackHoleParticles: basic + NumberOfDarkMatterParticles: basic + NumberOfGasParticles: basic + NumberOfStarParticles: basic + StarFormationRate: basic + StarFormingGasMass: general + StarFormingGasMassFractionInIron: general + StarFormingGasMassFractionInMetals: basic + StarFormingGasMassFractionInOxygen: general + StellarCentreOfMass: star + StellarCentreOfMassVelocity: star + StellarInitialMass: star + StellarLuminosity: star + StellarMass: basic + StellarMassFractionInIron: star + StellarMassFractionInMetals: basic + StellarMassFractionInOxygen: star + TotalMass: basic variations: exclusive_1000_kpc: inclusive: false @@ -227,75 +151,41 @@ ApertureProperties: radius_in_kpc: 50.0 ProjectedApertureProperties: properties: - AtomicHydrogenMass: false - BlackHolesDynamicalMass: true - BlackHolesLastEventScalefactor: true - BlackHolesSubgridMass: true - BlackHolesTotalInjectedThermalEnergy: false - BlackHolesTotalInjectedJetEnergy: false - CentreOfMass: true - CentreOfMassVelocity: true - DarkMatterMass: true - DarkMatterProjectedVelocityDispersion: true - GasMass: true - GasMassFractionInMetals: false - GasProjectedVelocityDispersion: true - HalfMassRadiusBaryons: true - HalfMassRadiusDarkMatter: true - HalfMassRadiusGas: true - HalfMassRadiusStars: true - HeliumMass: false - HydrogenMass: false - MolecularHydrogenMass: false - MostMassiveBlackHoleID: true - MostMassiveBlackHoleAccretionRate: true - MostMassiveBlackHoleAveragedAccretionRate: false - MostMassiveBlackHoleLastEventScalefactor: true - MostMassiveBlackHoleMass: true - MostMassiveBlackHolePosition: true - MostMassiveBlackHoleVelocity: true - MostMassiveBlackHoleInjectedThermalEnergy: false - MostMassiveBlackHoleNumberOfAGNEvents: false - MostMassiveBlackHoleAccretionMode: false - MostMassiveBlackHoleGWMassLoss: false - MostMassiveBlackHoleInjectedJetEnergyByMode: false - MostMassiveBlackHoleLastJetEventScalefactor: false - MostMassiveBlackHoleNumberOfAGNJetEvents: false - MostMassiveBlackHoleNumberOfMergers: false - MostMassiveBlackHoleRadiatedEnergyByMode: false - MostMassiveBlackHoleTotalAccretedMassesByMode: false - MostMassiveBlackHoleWindEnergyByMode: false - MostMassiveBlackHoleTotalAccretedMass: false - MostMassiveBlackHoleFormationScalefactor: false - MostMassiveBlackHoleSpin: false - NumberOfBlackHoleParticles: true - NumberOfDarkMatterParticles: true - NumberOfGasParticles: true - NumberOfStarParticles: true - ProjectedTotalInertiaTensor: false - ProjectedTotalInertiaTensorReduced: false - ProjectedTotalInertiaTensorNoniterative: true - ProjectedTotalInertiaTensorReducedNoniterative: true - ProjectedGasInertiaTensor: false - ProjectedGasInertiaTensorReduced: false - ProjectedGasInertiaTensorNoniterative: true - ProjectedGasInertiaTensorReducedNoniterative: true - ProjectedStellarInertiaTensor: false - ProjectedStellarInertiaTensorReduced: false - ProjectedStellarInertiaTensorNoniterative: true - ProjectedStellarInertiaTensorReducedNoniterative: true - StarFormationRate: true - AveragedStarFormationRate: false - StarFormingGasMassFractionInMetals: false - StellarInitialMass: true - StellarLuminosity: true - StellarMass: true - StellarMassFractionInIron: false - StellarMassFractionInMagnesium: false - StellarMassFractionInMetals: false - StellarMassFractionInOxygen: false - StellarProjectedVelocityDispersion: true - TotalMass: true + BlackHolesDynamicalMass: basic + BlackHolesLastEventScalefactor: general + BlackHolesSubgridMass: basic + CentreOfMass: basic + CentreOfMassVelocity: basic + DarkMatterMass: basic + DarkMatterProjectedVelocityDispersion: dm + GasMass: basic + GasProjectedVelocityDispersion: gas + HalfMassRadiusBaryons: baryon + HalfMassRadiusDarkMatter: dm + HalfMassRadiusGas: gas + HalfMassRadiusStars: basic + MostMassiveBlackHoleID: basic + MostMassiveBlackHoleAccretionRate: general + MostMassiveBlackHoleLastEventScalefactor: general + MostMassiveBlackHoleMass: basic + MostMassiveBlackHolePosition: general + MostMassiveBlackHoleVelocity: general + NumberOfBlackHoleParticles: basic + NumberOfDarkMatterParticles: basic + NumberOfGasParticles: basic + NumberOfStarParticles: basic + ProjectedTotalInertiaTensorNoniterative: general + ProjectedTotalInertiaTensorReducedNoniterative: general + ProjectedGasInertiaTensorNoniterative: gas + ProjectedGasInertiaTensorReducedNoniterative: gas + ProjectedStellarInertiaTensorNoniterative: star + ProjectedStellarInertiaTensorReducedNoniterative: star + StarFormationRate: basic + StellarInitialMass: star + StellarLuminosity: star + StellarMass: basic + StellarProjectedVelocityDispersion: star + TotalMass: basic variations: 100_kpc: radius_in_kpc: 100.0 @@ -311,148 +201,105 @@ ProjectedApertureProperties: filter: general SOProperties: properties: - AngularMomentumBaryons: true - AngularMomentumDarkMatter: true - AngularMomentumGas: true - AngularMomentumStars: true - BlackHolesDynamicalMass: true - BlackHolesLastEventScalefactor: true - BlackHolesSubgridMass: true - BlackHolesTotalInjectedThermalEnergy: false - BlackHolesTotalInjectedJetEnergy: false - CentreOfMass: true - CentreOfMassVelocity: true - ComptonY: true - ComptonYWithoutRecentAGNHeating: true - Concentration: true - ConcentrationUnsoftened: true - DarkMatterMass: true - DarkMatterConcentration: true - DarkMatterConcentrationUnsoftened: true - DiscToTotalGasMassFraction: true - DiscToTotalStellarMassFraction: true - DopplerB: true - GasCentreOfMass: true - GasCentreOfMassVelocity: true - GasComptonYTemperature: true - GasComptonYTemperatureWithoutRecentAGNHeating: true - GasComptonYTemperatureCoreExcision: true - GasComptonYTemperatureWithoutRecentAGNHeatingCoreExcision: true - GasMass: true - GasMassFractionInIron: true - GasMassFractionInMetals: true - GasMassFractionInOxygen: true - GasMassInMetals: false - GasTemperature: true - GasTemperatureCoreExcision: true - GasTemperatureWithoutCoolGas: true - GasTemperatureWithoutCoolGasAndRecentAGNHeating: true - GasTemperatureWithoutCoolGasAndRecentAGNHeatingCoreExcision: true - GasTemperatureWithoutCoolGasCoreExcision: true - GasTemperatureWithoutRecentAGNHeating: true - GasTemperatureWithoutRecentAGNHeatingCoreExcision: true - HotGasMass: true - KineticEnergyGas: true - KineticEnergyStars: true - MassFractionSatellites: true - MassFractionExternal: true - MostMassiveBlackHoleAccretionRate: true - MostMassiveBlackHoleAveragedAccretionRate: false - MostMassiveBlackHoleID: true - MostMassiveBlackHoleLastEventScalefactor: true - MostMassiveBlackHoleMass: true - MostMassiveBlackHolePosition: true - MostMassiveBlackHoleVelocity: true - MostMassiveBlackHoleInjectedThermalEnergy: false - MostMassiveBlackHoleNumberOfAGNEvents: false - MostMassiveBlackHoleAccretionMode: false - MostMassiveBlackHoleGWMassLoss: false - MostMassiveBlackHoleInjectedJetEnergyByMode: false - MostMassiveBlackHoleLastJetEventScalefactor: false - MostMassiveBlackHoleNumberOfAGNJetEvents: false - MostMassiveBlackHoleNumberOfMergers: false - MostMassiveBlackHoleRadiatedEnergyByMode: false - MostMassiveBlackHoleTotalAccretedMassesByMode: false - MostMassiveBlackHoleWindEnergyByMode: false - MostMassiveBlackHoleTotalAccretedMass: false - MostMassiveBlackHoleFormationScalefactor: false - MostMassiveBlackHoleSpin: false - NoiseSuppressedNeutrinoMass: true - NumberOfBlackHoleParticles: true - NumberOfDarkMatterParticles: true - NumberOfGasParticles: true - NumberOfNeutrinoParticles: true - NumberOfStarParticles: true - RawNeutrinoMass: true - SORadius: true - SpectroscopicLikeTemperature: true - SpectroscopicLikeTemperatureCoreExcision: true - SpectroscopicLikeTemperatureWithoutRecentAGNHeating: true - SpectroscopicLikeTemperatureWithoutRecentAGNHeatingCoreExcision: true - SpinParameter: true - StarFormationRate: true - AveragedStarFormationRate: false - StarFormingGasMassFractionInMetals: true - StellarCentreOfMass: true - StellarCentreOfMassVelocity: true - StellarInitialMass: true - StellarLuminosity: true - StellarMass: true - StellarMassFractionInIron: true - StellarMassFractionInMetals: true - StellarMassFractionInOxygen: true - StellarMassInMetals: false - TotalInertiaTensor: false - GasInertiaTensor: false - DarkMatterInertiaTensor: false - StellarInertiaTensor: false - TotalInertiaTensorReduced: false - GasInertiaTensorReduced: false - DarkMatterInertiaTensorReduced: false - StellarInertiaTensorReduced: false - TotalInertiaTensorNoniterative: true - GasInertiaTensorNoniterative: true - DarkMatterInertiaTensorNoniterative: true - StellarInertiaTensorNoniterative: true - TotalInertiaTensorReducedNoniterative: true - GasInertiaTensorReducedNoniterative: true - DarkMatterInertiaTensorReducedNoniterative: true - StellarInertiaTensorReducedNoniterative: true - ThermalEnergyGas: true - TotalMass: true - XRayLuminosity: true - XRayLuminosityCoreExcision: true - XRayLuminosityWithoutRecentAGNHeating: true - XRayLuminosityWithoutRecentAGNHeatingCoreExcision: true - XRayLuminosityInRestframe: true - XRayLuminosityInRestframeCoreExcision: true - XRayLuminosityInRestframeWithoutRecentAGNHeating: true - XRayLuminosityInRestframeWithoutRecentAGNHeatingCoreExcision: true - XRayPhotonLuminosity: true - XRayPhotonLuminosityCoreExcision: true - XRayPhotonLuminosityWithoutRecentAGNHeating: true - XRayPhotonLuminosityWithoutRecentAGNHeatingCoreExcision: true - XRayPhotonLuminosityInRestframe: true - XRayPhotonLuminosityInRestframeCoreExcision: true - XRayPhotonLuminosityInRestframeWithoutRecentAGNHeating: true - XRayPhotonLuminosityInRestframeWithoutRecentAGNHeatingCoreExcision: true - DarkMatterMassFlowRate: false - ColdGasMassFlowRate: false - CoolGasMassFlowRate: false - WarmGasMassFlowRate: false - HotGasMassFlowRate: false - HIMassFlowRate: false - H2MassFlowRate: false - MetalMassFlowRate: false - StellarMassFlowRate: false - ColdGasEnergyFlowRate: false - CoolGasEnergyFlowRate: false - WarmGasEnergyFlowRate: false - HotGasEnergyFlowRate: false - ColdGasMomentumFlowRate: false - CoolGasMomentumFlowRate: false - WarmGasMomentumFlowRate: false - HotGasMomentumFlowRate: false + AngularMomentumBaryons: baryon + AngularMomentumDarkMatter: dm + AngularMomentumGas: gas + AngularMomentumStars: star + BlackHolesDynamicalMass: basic + BlackHolesLastEventScalefactor: general + BlackHolesSubgridMass: basic + CentreOfMass: basic + CentreOfMassVelocity: basic + ComptonY: general + ComptonYWithoutRecentAGNHeating: general + Concentration: basic + ConcentrationUnsoftened: basic + DarkMatterMass: basic + DarkMatterConcentration: basic + DarkMatterConcentrationUnsoftened: basic + DiscToTotalGasMassFraction: gas + DiscToTotalStellarMassFraction: star + DopplerB: general + GasCentreOfMass: gas + GasCentreOfMassVelocity: gas + GasComptonYTemperature: general + GasComptonYTemperatureWithoutRecentAGNHeating: general + GasComptonYTemperatureCoreExcision: general + GasComptonYTemperatureWithoutRecentAGNHeatingCoreExcision: general + GasMass: basic + GasMassFractionInIron: general + GasMassFractionInMetals: basic + GasMassFractionInOxygen: general + GasTemperature: general + GasTemperatureCoreExcision: general + GasTemperatureWithoutCoolGas: general + GasTemperatureWithoutCoolGasAndRecentAGNHeating: general + GasTemperatureWithoutCoolGasAndRecentAGNHeatingCoreExcision: general + GasTemperatureWithoutCoolGasCoreExcision: general + GasTemperatureWithoutRecentAGNHeating: general + GasTemperatureWithoutRecentAGNHeatingCoreExcision: general + HotGasMass: general + KineticEnergyGas: gas + KineticEnergyStars: star + MassFractionSatellites: general + MassFractionExternal: general + MostMassiveBlackHoleAccretionRate: general + MostMassiveBlackHoleID: basic + MostMassiveBlackHoleLastEventScalefactor: general + MostMassiveBlackHoleMass: basic + MostMassiveBlackHolePosition: general + MostMassiveBlackHoleVelocity: general + NoiseSuppressedNeutrinoMass: basic + NumberOfBlackHoleParticles: basic + NumberOfDarkMatterParticles: basic + NumberOfGasParticles: basic + NumberOfNeutrinoParticles: basic + NumberOfStarParticles: basic + RawNeutrinoMass: basic + SORadius: basic + SpectroscopicLikeTemperature: general + SpectroscopicLikeTemperatureCoreExcision: general + SpectroscopicLikeTemperatureWithoutRecentAGNHeating: general + SpectroscopicLikeTemperatureWithoutRecentAGNHeatingCoreExcision: general + SpinParameter: general + StarFormationRate: basic + StarFormingGasMassFractionInMetals: basic + StellarCentreOfMass: star + StellarCentreOfMassVelocity: star + StellarInitialMass: star + StellarLuminosity: star + StellarMass: basic + StellarMassFractionInIron: star + StellarMassFractionInMetals: basic + StellarMassFractionInOxygen: star + TotalInertiaTensorNoniterative: general + GasInertiaTensorNoniterative: gas + DarkMatterInertiaTensorNoniterative: dm + StellarInertiaTensorNoniterative: star + TotalInertiaTensorReducedNoniterative: general + GasInertiaTensorReducedNoniterative: gas + DarkMatterInertiaTensorReducedNoniterative: dm + StellarInertiaTensorReducedNoniterative: star + MaximumCircularVelocity: basic + MaximumCircularVelocityRadius: basic + ThermalEnergyGas: general + TotalMass: basic + XRayLuminosity: general + XRayLuminosityCoreExcision: general + XRayLuminosityWithoutRecentAGNHeating: general + XRayLuminosityWithoutRecentAGNHeatingCoreExcision: general + XRayLuminosityInRestframe: general + XRayLuminosityInRestframeCoreExcision: general + XRayLuminosityInRestframeWithoutRecentAGNHeating: general + XRayLuminosityInRestframeWithoutRecentAGNHeatingCoreExcision: general + XRayPhotonLuminosity: general + XRayPhotonLuminosityCoreExcision: general + XRayPhotonLuminosityWithoutRecentAGNHeating: general + XRayPhotonLuminosityWithoutRecentAGNHeatingCoreExcision: general + XRayPhotonLuminosityInRestframe: general + XRayPhotonLuminosityInRestframeCoreExcision: general + XRayPhotonLuminosityInRestframeWithoutRecentAGNHeating: general + XRayPhotonLuminosityInRestframeWithoutRecentAGNHeatingCoreExcision: general variations: 200_crit: type: crit @@ -491,112 +338,78 @@ SOProperties: filter: general SubhaloProperties: properties: - AngularMomentumBaryons: true - AngularMomentumDarkMatter: true - AngularMomentumGas: true - AngularMomentumStars: true - BlackHolesDynamicalMass: true - BlackHolesLastEventScalefactor: true - BlackHolesSubgridMass: true - BlackHolesTotalInjectedThermalEnergy: false - BlackHolesTotalInjectedJetEnergy: false - CentreOfMass: true - CentreOfMassVelocity: true - DarkMatterMass: true - DarkMatterVelocityDispersionMatrix: true - DiscToTotalGasMassFraction: true - DiscToTotalStellarMassFraction: true - GasMass: true - GasMassFractionInMetals: true - GasMassInMetals: false - GasTemperature: true - GasTemperatureWithoutCoolGas: true - GasTemperatureWithoutCoolGasAndRecentAGNHeating: true - GasTemperatureWithoutRecentAGNHeating: true - GasVelocityDispersionMatrix: true - HalfMassRadiusBaryons: true - HalfMassRadiusDarkMatter: true - HalfMassRadiusGas: true - HalfMassRadiusStars: true - HalfMassRadiusTotal: true - EncloseRadius: true - KappaCorotBaryons: true - KappaCorotGas: true - KappaCorotStars: true - LastSupernovaEventMaximumGasDensity: false - LuminosityWeightedMeanStellarAge: true - MassWeightedMeanStellarAge: true - MaximumCircularVelocity: true - MaximumCircularVelocityRadiusUnsoftened: true - MaximumCircularVelocityUnsoftened: true - MaximumDarkMatterCircularVelocity: true - MaximumDarkMatterCircularVelocityRadius: true - MedianStellarBirthDensity: false - MaximumStellarBirthDensity: false - MinimumStellarBirthDensity: false - MedianStellarBirthTemperature: false - MaximumStellarBirthTemperature: false - MinimumStellarBirthTemperature: false - MedianStellarBirthPressure: false - MaximumStellarBirthPressure: false - MinimumStellarBirthPressure: false - MostMassiveBlackHoleAccretionRate: true - MostMassiveBlackHoleAveragedAccretionRate: false - MostMassiveBlackHoleID: true - MostMassiveBlackHoleLastEventScalefactor: true - MostMassiveBlackHoleMass: true - MostMassiveBlackHolePosition: true - MostMassiveBlackHoleVelocity: true - MostMassiveBlackHoleInjectedThermalEnergy: false - MostMassiveBlackHoleNumberOfAGNEvents: false - MostMassiveBlackHoleAccretionMode: false - MostMassiveBlackHoleGWMassLoss: false - MostMassiveBlackHoleInjectedJetEnergyByMode: false - MostMassiveBlackHoleLastJetEventScalefactor: false - MostMassiveBlackHoleNumberOfAGNJetEvents: false - MostMassiveBlackHoleNumberOfMergers: false - MostMassiveBlackHoleRadiatedEnergyByMode: false - MostMassiveBlackHoleTotalAccretedMassesByMode: false - MostMassiveBlackHoleWindEnergyByMode: false - MostMassiveBlackHoleSpin: false - MostMassiveBlackHoleTotalAccretedMass: false - MostMassiveBlackHoleFormationScalefactor: false - NumberOfBlackHoleParticles: true - NumberOfDarkMatterParticles: true - NumberOfGasParticles: true - NumberOfStarParticles: true - TotalInertiaTensor: true - GasInertiaTensor: true - DarkMatterInertiaTensor: true - StellarInertiaTensor: true - TotalInertiaTensorReduced: true - GasInertiaTensorReduced: true - DarkMatterInertiaTensorReduced: true - StellarInertiaTensorReduced: true - TotalInertiaTensorNoniterative: true - GasInertiaTensorNoniterative: true - DarkMatterInertiaTensorNoniterative: true - StellarInertiaTensorNoniterative: true - TotalInertiaTensorReducedNoniterative: true - GasInertiaTensorReducedNoniterative: true - DarkMatterInertiaTensorReducedNoniterative: true - StellarInertiaTensorReducedNoniterative: true - MaximumCircularVelocity: true - SpinParameter: true - StarFormationRate: true - AveragedStarFormationRate: false - StarFormingGasMass: true - StarFormingGasMassFractionInMetals: true - StellarInitialMass: true - StellarLuminosity: true - StellarMass: true - StellarMassFractionInMetals: true - StellarMassInMetals: false - StellarVelocityDispersionMatrix: true - TotalMass: true - variations: - Bound: - bound_only: true + AngularMomentumBaryons: baryon + AngularMomentumDarkMatter: dm + AngularMomentumGas: gas + AngularMomentumStars: star + BlackHolesDynamicalMass: basic + BlackHolesLastEventScalefactor: general + BlackHolesSubgridMass: basic + CentreOfMass: basic + CentreOfMassVelocity: basic + DarkMatterMass: basic + DarkMatterVelocityDispersionMatrix: dm + DiscToTotalGasMassFraction: gas + DiscToTotalStellarMassFraction: star + GasMass: basic + GasMassFractionInMetals: basic + GasTemperature: general + GasTemperatureWithoutCoolGas: general + GasTemperatureWithoutCoolGasAndRecentAGNHeating: general + GasTemperatureWithoutRecentAGNHeating: general + GasVelocityDispersionMatrix: gas + HalfMassRadiusBaryons: baryon + HalfMassRadiusDarkMatter: dm + HalfMassRadiusGas: gas + HalfMassRadiusStars: basic + HalfMassRadiusTotal: general + EncloseRadius: basic + KappaCorotBaryons: baryon + KappaCorotGas: gas + KappaCorotStars: star + LuminosityWeightedMeanStellarAge: star + MassWeightedMeanStellarAge: star + MaximumCircularVelocity: basic + MaximumCircularVelocityRadiusUnsoftened: basic + MaximumCircularVelocityUnsoftened: basic + MaximumDarkMatterCircularVelocity: dm + MaximumDarkMatterCircularVelocityRadius: dm + MostMassiveBlackHoleAccretionRate: general + MostMassiveBlackHoleID: basic + MostMassiveBlackHoleLastEventScalefactor: general + MostMassiveBlackHoleMass: basic + MostMassiveBlackHolePosition: general + MostMassiveBlackHoleVelocity: general + NumberOfBlackHoleParticles: basic + NumberOfDarkMatterParticles: basic + NumberOfGasParticles: basic + NumberOfStarParticles: basic + TotalInertiaTensor: general + GasInertiaTensor: gas + DarkMatterInertiaTensor: dm + StellarInertiaTensor: star + TotalInertiaTensorReduced: general + GasInertiaTensorReduced: gas + DarkMatterInertiaTensorReduced: dm + StellarInertiaTensorReduced: star + TotalInertiaTensorNoniterative: general + GasInertiaTensorNoniterative: gas + DarkMatterInertiaTensorNoniterative: dm + StellarInertiaTensorNoniterative: star + TotalInertiaTensorReducedNoniterative: general + GasInertiaTensorReducedNoniterative: gas + DarkMatterInertiaTensorReducedNoniterative: dm + StellarInertiaTensorReducedNoniterative: star + SpinParameter: general + StarFormationRate: basic + StarFormingGasMass: general + StarFormingGasMassFractionInMetals: basic + StellarInitialMass: star + StellarLuminosity: star + StellarMass: basic + StellarMassFractionInMetals: basic + StellarVelocityDispersionMatrix: star + TotalMass: basic aliases: PartType0/ElementMassFractions: PartType0/SmoothedElementMassFractions PartType4/ElementMassFractions: PartType4/SmoothedElementMassFractions @@ -632,7 +445,7 @@ defined_constants: Fe_H_sun: 2.82e-5 calculations: min_read_radius_cmpc: 5 - calculate_missing_properties: true + calculate_missing_properties: false reduced_snapshots: min_halo_mass: 1e13 halo_bin_size_dex: .05 diff --git a/parameter_files/MINIMAL_FLAMINGO.yml b/parameter_files/MINIMAL_FLAMINGO.yml index de73487a..87506330 100644 --- a/parameter_files/MINIMAL_FLAMINGO.yml +++ b/parameter_files/MINIMAL_FLAMINGO.yml @@ -45,6 +45,8 @@ SOProperties: ConcentrationUnsoftened: true MassFractionSatellites: true MassFractionExternal: true + MaximumCircularVelocity: true + MaximumCircularVelocityRadius: true NumberOfDarkMatterParticles: true SORadius: true SpinParameter: true @@ -63,6 +65,7 @@ SubhaloProperties: properties: CentreOfMass: true CentreOfMassVelocity: true + EncloseRadius: true NumberOfDarkMatterParticles: true NumberOfGasParticles: true NumberOfStarParticles: true @@ -72,9 +75,6 @@ SubhaloProperties: MaximumCircularVelocityRadiusUnsoftened: true SpinParameter: true TotalMass: true - variations: - Bound: - bound_only: true filters: general: limit: 100 diff --git a/parameter_files/README.md b/parameter_files/README.md index 6e78265a..6e058b95 100644 --- a/parameter_files/README.md +++ b/parameter_files/README.md @@ -2,6 +2,14 @@ The parameter files are a YAML dictionary which define the parameters and settings for running SOAP. This file describes the structure of a parameter file, including all possible fields which can be specified. +This file does not detail what the differences are between the various aperture types, for that +see the main pdf documenation. + +### DMO runs + +SOAP does not require separate parameter files for DMO & HYDRO runs. Instead you +must pass the `--dmo` flag when running on a DMO simulation, and in that case +any hydro-only properties will be skipped. ### Parameters @@ -33,9 +41,14 @@ If a dataset is present in both the snapshot and the extra input files, the valu Settings for the halo finding algorithm and output file locations. -- **type**: The halo finder being used. Possible options are `HBTplus`, `VR`, `Subfind`, and `Rockstar`. -- **filename**: Template for input halo catalogue files. The format of this depends on the halo finder as they have different output structure. HBTplus example: `"{sim_dir}/{sim_name}/HBT/{snap_nr:03d}/SubSnap_{snap_nr:03d}"` -- **fof_filename**: Template for FOF catalog files. Used for storing host FOF information for central subhalos. This is currently only supported for HBTplus +- **type**: The subhalo finder being used. Possible options are `HBTplus`, `VR`, `Subfind`, and `Rockstar`. +- **filename**: Template for input halo catalogue files. The format of this depends on the halo finder as they each have a different output structure. + - HBTplus: `"{sim_dir}/{sim_name}/HBT/{snap_nr:03d}/SubSnap_{snap_nr:03d}"` + - Sorted HBTplus: `"{sim_dir}/{sim_name}/HBT/{snap_nr:03d}/OrderedSubSnap_{snap_nr:03d}.hdf5"` +- **fof_filename**: Template for FOF catalog files. Used for storing host FOF information for central subhalos. Only supported for HBTplus +- **fof_radius_filename**: Template for FOF catalog files which contain the "Groups/Radii" dataset. These were produced by a post-processing script, and are missing from the main FOFs +- **read_potential_energies**: Optional boolean value, defaults to False. Whether to read potential energies and place them in the membership files. Only supported for HBTplus + ### Group Membership @@ -50,21 +63,38 @@ Settings for writing the output SOAP catalogues properties, and for handling tem - **filename**: Template for the halo properties file paths, e.g. `"{output_dir}/{sim_name}/SOAP_uncompressed/{halo_finder}/halo_properties_{snap_nr:04d}.hdf5"` - **chunk_dir**: Directory for temporary chunk output files. e.g. `"{scratch_dir}/{sim_name}/SOAP-tmp/{halo_finder}/"` -### ApertureProperties +### SubhaloProperties -Define which fixed spherical apertures to compute, and what properties to compute within them. +Define which properties to compute for each subhalo. -- **properties**: A list of properties to compute. Each property can be passed a boolean, which simply flags whether to enable or disable calculation. Alternatively a dictionary can be passed with two entries: `snapshot` and `snipshot`. In this case the behaviour will be determined based on whether the `--snipshot` flag is passed when running SOAP. -- **variations**: A list of which apertures to compute. Each aperture must have a name, a boolean key `inclusive` to indicate whether to include unbound particles, and a key `radius_in_kpc` to indicate the **physical size** of the aperture. Each aperture can optionally be passed the `filter` key. If the filter key is set then the aperture will only be computed for subhalos that fulfill the filter criteria. If no `filter` key is passed then the aperture will be computed for all halos. +- **properties**: A list of properties to compute. The key for each property should be either a boolean, a string, or a dictionary. Passing a boolean simply flags whether to enable or disable that calculation (for all subhalos). If the property should only be computed for subhalos that meet a certain criteria then the name of the filter should be passed as a string (see the Filters section below). Alternatively a dictionary can be passed with two entries: `snapshot` and `snipshot`. In this case the behaviour will be determined based on whether the `--snipshot` flag is passed when running SOAP. An example is as follows ``` -ApertureProperties: +SubhaloProperties: properties: TotalMass: true + StellarMass: general GasMass: snapshot: true snipshot: false +``` + +### ApertureProperties + +Define which fixed spherical apertures to compute, and what properties to compute within them. + +- **properties**: The same format as used for SubhaloProperties. +- **variations**: A list of which apertures to compute. Each aperture must have a name, a boolean key `inclusive` to indicate whether to include unbound particles. The aperture can be specified in two different ways: + - Passing a value of `radius_in_kpc` to indicate the **physical size** of the aperture. + - Passing a string as `property`, which is the name of another property which has been computed by SOAP. In this case each subhalo will use it's own value of that property as the aperture radius. `radius_muliple` can optionally be passed, in which case the aperture radius will be equal to the multiplier times the property value. +Each aperture can optionally be passed the `filter` key. If the filter key is set then the aperture will only be computed for subhalos that fulfill the filter criteria. If no `filter` key is passed then the aperture will be computed for all subhalos. For inclusive apertures the boolean flag `skip_gt_enclose_radius` can be set (defaults to False). If it is set then properties will not be calculated for any subhalos where all bound particles are within the aperture radius. + +An example is as follows +``` +ApertureProperties: + properties: + TotalMass: true variations: exclusive_50kpc: inclusive: false @@ -73,17 +103,54 @@ ApertureProperties: inclusive: true radius_in_kpc: 50.0 filter: general + skip_gt_enclose_radius: false + exclusive_half_mass: + inclusive: false + property: BoundSubhalo/HalfMassRadiusTotal + exclusive_twice_half_mass: + inclusive: false + property: BoundSubhalo/HalfMassRadiusTotal + radius_multiple: 2.0 +``` + +If you do not wish to calculate any apertures then pass any empty dict to both the properties and the variations, e.g. + +``` +ApertureProperties: + properties: + {} + variations: + {} ``` ### ProjectedApertureProperties -Define which fixed projected apertures to compute, and what properties to compute within them. The structure is exactly the same as for ApertureProperties, with the exception that no `inclusive` key can be set for a variation. This is because ProjectedApertures all always computed using only bound particles. +Define which fixed projected apertures to compute, and what properties to compute within them. The structure is exactly the same as for ApertureProperties, with the exception that the keys `inclusive` and `skip_gt_enclose_radius` can be not set for the variations. This is because ProjectedApertures will always computed using only bound particles. + +An example is as follows +``` +ProjectedApertureProperties: + properties: + TotalMass: true + GasMass: + snapshot: true + snipshot: false + variations: + 50_kpc: + radius_in_kpc: 50.0 + 100_kpc: + radius_in_kpc: 100.0 + filter: general + twice_half_mass: + property: BoundSubhalo/HalfMassRadiusTotal + radius_multiple: 2.0 +``` ### SOProperties Define which spherical overdensity aperture to compute, and what properties to compute within them. -- **properties**: The same as for ApertureProperties. +- **properties**: The same as for SubhaloProperties. - **variations**: A list of which apertures to compute. Each aperture must have a name, a `type` (options: `crit`, `mean`, `BN98`), and a `value` which indicates what multiple of the crit/mean density to use. As with ApertureProperties, a `filter` can optionally be passed. `core_excision_fraction` can optionally be passed to set the size of the core when calculating core_excised properties (no core_excised properties will be calculated if this is not passed). `radius_multiple` can optionally be passed for calculating an aperture which is a multiple of one of the previous aperture. An example is as follows @@ -117,27 +184,19 @@ SOProperties: filter: general ``` -### SubhaloProperties - -Define which properties to compute for the bound subhalos. - -- **properties**: The same as for ApertureProperties. - -Unless using VR catalogues, add the following for variations -``` -variations: - Bound: - bound_only: true -``` - ### Aliases -Used if field names in the snapshots do not agree with what SOAP expects, e.g. +Optional. Used if field names in the snapshots do not agree with what SOAP expects. If the `snipshot` section is passed then those aliases will be used when running in snipshot mode (the snipshot values will not be combined with the snapshot aliases present. If no `snipshot` section is present then the snapshot aliases will be used when running in snipshot mode). E.g. ``` aliases: PartType0/ElementMassFractions: PartType0/SmoothedElementMassFractions PartType4/ElementMassFractions: PartType4/SmoothedElementMassFractions + snipshot: + PartType0/ElementMassFractions: PartType0/ReducedElementMassFractions + PartType4/ElementMassFractions: PartType4/SmoothedElementMassFractions + ``` +For each alias the key is the name of the property that SOAP expects, and the value is the name of the property in the snapshot being passed. ### Filters @@ -177,7 +236,7 @@ filters: ### Defined constants -Constants used when running SOAP +Optional. Constants used when running SOAP ``` defined_constants: @@ -189,8 +248,8 @@ defined_constants: Contains information about how to run SOAP -- **min_read_radius_cmpc**: SOAP makes an initial guess of the radius around each halo to read in. -- **calculate_missing_properties**: Boolen value. If set to true then SOAP will calculate any properties which are not listed in the parameter file. If set to false then SOAP will ignore these properties +- **min_read_radius_cmpc**: Optional. Using the input halo catalogues SOAP makes an initial guess of the radius around each halo to read in. This value can be set so SOAP will read a minimum radius by default, which can be useful if large SOs are being calculated. +- **calculate_missing_properties**: Optional, default True. If set to true then SOAP will calculate any properties which are not listed in the parameter file. If set to false then SOAP will ignore these properties - **reduced_snapshots**: Optional. We create reduced snapshots where we keep the particles within the virial radius of certain objects. The values here determine which halos to keep. - **min_halo_mass**: The minimumum M200 halo mass to keep - **halo_bin_size_dex**: The size of the halo mass bins @@ -201,3 +260,4 @@ Contains information about how to run SOAP - **cold_dense_gas_filter**: Optional. How to determine which gas particles count as being cold & dense - **maximum_temperature_K**: Value above which gas is not considered to be cold - **minimum_hydrogen_number_density_cm3**: Value below which gas gas is not considered to be dense +- **strict_halo_copy**: Optional, default False. When a halo has multiple ExclusiveSphere/ProjectedAperture halo types which encompass all the bound particles then we just copy across the values rather than recomputing them. There are a small number of properties for which this is not correct. If this flag is set then these properties are set to zero for the larger apertures instead of being copied across. diff --git a/parameter_files/XRAY_ONLY.yml b/parameter_files/XRAY_ONLY.yml new file mode 100644 index 00000000..f1dbd640 --- /dev/null +++ b/parameter_files/XRAY_ONLY.yml @@ -0,0 +1,147 @@ +# Values in this section are substituted into the other sections +Parameters: + sim_dir: /cosma8/data/dp004/colibre/Runs + output_dir: /snap8/scratch/dp004/dc-mcgi1/soap_xray_props + scratch_dir: /snap8/scratch/dp004/dc-mcgi1/soap_xray_props + +# Location of the Swift snapshots: +Snapshots: + # Use {snap_nr:04d} for the snapshot number and {file_nr} for the file number. + filename: "{sim_dir}/{sim_name}/snapshots/colibre_{snap_nr:04d}/colibre_{snap_nr:04d}.{file_nr}.hdf5" + +# Which halo finder we're using, and base name for halo finder output files +HaloFinder: + type: HBTplus + filename: "{sim_dir}/{sim_name}/HBTplus/{snap_nr:03d}/SubSnap_{snap_nr:03d}" + # fof_filename: "{sim_dir}/{sim_name}/fof/fof_output_{snap_nr:04d}/fof_output_{snap_nr:04d}.{file_nr}.hdf5" + #type: VR + #filename: "{sim_dir}/halo_{snap_nr:04d}" + #type: Subfind + #filename: "{sim_dir}/snapdir_{snap_nr:03d}/snapshot_{snap_nr:03d}" + +GroupMembership: + # Where to write the group membership files + filename: "{sim_dir}/{sim_name}/SOAP/membership_{snap_nr:04d}/membership_{snap_nr:04d}.{file_nr}.hdf5" + +HaloProperties: + # Where to write the halo properties file + filename: "{output_dir}/{sim_name}/SOAP_uncompressed/halo_properties_{snap_nr:04d}.hdf5" + # Where to write temporary chunk output + chunk_dir: "{scratch_dir}/{sim_name}/SOAP-tmp/" + +ApertureProperties: + properties: + StellarMass: true + variations: + exclusive_50_kpc: + inclusive: false + radius_in_kpc: 50.0 +ProjectedApertureProperties: + properties: + {} + variations: + {} +SOProperties: + properties: + XRayLuminosityDensityCut: true + XRayLuminosity: + snapshot: true + snipshot: false + XRayLuminosityWithoutRecentAGNHeating: + snapshot: true + snipshot: false + XRayLuminosityCoreExcision: + snapshot: true + snipshot: false + XRayLuminosityWithoutRecentAGNHeatingCoreExcision: + snapshot: true + snipshot: false + XRayLuminosityInRestframe: false + XRayLuminosityInRestframeWithoutRecentAGNHeating: false + XRayLuminosityInRestframeCoreExcision: false + XRayLuminosityInRestframeWithoutRecentAGNHeatingCoreExcision: false + XRayPhotonLuminosity: + snapshot: true + snipshot: false + XRayPhotonLuminosityWithoutRecentAGNHeating: + snapshot: true + snipshot: false + XRayPhotonLuminosityCoreExcision: + snapshot: true + snipshot: false + XRayPhotonLuminosityWithoutRecentAGNHeatingCoreExcision: + snapshot: true + snipshot: false + XRayPhotonLuminosityInRestframe: false + XRayPhotonLuminosityInRestframeWithoutRecentAGNHeating: false + XRayPhotonLuminosityInRestframeCoreExcision: false + XRayPhotonLuminosityInRestframeWithoutRecentAGNHeatingCoreExcision: false + TotalMass: true + XRayLuminosityNoSat: true + XRayLuminosityNoSatCoreExcision: true + variations: + 200_crit: + type: crit + value: 200.0 + 500_crit: + type: crit + value: 500.0 + core_excision_fraction: 0.15 +SubhaloProperties: + properties: + EncloseRadius: true + NumberOfBlackHoleParticles: true + NumberOfDarkMatterParticles: true + NumberOfGasParticles: true + NumberOfStarParticles: true + TotalMass: true +aliases: + PartType0/LastSNIIKineticFeedbackDensities: PartType0/DensitiesAtLastSupernovaEvent + PartType0/LastSNIIThermalFeedbackDensities: PartType0/DensitiesAtLastSupernovaEvent + snipshot: + PartType0/SpeciesFractions: PartType0/ReducedSpeciesFractions + PartType0/ElementMassFractions: PartType0/ReducedElementMassFractions + PartType0/LastSNIIKineticFeedbackDensities: PartType0/DensitiesAtLastSupernovaEvent + PartType0/LastSNIIThermalFeedbackDensities: PartType0/DensitiesAtLastSupernovaEvent +filters: + general: + limit: 100 + properties: + - BoundSubhalo/NumberOfGasParticles + - BoundSubhalo/NumberOfDarkMatterParticles + - BoundSubhalo/NumberOfStarParticles + - BoundSubhalo/NumberOfBlackHoleParticles + combine_properties: sum + baryon: + limit: 0 + properties: + - BoundSubhalo/NumberOfGasParticles + - BoundSubhalo/NumberOfStarParticles + combine_properties: sum + dm: + limit: 0 + properties: + - BoundSubhalo/NumberOfDarkMatterParticles + gas: + limit: 0 + properties: + - BoundSubhalo/NumberOfGasParticles + star: + limit: 0 + properties: + - BoundSubhalo/NumberOfStarParticles +defined_constants: + O_H_sun: 4.9e-4 + Fe_H_sun: 3.16e-5 + N_O_sun: 0.138 + C_O_sun: 0.549 + Mg_H_sun: 3.98e-5 +calculations: + calculate_missing_properties: false + stict_halo_copy: false + recently_heated_gas_filter: + delta_time_myr: 15 + use_AGN_delta_T: false + cold_dense_gas_filter: + maximum_temperature_K: 3.16e4 + minimum_hydrogen_number_density_cm3: 0.1 diff --git a/property_table.py b/property_table.py deleted file mode 100644 index d818ef7a..00000000 --- a/property_table.py +++ /dev/null @@ -1,4855 +0,0 @@ -#!/bin/env python - -""" -property_table.py - -This file contains all the properties that can be calculated by SOAP, and some -functionality to automatically generate the documentation (PDF) containing these -properties. - -The rationale for having all of this in one file (and what is essentially one -big dictionary) is consistency: every property is defined exactly once, with -one data type, one unit, one description... Every type of halo still -implements its own calculation of each property, but everything that is exposed -to the user is guaranteed to be consistent for all halo types. To change the -documentation, you need to change the dictionary, so you will automatically -change the code as well. If you remember to regenerate the documentation, the -code will hence always be consistent with its documentation. The documentation -includes a version string to help identify it. - -When a specific type of halo wants to implement a property, it should import the -property table from this file and grab all of the information for the -corresponding dictionary element, e.g. (taken from aperture_properties.py) - - from property_table import PropertyTable - property_list = [ - (prop, *PropertyTable.full_property_list[prop]) - for prop in [ - "Mtot", - "Mgas", - "Mdm", - "Mstar", - ] - ] - -The elements of each row are documented later in this file. - -Note that this file contains some code that helps to regenerate the dictionary -itself. That is useful for adding additional rows to the table. -""" - -import numpy as np -import unyt -import subprocess -import datetime -import os -from typing import Dict, List -from halo_properties import HaloProperty - - -def get_version_string() -> str: - """ - Generate a version string that uniquely identifies the documentation file. - - The version string will have the format - SOAP version a7baa6e -- Compiled by user ``vandenbroucke'' on winkel - on Tuesday 15 November 2022, 10:49:10 - or - Unknown SOAP version -- Compiled by user ``vandenbroucke'' on winkel - on Tuesday 15 November 2022, 10:49:10 - if no git version string can be obtained. - """ - - handle = subprocess.run("git describe --always", shell=True, stdout=subprocess.PIPE) - if handle.returncode != 0: - git_version = "Unknown SOAP version" - else: - git_version = handle.stdout.decode("utf-8").strip() - git_version = f"SOAP version ``{git_version}''" - timestamp = datetime.datetime.now().strftime("%A %-d %B %Y, %H:%M:%S") - username = os.getlogin() - hostname = os.uname().nodename - return ( - f"Generated by user ``{username}'' on {hostname} on {timestamp}. {git_version}" - ) - - -def word_wrap_name(name): - """ - Put a line break in if a name gets too long - """ - maxlen = 20 - count = 0 - output = [] - last_was_lower = False - for i in range(len(name)): - next_char = name[i] - count += 1 - if count > maxlen and next_char.isupper() and last_was_lower: - output.append(r"\-") - output.append(next_char) - last_was_lower = next_char.isupper() == False - return "".join(output) - - -class PropertyTable: - """ - Auxiliary object to manipulate the property table. - - You should only create a PropertyTable object if you actually want to use - it to generate an updated version of the internal property dictionary or - to generate the documentation. If you just want to grab the information for - a particular property from the table, you should directly access the - static table, e.g. - Mstar_info = PropertyTable.full_property_list["Mstar"] - """ - - # some properties require an additional explanation in the form of a - # footnote. These footnotes are .tex files in the 'documentation' folder - # (that should exist). The name of the file acts as a key in the dictionary - # below; the corresponding value is a list of all properties that should - # include a footnote link to this particular explanation. - explanation = { - "footnote_MBH.tex": ["BHmaxM"], - "footnote_com.tex": ["com", "vcom"], - "footnote_AngMom.tex": ["Lgas", "Ldm", "Lstar", "Lbaryons"], - "footnote_kappa.tex": [ - "kappa_corot_gas", - "kappa_corot_star", - "kappa_corot_baryons", - ], - "footnote_disc_fraction.tex": ["DtoTstar", "DtoTgas"], - "footnote_SF.tex": [ - "SFR", - "gasFefrac_SF", - "gasOfrac_SF", - "Mgas_SF", - "gasmetalfrac_SF", - ], - "footnote_Tgas.tex": [ - "Tgas", - "Tgas_no_agn", - "Tgas_no_cool", - "Tgas_no_cool_no_agn", - ], - "footnote_lum.tex": ["StellarLuminosity"], - "footnote_circvel.tex": ["R_vmax_unsoft", "Vmax_unsoft", "Vmax_soft"], - "footnote_spin.tex": ["spin_parameter"], - "footnote_veldisp_matrix.tex": [ - "veldisp_matrix_gas", - "veldisp_matrix_dm", - "veldisp_matrix_star", - ], - "footnote_proj_veldisp.tex": [ - "proj_veldisp_gas", - "proj_veldisp_dm", - "proj_veldisp_star", - ], - "footnote_elements.tex": [ - "gasOfrac", - "gasOfrac_SF", - "gasFefrac", - "gasFefrac_SF", - "gasmetalfrac", - "gasmetalfrac_SF", - ], - "footnote_halfmass.tex": [ - "HalfMassRadiusTot", - "HalfMassRadiusGas", - "HalfMassRadiusDM", - "HalfMassRadiusStar", - ], - "footnote_satfrac.tex": ["Mfrac_satellites", "Mfrac_external"], - "footnote_Ekin.tex": ["Ekin_gas", "Ekin_star"], - "footnote_Etherm.tex": ["Etherm_gas"], - "footnote_Mnu.tex": ["Mnu", "MnuNS"], - "footnote_Xray.tex": [ - "Xraylum", - "Xraylum_restframe", - "Xrayphlum", - "Xrayphlum_restframe", - ], - "footnote_compY.tex": ["compY", "compY_no_agn"], - "footnote_dopplerB.tex": ["DopplerB"], - "footnote_coreexcision.tex": [ - "Tgas_cy_weighted_core_excision", - "Tgas_cy_weighted_core_excision_no_agn", - "Tgas_core_excision", - "Tgas_no_cool_core_excision", - "Tgas_no_agn_core_excision", - "Tgas_no_cool_no_agn_core_excision", - "Xraylum_core_excision", - "Xraylum_no_agn_core_excision", - "Xrayphlum_core_excision", - "Xrayphlum_no_agn_core_excision", - "SpectroscopicLikeTemperature_core_excision", - "SpectroscopicLikeTemperature_no_agn_core_excision", - ], - "footnote_cytemp.tex": [ - "Tgas_cy_weighted", - "Tgas_cy_weighted_no_agn", - "Tgas_cy_weighted_core_excision", - "Tgas_cy_weighted_core_excision_no_agn", - ], - "footnote_spectroscopicliketemperature.tex": [ - "SpectroscopicLikeTemperature", - "SpectroscopicLikeTemperature_core_excision", - "SpectroscopicLikeTemperature_no_agn", - "SpectroscopicLikeTemperature_no_agn_core_excision", - ], - "footnote_dust.tex": [ - "DustGraphiteMass", - "DustGraphiteMassInMolecularGas", - "DustGraphiteMassInAtomicGas", - "DustSilicatesMass", - "DustSilicatesMassInMolecularGas", - "DustSilicatesMassInAtomicGas", - "DustLargeGrainMass", - "DustLargeGrainMassInMolecularGas", - "DustSmallGrainMass", - "DustSmallGrainMassInMolecularGas", - ], - "footnote_diffuse.tex": [ - "DiffuseCarbonMass", - "DiffuseOxygenMass", - "DiffuseMagnesiumMass", - "DiffuseSiliconMass", - "DiffuseIronMass", - ], - "footnote_concentration.tex": [ - "concentration", - "concentration_soft", - "concentration_dmo", - "concentration_dmo_soft", - ], - "footnote_flow_rates.tex": [ - "DarkMatterMassFlowRate", - "ColdGasMassFlowRate", - "CoolGasMassFlowRate", - "WarmGasMassFlowRate", - "HotGasMassFlowRate", - "HIMassFlowRate", - "H2MassFlowRate", - "MetalMassFlowRate", - "StellarMassFlowRate", - "ColdGasEnergyFlowRate", - "CoolGasEnergyFlowRate", - "WarmGasEnergyFlowRate", - "HotGasEnergyFlowRate", - "ColdGasMomentumFlowRate", - "CoolGasMomentumFlowRate", - "WarmGasMomentumFlowRate", - "HotGasMomentumFlowRate", - ], - "footnote_tensor.tex": [ - "TotalInertiaTensor", - "TotalInertiaTensorReduced", - "TotalInertiaTensorNoniterative", - "TotalInertiaTensorReducedNoniterative", - "ProjectedTotalInertiaTensor", - "ProjectedTotalInertiaTensorReduced", - "ProjectedTotalInertiaTensorNoniterative", - "ProjectedTotalInertiaTensorReducedNoniterative", - ], - "footnote_metallicity.tex": [ - "LogarithmicMassWeightedDiffuseNitrogenOverOxygenOfGasLowLimit", - "LogarithmicMassWeightedDiffuseNitrogenOverOxygenOfGasHighLimit", - "LogarithmicMassWeightedDiffuseCarbonOverOxygenOfGasLowLimit", - "LogarithmicMassWeightedDiffuseCarbonOverOxygenOfGasHighLimit", - "LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfGasLowLimit", - "LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfGasHighLimit", - "LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfAtomicGasLowLimit", - "LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfAtomicGasHighLimit", - "LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfMolecularGasLowLimit", - "LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfMolecularGasHighLimit", - "LogarithmicMassWeightedIronOverHydrogenOfStarsLowLimit", - "LogarithmicMassWeightedIronOverHydrogenOfStarsHighLimit", - "LogarithmicMassWeightedMagnesiumOverHydrogenOfStarsLowLimit", - "LogarithmicMassWeightedMagnesiumOverHydrogenOfStarsHighLimit", - "LogarithmicMassWeightedIronFromSNIaOverHydrogenOfStarsLowLimit", - "LinearMassWeightedOxygenOverHydrogenOfGas", - "LinearMassWeightedNitrogenOverOxygenOfGas", - "LinearMassWeightedCarbonOverOxygenOfGas", - "LinearMassWeightedDiffuseNitrogenOverOxygenOfGas", - "LinearMassWeightedDiffuseCarbonOverOxygenOfGas", - "LinearMassWeightedDiffuseOxygenOverHydrogenOfGas", - "LinearMassWeightedIronOverHydrogenOfStars", - "LinearMassWeightedMagnesiumOverHydrogenOfStars", - "LinearMassWeightedIronFromSNIaOverHydrogenOfStars", - ], - } - - # dictionary with human-friendly descriptions of the various lossy - # compression filters that can be applied to data. - # The key is the name of a lossy compression filter (same names as used - # by SWIFT), the value is the corresponding description, which can be either - # an actual description or a representative example. - compression_description = { - "FMantissa9": "$1.36693{\\rm{}e}10 \\rightarrow{} 1.367{\\rm{}e}10$", - "FMantissa13": "$1.36693{\\rm{}e}10 \\rightarrow{} 1.3669{\\rm{}e}10$", - "DMantissa9": "$1.36693{\\rm{}e}10 \\rightarrow{} 1.367{\\rm{}e}10$", - "DScale6": "1 pc accurate", - "DScale5": "10 pc accurate", - "DScale1": "0.1 km/s accurate", - "Nbit40": "Store less bits", - "None": "no compression", - } - - # List of properties that get computed - # The key for each property is the name that is used internally in SOAP - # For each property, we have the following columns: - # - name: Name of the property within the output file - # - shape: Shape of this property for a single halo (1: scalar, - # 3: vector...) - # - dtype: Data type that will be used. Should have enough precision to - # avoid over/underflow - # - unit: Units that will be used internally and for the output. - # - description: Description string that will be used to describe the - # property in the output. - # - category: Category used to decide if this property should be calculated - # for a particular halo (filtering). - # - lossy compression filter: Lossy compression filter used in the output - # to reduce the file size. Note that SOAP does not actually compress - # the output; this is done by a separate script. We support all lossy - # compression filters available in SWIFT. - # - DMO property: Should this property be calculated for a DMO run? - # - Particle properties: Particle fields that are required to compute this - # property. Used to determine which particle fields to read for a - # particular SOAP configuration (as defined in the parameter file). - # - Output physical: Whether to output this value as physical or co-moving. - # - a-scale exponent: What a-scale exponent to set for this property. If set - # to None this marks that the property can not be converted to comoving - # - # Note that there is no good reason to have a diffent internal name and - # output name; this was mostly done for historical reasons. This means that - # you can easily change the name in the output without having to change all - # of the other .py files that use this property. - full_property_list = { - "AtomicHydrogenMass": ( - "AtomicHydrogenMass", - 1, - np.float32, - "snap_mass", - "Total gas mass in atomic hydrogen.", - "basic", - "FMantissa9", - False, - [ - "PartType0/Masses", - "PartType0/SpeciesFractions", - "PartType0/ElementMassFractions", - ], - True, - 0, - ), - "BHlasteventa": ( - "BlackHolesLastEventScalefactor", - 1, - np.float32, - "dimensionless", - "Scale-factor of last AGN event.", - "general", - "FMantissa9", - False, - ["PartType5/LastAGNFeedbackScaleFactors"], - True, - None, - ), - "BlackHolesTotalInjectedThermalEnergy": ( - "BlackHolesTotalInjectedThermalEnergy", - 1, - np.float32, - "snap_mass*snap_length**2/snap_time**2", - "Total thermal energy injected into gas particles by all black holes.", - "general", - "FMantissa9", - False, - ["PartType5/AGNTotalInjectedEnergies"], - True, - None, - ), - "BlackHolesTotalInjectedJetEnergy": ( - "BlackHolesTotalInjectedJetEnergy", - 1, - np.float32, - "snap_mass*snap_length**2/snap_time**2", - "Total jet energy injected into gas particles by all black holes.", - "general", - "FMantissa9", - False, - ["PartType5/InjectedJetEnergies"], - True, - None, - ), - "BHmaxAR": ( - "MostMassiveBlackHoleAccretionRate", - 1, - np.float32, - "snap_mass/snap_time", - "Gas accretion rate of most massive black hole.", - "general", - "FMantissa9", - False, - ["PartType5/SubgridMasses", "PartType5/AccretionRates"], - True, - None, - ), - "MostMassiveBlackHoleAveragedAccretionRate": ( - "MostMassiveBlackHoleAveragedAccretionRate", - 2, - np.float32, - "snap_mass/snap_time", - "Gas accretion rate of the most massive black hole, averaged over past 100Myr and past 10Myr. If the time between this snapshot and the previous one was less than the averaging time, then the value is averaged over the time between the snapshots.", - "general", - "FMantissa9", - False, - ["PartType5/SubgridMasses", "PartType5/AveragedAccretionRates"], - True, - None, - ), - "MostMassiveBlackHoleInjectedThermalEnergy": ( - "MostMassiveBlackHoleInjectedThermalEnergy", - 1, - np.float32, - "snap_mass*snap_length**2/snap_time**2", - "Total thermal energy injected into gas particles by the most massive black hole.", - "general", - "FMantissa9", - False, - ["PartType5/SubgridMasses", "PartType5/AGNTotalInjectedEnergies"], - True, - None, - ), - "MostMassiveBlackHoleAccretionMode": ( - "MostMassiveBlackHoleAccretionMode", - 1, - np.int32, - "dimensionless", - "Accretion flow regime of the most massive black hole. 0 - Thick disk, 1 - Thin disk, 2 - Slim disk", - "general", - "None", - False, - ["PartType5/SubgridMasses", "PartType5/AccretionModes"], - True, - 0, - ), - "MostMassiveBlackHoleGWMassLoss": ( - "MostMassiveBlackHoleGWMassLoss", - 1, - np.float32, - "snap_mass", - "Cumulative mass lost to GW via BH-BH mergers over the history of the most massive black holes. This includes the mass loss from all the progenitors.", - "general", - "FMantissa9", - False, - ["PartType5/SubgridMasses", "PartType5/GWMassLosses"], - True, - 0, - ), - "MostMassiveBlackHoleInjectedJetEnergyByMode": ( - "MostMassiveBlackHoleInjectedJetEnergyByMode", - 3, - np.float32, - "snap_mass*snap_length**2/snap_time**2", - "The total energy injected in the kinetic jet AGN feedback mode by the mostmassive black hole, split by accretion mode. The components correspond to the jet energy dumped in the thick, thin and slim disc modes, respectively.", - "general", - "FMantissa9", - False, - ["PartType5/SubgridMasses", "PartType5/InjectedJetEnergiesByMode"], - True, - None, - ), - "MostMassiveBlackHoleFormationScalefactor": ( - "MostMassiveBlackHoleFormationScalefactor", - 1, - np.float32, - "dimensionless", - "Scale-factor when most massive black hole was formed.", - "general", - "FMantissa13", - False, - ["PartType5/SubgridMasses", "PartType5/FormationScaleFactors"], - True, - None, - ), - "MostMassiveBlackHoleLastJetEventScalefactor": ( - "MostMassiveBlackHoleLastJetEventScalefactor", - 1, - np.float32, - "dimensionless", - "Scale-factor of last jet event for most massive black hole.", - "general", - "FMantissa13", - False, - ["PartType5/SubgridMasses", "PartType5/LastAGNJetScaleFactors"], - True, - None, - ), - "MostMassiveBlackHoleNumberOfAGNEvents": ( - "MostMassiveBlackHoleNumberOfAGNEvents", - 1, - np.int32, - "dimensionless", - "Number of thermal AGN events the most massive black hole has had so far", - "general", - "None", - False, - ["PartType5/SubgridMasses", "PartType5/NumberOfAGNEvents"], - True, - 0, - ), - "MostMassiveBlackHoleNumberOfAGNJetEvents": ( - "MostMassiveBlackHoleNumberOfAGNJetEvents", - 1, - np.int32, - "dimensionless", - "Number of jet events the most massive black hole has had so far", - "general", - "None", - False, - ["PartType5/SubgridMasses", "PartType5/NumberOfAGNJetEvents"], - True, - 0, - ), - "MostMassiveBlackHoleNumberOfMergers": ( - "MostMassiveBlackHoleNumberOfMergers", - 1, - np.int32, - "dimensionless", - "Number of mergers experienced by the most massive black hole.", - "general", - "None", - False, - ["PartType5/SubgridMasses", "PartType5/NumberOfMergers"], - True, - 0, - ), - "MostMassiveBlackHoleRadiatedEnergyByMode": ( - "MostMassiveBlackHoleRadiatedEnergyByMode", - 3, - np.float32, - "snap_mass*snap_length**2/snap_time**2", - "The total energy launched into radiation by the most massive black hole, split by accretion mode. The components correspond to the radiative energy dumped in the thick, thin and slim disc modes, respectively.", - "general", - "FMantissa9", - False, - ["PartType5/SubgridMasses", "PartType5/RadiatedEnergiesByMode"], - True, - None, - ), - "MostMassiveBlackHoleTotalAccretedMass": ( - "MostMassiveBlackHoleTotalAccretedMass", - 1, - np.float32, - "snap_mass", - "The total mass accreted by the most massive black hole.", - "general", - "FMantissa9", - False, - ["PartType5/SubgridMasses", "PartType5/TotalAccretedMasses"], - True, - 0, - ), - "MostMassiveBlackHoleTotalAccretedMassesByMode": ( - "MostMassiveBlackHoleTotalAccretedMassesByMode", - 3, - np.float32, - "snap_mass", - "The total mass accreted by the most massive black hole in each accretion mode. The components correspond to the mass accreted in the thick, thin and slim disc modes, respectively.", - "general", - "FMantissa9", - False, - ["PartType5/SubgridMasses", "PartType5/TotalAccretedMassesByMode"], - True, - 0, - ), - "MostMassiveBlackHoleSpin": ( - "MostMassiveBlackHoleSpin", - 1, - np.float32, - "dimensionless", - "Dimensionless spin of the most massive black hole. Negative values indicate retrograde accretion.", - "general", - "FMantissa9", - False, - ["PartType5/SubgridMasses", "PartType5/Spins"], - True, - 0, - ), - "MostMassiveBlackHoleWindEnergyByMode": ( - "MostMassiveBlackHoleWindEnergyByMode", - 3, - np.float32, - "snap_mass*snap_length**2/snap_time**2", - "The total energy launched into accretion disc winds by the most massive black hole, split by accretion mode. The components correspond to the radiative energy dumped in the thick, thin and slim disc modes, respectively.", - "general", - "FMantissa9", - False, - ["PartType5/SubgridMasses", "PartType5/WindEnergiesByMode"], - True, - None, - ), - "BHmaxID": ( - "MostMassiveBlackHoleID", - 1, - np.uint64, - "dimensionless", - "ID of most massive black hole.", - "basic", - "None", - False, - ["PartType5/SubgridMasses", "PartType5/ParticleIDs"], - True, - None, - ), - "BHmaxM": ( - "MostMassiveBlackHoleMass", - 1, - np.float32, - "snap_mass", - "Mass of most massive black hole.", - "basic", - "FMantissa9", - False, - ["PartType5/SubgridMasses"], - True, - 0, - ), - "BHmaxlasteventa": ( - "MostMassiveBlackHoleLastEventScalefactor", - 1, - np.float32, - "dimensionless", - "Scale-factor of last thermal AGN event for most massive black hole.", - "general", - "FMantissa13", - False, - ["PartType5/SubgridMasses", "PartType5/LastAGNFeedbackScaleFactors"], - True, - None, - ), - "BHmaxpos": ( - "MostMassiveBlackHolePosition", - 3, - np.float64, - "snap_length", - "Position of most massive black hole.", - "general", - "DScale6", - False, - ["PartType5/Coordinates", "PartType5/SubgridMasses"], - False, - 1, - ), - "BHmaxvel": ( - "MostMassiveBlackHoleVelocity", - 3, - np.float32, - "snap_length/snap_time", - "Velocity of most massive black hole relative to the simulation volume.", - "general", - "FMantissa9", - False, - ["PartType5/SubgridMasses", "PartType5/Velocities"], - False, - 1, - ), - "DarkMatterInertiaTensor": ( - "DarkMatterInertiaTensor", - 6, - np.float32, - "snap_length**2", - "3D inertia tensor computed iteratively from the DM mass distribution, relative to the halo centre. Diagonal components and one off-diagonal triangle as (1,1), (2,2), (3,3), (1,2), (1,3), (2,3). Only calculated when we have more than 20 particles.", - "dm", - "FMantissa9", - True, - ["PartType1/Coordinates", "PartType1/Masses"], - True, - 2, - ), - "DarkMatterInertiaTensorReduced": ( - "DarkMatterInertiaTensorReduced", - 6, - np.float32, - "dimensionless", - "Reduced 3D inertia tensor computed iteratively from the DM mass distribution, relative to the halo centre. Diagonal components and one off-diagonal triangle as (1,1), (2,2), (3,3), (1,2), (1,3), (2,3). Only calculated when we have more than 20 particles.", - "dm", - "FMantissa9", - True, - ["PartType1/Coordinates", "PartType1/Masses"], - True, - 0, - ), - "DarkMatterInertiaTensorNoniterative": ( - "DarkMatterInertiaTensorNoniterative", - 6, - np.float32, - "snap_length**2", - "3D inertia tensor computed in a single interation from the DM mass distribution, relative to the halo centre. Diagonal components and one off-diagonal triangle as (1,1), (2,2), (3,3), (1,2), (1,3), (2,3). Only calculated when we have more than 20 particles.", - "dm", - "FMantissa9", - True, - ["PartType1/Coordinates", "PartType1/Masses"], - True, - 2, - ), - "DarkMatterInertiaTensorReducedNoniterative": ( - "DarkMatterInertiaTensorReducedNoniterative", - 6, - np.float32, - "dimensionless", - "Reduced 3D inertia tensor computed in a single interation from the DM mass distribution, relative to the halo centre. Diagonal components and one off-diagonal triangle as (1,1), (2,2), (3,3), (1,2), (1,3), (2,3). Only calculated when we have more than 20 particles.", - "dm", - "FMantissa9", - True, - ["PartType1/Coordinates", "PartType1/Masses"], - True, - 0, - ), - "DiffuseCarbonMass": ( - "DiffuseCarbonMass", - 1, - np.float32, - "snap_mass", - "Total gas mass in carbon that is not contained in dust.", - "gas", - "FMantissa9", - False, - ["PartType0/Masses", "PartType0/ElementMassFractionsDiffuse"], - True, - 0, - ), - "DiffuseIronMass": ( - "DiffuseIronMass", - 1, - np.float32, - "snap_mass", - "Total gas mass in iron that is not contained in dust.", - "gas", - "FMantissa9", - False, - ["PartType0/Masses", "PartType0/ElementMassFractionsDiffuse"], - True, - 0, - ), - "DiffuseMagnesiumMass": ( - "DiffuseMagnesiumMass", - 1, - np.float32, - "snap_mass", - "Total gas mass in magnesium that is not contained in dust.", - "gas", - "FMantissa9", - False, - ["PartType0/Masses", "PartType0/ElementMassFractionsDiffuse"], - True, - 0, - ), - "DiffuseOxygenMass": ( - "DiffuseOxygenMass", - 1, - np.float32, - "snap_mass", - "Total gas mass in oxygen that is not contained in dust.", - "gas", - "FMantissa9", - False, - ["PartType0/Masses", "PartType0/ElementMassFractionsDiffuse"], - True, - 0, - ), - "DiffuseSiliconMass": ( - "DiffuseSiliconMass", - 1, - np.float32, - "snap_mass", - "Total gas mass in silicon that is not contained in dust.", - "gas", - "FMantissa9", - False, - ["PartType0/Masses", "PartType0/ElementMassFractionsDiffuse"], - True, - 0, - ), - "DopplerB": ( - "DopplerB", - 1, - np.float32, - "dimensionless", - "Kinetic Sunyaey-Zel'dovich effect, assuming a line of sight towards the position of the first lightcone observer.", - "general", - "FMantissa9", - False, - [ - "PartType0/Coordinates", - "PartType0/Velocities", - "PartType0/ElectronNumberDensities", - "PartType0/Densities", - ], - False, - 1, - ), - "DtoTgas": ( - "DiscToTotalGasMassFraction", - 1, - np.float32, - "dimensionless", - "Fraction of the total gas mass that is in the disc.", - "gas", - "FMantissa9", - False, - ["PartType0/Coordinates", "PartType0/Masses", "PartType0/Velocities"], - True, - 0, - ), - "DtoTstar": ( - "DiscToTotalStellarMassFraction", - 1, - np.float32, - "dimensionless", - "Fraction of the total stellar mass that is in the disc.", - "star", - "FMantissa9", - False, - ["PartType4/Coordinates", "PartType4/Velocities", "PartType4/Masses"], - True, - 0, - ), - "DustGraphiteMass": ( - "DustGraphiteMass", - 1, - np.float32, - "snap_mass", - "Total dust mass in graphite grains.", - "gas", - "FMantissa9", - False, - ["PartType0/Masses", "PartType0/DustMassFractions"], - True, - 0, - ), - "DustGraphiteMassInAtomicGas": ( - "DustGraphiteMassInAtomicGas", - 1, - np.float32, - "snap_mass", - "Total dust mass in graphite grains in atomic gas.", - "gas", - "FMantissa9", - False, - [ - "PartType0/Masses", - "PartType0/DustMassFractions", - "PartType0/ElementMassFractions", - "PartType0/SpeciesFractions", - ], - True, - 0, - ), - "DustGraphiteMassInMolecularGas": ( - "DustGraphiteMassInMolecularGas", - 1, - np.float32, - "snap_mass", - "Total dust mass in graphite grains in molecular gas.", - "gas", - "FMantissa9", - False, - [ - "PartType0/Masses", - "PartType0/DustMassFractions", - "PartType0/SpeciesFractions", - "PartType0/ElementMassFractions", - ], - True, - 0, - ), - "DustGraphiteMassInColdDenseGas": ( - "DustGraphiteMassInColdDenseGas", - 1, - np.float32, - "snap_mass", - "Total dust mass in graphite grains in cold, dense gas.", - "gas", - "FMantissa9", - False, - [ - "PartType0/Masses", - "PartType0/DustMassFractions", - "PartType0/Densities", - "PartType0/Temperatures", - ], - True, - 0, - ), - "DustLargeGrainMass": ( - "DustLargeGrainMass", - 1, - np.float32, - "snap_mass", - "Total dust mass in large grains.", - "gas", - "FMantissa9", - False, - ["PartType0/Masses", "PartType0/DustMassFractions"], - True, - 0, - ), - "DustLargeGrainMassInMolecularGas": ( - "DustLargeGrainMassInMolecularGas", - 1, - np.float32, - "snap_mass", - "Total dust mass in large grains in molecular gas.", - "gas", - "FMantissa9", - False, - [ - "PartType0/Masses", - "PartType0/DustMassFractions", - "PartType0/SpeciesFractions", - "PartType0/ElementMassFractions", - ], - True, - 0, - ), - "DustLargeGrainMassInColdDenseGas": ( - "DustLargeGrainMassInColdDenseGas", - 1, - np.float32, - "snap_mass", - "Total dust mass in large grains in cold, dense gas.", - "gas", - "FMantissa9", - False, - [ - "PartType0/Masses", - "PartType0/DustMassFractions", - "PartType0/Densities", - "PartType0/Temperatures", - ], - True, - 0, - ), - "DustSilicatesMass": ( - "DustSilicatesMass", - 1, - np.float32, - "snap_mass", - "Total dust mass in silicate grains.", - "gas", - "FMantissa9", - False, - ["PartType0/Masses", "PartType0/DustMassFractions"], - True, - 0, - ), - "DustSilicatesMassInAtomicGas": ( - "DustSilicatesMassInAtomicGas", - 1, - np.float32, - "snap_mass", - "Total dust mass in silicate grains in atomic gas.", - "gas", - "FMantissa9", - False, - [ - "PartType0/Masses", - "PartType0/DustMassFractions", - "PartType0/SpeciesFractions", - "PartType0/ElementMassFractions", - ], - True, - 0, - ), - "DustSilicatesMassInMolecularGas": ( - "DustSilicatesMassInMolecularGas", - 1, - np.float32, - "snap_mass", - "Total dust mass in silicate grains in molecular gas.", - "gas", - "FMantissa9", - False, - [ - "PartType0/Masses", - "PartType0/DustMassFractions", - "PartType0/SpeciesFractions", - "PartType0/ElementMassFractions", - ], - True, - 0, - ), - "DustSilicatesMassInColdDenseGas": ( - "DustSilicatesMassInColdDenseGas", - 1, - np.float32, - "snap_mass", - "Total dust mass in silicate grains in cold, dense gas.", - "gas", - "FMantissa9", - False, - [ - "PartType0/Masses", - "PartType0/DustMassFractions", - "PartType0/Densities", - "PartType0/Temperatures", - ], - True, - 0, - ), - "DustSmallGrainMass": ( - "DustSmallGrainMass", - 1, - np.float32, - "snap_mass", - "Total dust mass in small grains.", - "gas", - "FMantissa9", - False, - [ - "PartType0/Masses", - "PartType0/DustMassFractions", - "PartType0/ElementMassFractions", - ], - True, - 0, - ), - "DustSmallGrainMassInMolecularGas": ( - "DustSmallGrainMassInMolecularGas", - 1, - np.float32, - "snap_mass", - "Total dust mass in small grains in molecular gas.", - "gas", - "FMantissa9", - False, - [ - "PartType0/Masses", - "PartType0/DustMassFractions", - "PartType0/SpeciesFractions", - "PartType0/ElementMassFractions", - ], - True, - 0, - ), - "DustSmallGrainMassInColdDenseGas": ( - "DustSmallGrainMassInColdDenseGas", - 1, - np.float32, - "snap_mass", - "Total dust mass in small grains in cold, dense gas.", - "gas", - "FMantissa9", - False, - [ - "PartType0/Masses", - "PartType0/DustMassFractions", - "PartType0/Densities", - "PartType0/Temperatures", - ], - True, - 0, - ), - "Ekin_gas": ( - "KineticEnergyGas", - 1, - np.float64, - "snap_mass*snap_length**2/snap_time**2", - "Total kinetic energy of the gas, relative to the gas centre of mass velocity.", - "gas", - "DMantissa9", - False, - ["PartType0/Masses", "PartType0/Velocities"], - True, - -2, - ), - "Ekin_star": ( - "KineticEnergyStars", - 1, - np.float64, - "snap_mass*snap_length**2/snap_time**2", - "Total kinetic energy of the stars, relative to the stellar centre of mass velocity.", - "star", - "DMantissa9", - False, - ["PartType4/Masses", "PartType4/Velocities"], - True, - -2, - ), - "Etherm_gas": ( - "ThermalEnergyGas", - 1, - np.float64, - "snap_mass*snap_length**2/snap_time**2", - "Total thermal energy of the gas.", - "general", - "DMantissa9", - False, - ["PartType0/Densities", "PartType0/Pressures", "PartType0/Masses"], - True, - -2, - ), - "GasInertiaTensor": ( - "GasInertiaTensor", - 6, - np.float32, - "snap_length**2", - "3D inertia tensor computed iteratively from the gas mass distribution, relative to the halo centre. Diagonal components and one off-diagonal triangle as (1,1), (2,2), (3,3), (1,2), (1,3), (2,3). Only calculated when we have more than 20 particles.", - "gas", - "FMantissa9", - False, - ["PartType0/Coordinates", "PartType0/Masses"], - True, - 2, - ), - "GasInertiaTensorReduced": ( - "GasInertiaTensorReduced", - 6, - np.float32, - "dimensionless", - "Reduced 3D inertia tensor computed iteratively from the gas mass distribution, relative to the halo centre. Diagonal components and one off-diagonal triangle as (1,1), (2,2), (3,3), (1,2), (1,3), (2,3). Only calculated when we have more than 20 particles.", - "gas", - "FMantissa9", - False, - ["PartType0/Coordinates", "PartType0/Masses"], - True, - 0, - ), - "GasInertiaTensorNoniterative": ( - "GasInertiaTensorNoniterative", - 6, - np.float32, - "snap_length**2", - "3D inertia tensor computed in a single iteration from the gas mass distribution, relative to the halo centre. Diagonal components and one off-diagonal triangle as (1,1), (2,2), (3,3), (1,2), (1,3), (2,3). Only calculated when we have more than 20 particles.", - "gas", - "FMantissa9", - False, - ["PartType0/Coordinates", "PartType0/Masses"], - True, - 2, - ), - "GasInertiaTensorReducedNoniterative": ( - "GasInertiaTensorReducedNoniterative", - 6, - np.float32, - "dimensionless", - "Reduced 3D inertia tensor computed in a single iteration from the gas mass distribution, relative to the halo centre. Diagonal components and one off-diagonal triangle as (1,1), (2,2), (3,3), (1,2), (1,3), (2,3). Only calculated when we have more than 20 particles.", - "gas", - "FMantissa9", - False, - ["PartType0/Coordinates", "PartType0/Masses"], - True, - 0, - ), - "HalfMassRadiusBaryon": ( - "HalfMassRadiusBaryons", - 1, - np.float32, - "snap_length", - "Baryonic (gas and stars) half mass radius.", - "baryon", - "FMantissa9", - False, - [ - "PartType0/Coordinates", - "PartType0/Masses", - "PartType4/Coordinates", - "PartType4/Masses", - ], - False, - 1, - ), - "HalfMassRadiusDM": ( - "HalfMassRadiusDarkMatter", - 1, - np.float32, - "snap_length", - "Dark matter half mass radius.", - "dm", - "FMantissa9", - True, - ["PartType1/Coordinates", "PartType1/Masses"], - False, - 1, - ), - "HalfMassRadiusGas": ( - "HalfMassRadiusGas", - 1, - np.float32, - "snap_length", - "Gas half mass radius.", - "gas", - "FMantissa9", - False, - ["PartType0/Coordinates", "PartType0/Masses"], - False, - 1, - ), - "HalfMassRadiusStar": ( - "HalfMassRadiusStars", - 1, - np.float32, - "snap_length", - "Stellar half mass radius.", - "basic", - "FMantissa9", - False, - ["PartType4/Coordinates", "PartType4/Masses"], - False, - 1, - ), - "HalfMassRadiusTot": ( - "HalfMassRadiusTotal", - 1, - np.float32, - "snap_length", - "Total half mass radius.", - "general", - "FMantissa9", - True, - [ - "PartType0/Coordinates", - "PartType0/Masses", - "PartType1/Coordinates", - "PartType1/Masses", - "PartType4/Coordinates", - "PartType4/Masses", - "PartType5/Coordinates", - "PartType5/DynamicalMasses", - ], - False, - 1, - ), - "EncloseRadius": ( - "EncloseRadius", - 1, - np.float32, - "snap_length", - "Radius of the particle furthest from the halo centre", - "basic", - "FMantissa9", - True, - [ - "PartType0/Coordinates", - "PartType1/Coordinates", - "PartType4/Coordinates", - "PartType5/Coordinates", - ], - False, - 1, - ), - "HeliumMass": ( - "HeliumMass", - 1, - np.float32, - "snap_mass", - "Total gas mass in helium.", - "gas", - "FMantissa9", - False, - ["PartType0/Masses", "PartType0/ElementMassFractions"], - True, - 0, - ), - "HydrogenMass": ( - "HydrogenMass", - 1, - np.float32, - "snap_mass", - "Total gas mass in hydrogen.", - "gas", - "FMantissa9", - False, - ["PartType0/Masses", "PartType0/ElementMassFractions"], - True, - 0, - ), - "IonisedHydrogenMass": ( - "IonisedHydrogenMass", - 1, - np.float32, - "snap_mass", - "Total gas mass in ionised hydrogen.", - "gas", - "FMantissa9", - False, - [ - "PartType0/Masses", - "PartType0/SpeciesFractions", - "PartType0/ElementMassFractions", - ], - True, - 0, - ), - "LastSupernovaEventMaximumGasDensity": ( - "LastSupernovaEventMaximumGasDensity", - 1, - np.float32, - "snap_mass/snap_length**3", - "Maximum gas density at the last supernova event for the last supernova event of each gas particle.", - "gas", - "FMantissa9", - False, - [ - "PartType0/LastSNIIThermalFeedbackDensities", - "PartType0/LastSNIIKineticFeedbackDensities", - ], - True, - None, - ), - "Lbaryons": ( - "AngularMomentumBaryons", - 3, - np.float32, - "snap_mass*snap_length**2/snap_time", - "Total angular momentum of baryons (gas and stars), relative to the centre of potential and baryonic centre of mass velocity.", - "baryon", - "FMantissa9", - False, - [ - "PartType0/Coordinates", - "PartType0/Masses", - "PartType0/Velocities", - "PartType4/Coordinates", - "PartType4/Masses", - "PartType4/Velocities", - ], - True, - 2, - ), - "Ldm": ( - "AngularMomentumDarkMatter", - 3, - np.float32, - "snap_mass*snap_length**2/snap_time", - "Total angular momentum of the dark matter, relative to the centre of potential and DM centre of mass velocity.", - "dm", - "FMantissa9", - True, - ["PartType1/Coordinates", "PartType1/Masses", "PartType1/Velocities"], - True, - 2, - ), - "Lgas": ( - "AngularMomentumGas", - 3, - np.float32, - "snap_mass*snap_length**2/snap_time", - "Total angular momentum of the gas, relative to the centre of potential and gas centre of mass velocity.", - "gas", - "FMantissa9", - False, - ["PartType0/Coordinates", "PartType0/Masses", "PartType0/Velocities"], - True, - 2, - ), - "MedianStellarBirthDensity": ( - "MedianStellarBirthDensity", - 1, - np.float32, - "snap_mass/snap_length**3", - "Median density of gas particles that were converted into a star particle.", - "star", - "FMantissa9", - False, - ["PartType4/BirthDensities"], - True, - None, - ), - "MedianStellarBirthTemperature": ( - "MedianStellarBirthTemperature", - 1, - np.float32, - "snap_temperature", - "Median temperature of gas particles that were converted into a star particle.", - "star", - "FMantissa9", - False, - ["PartType4/BirthTemperatures"], - True, - None, - ), - "MedianStellarBirthPressure": ( - "MedianStellarBirthPressure", - 1, - np.float64, - "snap_temperature/snap_length**3", - "Median pressure of gas particles that were converted into a star particle.", - "star", - "DMantissa9", - False, - ["PartType4/BirthTemperatures", "PartType4/BirthDensities"], - True, - None, - ), - "Lstar": ( - "AngularMomentumStars", - 3, - np.float32, - "snap_mass*snap_length**2/snap_time", - "Total angular momentum of the stars, relative to the centre of potential and stellar centre of mass velocity.", - "star", - "FMantissa9", - False, - ["PartType4/Coordinates", "PartType4/Masses", "PartType4/Velocities"], - True, - 2, - ), - "MaximumStellarBirthDensity": ( - "MaximumStellarBirthDensity", - 1, - np.float32, - "snap_mass/snap_length**3", - "Maximum density of gas that was converted into a star particle.", - "star", - "FMantissa9", - False, - ["PartType4/BirthDensities"], - True, - None, - ), - "MaximumStellarBirthTemperature": ( - "MaximumStellarBirthTemperature", - 1, - np.float32, - "snap_temperature", - "Maximum temperature of gas that was converted into a star particle.", - "star", - "FMantissa9", - False, - ["PartType4/BirthTemperatures"], - True, - None, - ), - "MaximumStellarBirthPressure": ( - "MaximumStellarBirthPressure", - 1, - np.float64, - "snap_temperature/snap_length**3", - "Maximum pressure of gas that was converted into a star particle.", - "star", - "DMantissa9", - False, - ["PartType4/BirthTemperatures", "PartType4/BirthDensities"], - True, - None, - ), - "Mbh_dynamical": ( - "BlackHolesDynamicalMass", - 1, - np.float32, - "snap_mass", - "Total BH dynamical mass.", - "basic", - "FMantissa9", - False, - ["PartType5/DynamicalMasses"], - True, - 0, - ), - "Mbh_subgrid": ( - "BlackHolesSubgridMass", - 1, - np.float32, - "snap_mass", - "Total BH subgrid mass.", - "basic", - "FMantissa9", - False, - ["PartType5/SubgridMasses"], - True, - 0, - ), - "Mdm": ( - "DarkMatterMass", - 1, - np.float32, - "snap_mass", - "Total DM mass.", - "basic", - "FMantissa9", - True, - ["PartType1/Masses"], - True, - 0, - ), - "Mfrac_satellites": ( - "MassFractionSatellites", - 1, - np.float32, - "dimensionless", - "Fraction of mass that is bound to a satellite in the same FOF group.", - "general", - "FMantissa9", - True, - [ - "PartType0/Masses", - "PartType1/Masses", - "PartType4/Masses", - "PartType5/DynamicalMasses", - "PartType0/FOFGroupIDs", - "PartType1/FOFGroupIDs", - "PartType4/FOFGroupIDs", - "PartType5/FOFGroupIDs", - "PartType0/GroupNr_bound", - "PartType1/GroupNr_bound", - "PartType4/GroupNr_bound", - "PartType5/GroupNr_bound", - ], - True, - 0, - ), - "Mfrac_external": ( - "MassFractionExternal", - 1, - np.float32, - "dimensionless", - "Fraction of mass that is bound to a satellite outside this FOF group.", - "general", - "FMantissa9", - True, - [ - "PartType0/Masses", - "PartType1/Masses", - "PartType4/Masses", - "PartType5/DynamicalMasses", - "PartType0/FOFGroupIDs", - "PartType1/FOFGroupIDs", - "PartType4/FOFGroupIDs", - "PartType5/FOFGroupIDs", - "PartType0/GroupNr_bound", - "PartType1/GroupNr_bound", - "PartType4/GroupNr_bound", - "PartType5/GroupNr_bound", - ], - True, - 0, - ), - "Mgas": ( - "GasMass", - 1, - np.float32, - "snap_mass", - "Total gas mass.", - "basic", - "FMantissa9", - False, - ["PartType0/Masses"], - True, - 0, - ), - "Mgas_SF": ( - "StarFormingGasMass", - 1, - np.float32, - "snap_mass", - "Total mass of star-forming gas.", - "general", - "FMantissa9", - False, - ["PartType0/Masses", "PartType0/StarFormationRates"], - True, - 0, - ), - "Mhotgas": ( - "HotGasMass", - 1, - np.float32, - "snap_mass", - "Total mass of gas with a temperature above 1e5 K.", - "general", - "FMantissa9", - False, - ["PartType0/Masses", "PartType0/Temperatures"], - True, - 0, - ), - "GasMassInColdDenseGas": ( - "GasMassInColdDenseGas", - 1, - np.float32, - "snap_mass", - "Total mass of gas in cold, dense gas.", - "gas", - "FMantissa9", - False, - ["PartType0/Masses", "PartType0/Densities", "PartType0/Temperatures"], - True, - 0, - ), - "MinimumStellarBirthDensity": ( - "MinimumStellarBirthDensity", - 1, - np.float32, - "snap_mass/snap_length**3", - "Minimum density of gas that was converted into a star particle.", - "star", - "FMantissa9", - False, - ["PartType4/BirthDensities"], - True, - None, - ), - "MinimumStellarBirthTemperature": ( - "MinimumStellarBirthTemperature", - 1, - np.float32, - "snap_temperature", - "Minimum temperature of gas that was converted into a star particle.", - "star", - "FMantissa9", - False, - ["PartType4/BirthTemperatures"], - True, - None, - ), - "MinimumStellarBirthPressure": ( - "MinimumStellarBirthPressure", - 1, - np.float64, - "snap_temperature/snap_length**3", - "Minimum pressure of gas that was converted into a star particle.", - "star", - "DMantissa9", - False, - ["PartType4/BirthTemperatures", "PartType4/BirthDensities"], - True, - None, - ), - "Mnu": ( - "RawNeutrinoMass", - 1, - np.float32, - "snap_mass", - "Total neutrino particle mass.", - "basic", - "FMantissa9", - True, - ["PartType6/Masses"], - True, - 0, - ), - "MnuNS": ( - "NoiseSuppressedNeutrinoMass", - 1, - np.float32, - "snap_mass", - "Noise suppressed total neutrino mass.", - "basic", - "FMantissa9", - True, - ["PartType6/Masses", "PartType6/Weights"], - True, - 0, - ), - "MolecularHydrogenMass": ( - "MolecularHydrogenMass", - 1, - np.float32, - "snap_mass", - "Total gas mass in molecular hydrogen.", - "basic", - "FMantissa9", - False, - [ - "PartType0/Masses", - "PartType0/SpeciesFractions", - "PartType0/ElementMassFractions", - ], - True, - 0, - ), - "Mstar": ( - "StellarMass", - 1, - np.float32, - "snap_mass", - "Total stellar mass.", - "basic", - "FMantissa9", - False, - ["PartType4/Masses"], - True, - 0, - ), - "Mstar_init": ( - "StellarInitialMass", - 1, - np.float32, - "snap_mass", - "Total stellar initial mass.", - "star", - "FMantissa9", - False, - ["PartType4/InitialMasses"], - True, - 0, - ), - "Mtot": ( - "TotalMass", - 1, - np.float32, - "snap_mass", - "Total mass.", - "basic", - "FMantissa9", - True, - [ - "PartType0/Masses", - "PartType1/Masses", - "PartType4/Masses", - "PartType5/DynamicalMasses", - ], - True, - 0, - ), - "Nbh": ( - "NumberOfBlackHoleParticles", - 1, - np.uint32, - "dimensionless", - "Number of black hole particles.", - "basic", - "None", - False, - [], - True, - None, - ), - "Ndm": ( - "NumberOfDarkMatterParticles", - 1, - np.uint32, - "dimensionless", - "Number of dark matter particles.", - "basic", - "None", - True, - [], - True, - None, - ), - "Ngas": ( - "NumberOfGasParticles", - 1, - np.uint32, - "dimensionless", - "Number of gas particles.", - "basic", - "None", - False, - [], - True, - None, - ), - "Nnu": ( - "NumberOfNeutrinoParticles", - 1, - np.uint32, - "dimensionless", - "Number of neutrino particles.", - "basic", - "None", - False, - [], - True, - None, - ), - "Nstar": ( - "NumberOfStarParticles", - 1, - np.uint32, - "dimensionless", - "Number of star particles.", - "basic", - "None", - False, - [], - True, - None, - ), - "ProjectedTotalInertiaTensor": ( - "ProjectedTotalInertiaTensor", - 3, - np.float32, - "snap_length**2", - "2D inertia tensor computed iteratively from the total mass distribution, relative to the halo centre. Diagonal components and one off-diagonal value as (1,1), (2,2), (1,2). Only calculated when we have more than 20 particles.", - "general", - "FMantissa9", - False, - [ - "PartType0/Coordinates", - "PartType0/Masses", - "PartType1/Coordinates", - "PartType1/Masses", - "PartType4/Coordinates", - "PartType4/Masses", - "PartType5/Coordinates", - "PartType5/DynamicalMasses", - ], - True, - 2, - ), - "ProjectedTotalInertiaTensorReduced": ( - "ProjectedTotalInertiaTensorReduced", - 3, - np.float32, - "dimensionless", - "Reduced 2D inertia tensor computed iteratively from the total mass distribution, relative to the halo centre. Diagonal components and one off-diagonal value as (1,1), (2,2), (1,2). Only calculated when we have more than 20 particles.", - "general", - "FMantissa9", - False, - [ - "PartType0/Coordinates", - "PartType0/Masses", - "PartType1/Coordinates", - "PartType1/Masses", - "PartType4/Coordinates", - "PartType4/Masses", - "PartType5/Coordinates", - "PartType5/DynamicalMasses", - ], - True, - 0, - ), - "ProjectedTotalInertiaTensorNoniterative": ( - "ProjectedTotalInertiaTensorNoniterative", - 3, - np.float32, - "snap_length**2", - "2D inertia tensor computed in a single iteration from the total mass distribution, relative to the halo centre. Diagonal components and one off-diagonal value as (1,1), (2,2), (1,2). Only calculated when we have more than 20 particles.", - "general", - "FMantissa9", - False, - [ - "PartType0/Coordinates", - "PartType0/Masses", - "PartType1/Coordinates", - "PartType1/Masses", - "PartType4/Coordinates", - "PartType4/Masses", - "PartType5/Coordinates", - "PartType5/DynamicalMasses", - ], - True, - 2, - ), - "ProjectedTotalInertiaTensorReducedNoniterative": ( - "ProjectedTotalInertiaTensorReducedNoniterative", - 3, - np.float32, - "dimensionless", - "Reduced 2D inertia tensor computed in a single iteration from the total mass distribution, relative to the halo centre. Diagonal components and one off-diagonal value as (1,1), (2,2), (1,2). Only calculated when we have more than 20 particles.", - "general", - "FMantissa9", - False, - [ - "PartType0/Coordinates", - "PartType0/Masses", - "PartType1/Coordinates", - "PartType1/Masses", - "PartType4/Coordinates", - "PartType4/Masses", - "PartType5/Coordinates", - "PartType5/DynamicalMasses", - ], - True, - 0, - ), - "ProjectedGasInertiaTensor": ( - "ProjectedGasInertiaTensor", - 3, - np.float32, - "snap_length**2", - "2D inertia tensor computed iteratively from the gas mass distribution, relative to the halo centre. Diagonal components and one off-diagonal value as (1,1), (2,2), (1,2). Only calculated when we have more than 20 particles.", - "gas", - "FMantissa9", - False, - ["PartType0/Coordinates", "PartType0/Masses"], - True, - 2, - ), - "ProjectedGasInertiaTensorReduced": ( - "ProjectedGasInertiaTensorReduced", - 3, - np.float32, - "dimensionless", - "Reduced 2D inertia tensor computed iteratively from the gas mass distribution, relative to the halo centre. Diagonal components and one off-diagonal value as (1,1), (2,2), (1,2). Only calculated when we have more than 20 particles.", - "gas", - "FMantissa9", - False, - ["PartType0/Coordinates", "PartType0/Masses"], - True, - 0, - ), - "ProjectedGasInertiaTensorNoniterative": ( - "ProjectedGasInertiaTensorNoniterative", - 3, - np.float32, - "snap_length**2", - "2D inertia tensor computed in a single iteration from the gas mass distribution, relative to the halo centre. Diagonal components and one off-diagonal value as (1,1), (2,2), (1,2). Only calculated when we have more than 20 particles.", - "gas", - "FMantissa9", - False, - ["PartType0/Coordinates", "PartType0/Masses"], - True, - 2, - ), - "ProjectedGasInertiaTensorReducedNoniterative": ( - "ProjectedGasInertiaTensorReducedNoniterative", - 3, - np.float32, - "dimensionless", - "Reduced 2D inertia tensor computed in a single iteration from the gas mass distribution, relative to the halo centre. Diagonal components and one off-diagonal value as (1,1), (2,2), (1,2). Only calculated when we have more than 20 particles.", - "gas", - "FMantissa9", - False, - ["PartType0/Coordinates", "PartType0/Masses"], - True, - 0, - ), - "ProjectedStellarInertiaTensor": ( - "ProjectedStellarInertiaTensor", - 3, - np.float32, - "snap_length**2", - "2D inertia tensor computed iteratively from the stellar mass distribution, relative to the halo centre. Diagonal components and one off-diagonal value as (1,1), (2,2), (1,2). Only calculated when we have more than 20 particles.", - "star", - "FMantissa9", - False, - ["PartType4/Coordinates", "PartType4/Masses"], - True, - 2, - ), - "ProjectedStellarInertiaTensorReduced": ( - "ProjectedStellarInertiaTensorReduced", - 3, - np.float32, - "dimensionless", - "Reduced 2D inertia tensor computed iteratively from the stellar mass distribution, relative to the halo centre. Diagonal components and one off-diagonal value as (1,1), (2,2), (1,2). Only calculated when we have more than 20 particles.", - "star", - "FMantissa9", - False, - ["PartType4/Coordinates", "PartType4/Masses"], - True, - 0, - ), - "ProjectedStellarInertiaTensorNoniterative": ( - "ProjectedStellarInertiaTensorNoniterative", - 3, - np.float32, - "snap_length**2", - "2D inertia tensor computed in a single iteration from the stellar mass distribution, relative to the halo centre. Diagonal components and one off-diagonal value as (1,1), (2,2), (1,2). Only calculated when we have more than 20 particles.", - "star", - "FMantissa9", - False, - ["PartType4/Coordinates", "PartType4/Masses"], - True, - 2, - ), - "ProjectedStellarInertiaTensorReducedNoniterative": ( - "ProjectedStellarInertiaTensorReducedNoniterative", - 3, - np.float32, - "dimensionless", - "Reduced 2D inertia tensor computed in a single iteration from the stellar mass distribution, relative to the halo centre. Diagonal components and one off-diagonal value as (1,1), (2,2), (1,2). Only calculated when we have more than 20 particles.", - "star", - "FMantissa9", - False, - ["PartType4/Coordinates", "PartType4/Masses"], - True, - 0, - ), - "SFR": ( - "StarFormationRate", - 1, - np.float32, - "snap_mass/snap_time", - "Total star formation rate.", - "basic", - "FMantissa9", - False, - ["PartType0/StarFormationRates"], - True, - None, - ), - "AveragedStarFormationRate": ( - "AveragedStarFormationRate", - 2, - np.float32, - "snap_mass/snap_time", - "Total star formation rate, averaged over past 100Myr and past 10Myr. If the time between this snapshot and the previous one was less than the averaging time, then the value is averaged over the time between the snapshots.", - "basic", - "FMantissa9", - False, - ["PartType0/AveragedStarFormationRates"], - True, - None, - ), - "StellarInertiaTensor": ( - "StellarInertiaTensor", - 6, - np.float32, - "snap_length**2", - "3D inertia tensor computed iteratively from the stellar mass distribution, relative to the halo centre. Diagonal components and one off-diagonal triangle as (1,1), (2,2), (3,3), (1,2), (1,3), (2,3). Only calculated when we have more than 20 particles.", - "star", - "FMantissa9", - False, - ["PartType4/Coordinates", "PartType4/Masses"], - True, - 2, - ), - "StellarInertiaTensorReduced": ( - "StellarInertiaTensorReduced", - 6, - np.float32, - "dimensionless", - "Reduced 3D inertia tensor computed iteratively from the stellar mass distribution, relative to the halo centre. Diagonal components and one off-diagonal triangle as (1,1), (2,2), (3,3), (1,2), (1,3), (2,3). Only calculated when we have more than 20 particles.", - "star", - "FMantissa9", - False, - ["PartType4/Coordinates", "PartType4/Masses"], - True, - 0, - ), - "StellarInertiaTensorNoniterative": ( - "StellarInertiaTensorNoniterative", - 6, - np.float32, - "snap_length**2", - "3D inertia tensor computed in a single iteration from the stellar mass distribution, relative to the halo centre. Diagonal components and one off-diagonal triangle as (1,1), (2,2), (3,3), (1,2), (1,3), (2,3). Only calculated when we have more than 20 particles.", - "star", - "FMantissa9", - False, - ["PartType4/Coordinates", "PartType4/Masses"], - True, - 2, - ), - "StellarInertiaTensorReducedNoniterative": ( - "StellarInertiaTensorReducedNoniterative", - 6, - np.float32, - "dimensionless", - "Reduced 3D inertia tensor computed in a single iteration from the stellar mass distribution, relative to the halo centre. Diagonal components and one off-diagonal triangle as (1,1), (2,2), (3,3), (1,2), (1,3), (2,3). Only calculated when we have more than 20 particles.", - "star", - "FMantissa9", - False, - ["PartType4/Coordinates", "PartType4/Masses"], - True, - 0, - ), - "StellarLuminosity": ( - "StellarLuminosity", - 9, - np.float32, - "dimensionless", - "Total stellar luminosity in the 9 GAMA bands.", - "star", - "FMantissa9", - False, - ["PartType4/Luminosities"], - True, - None, - ), - "Tgas": ( - "GasTemperature", - 1, - np.float32, - "snap_temperature", - "Mass-weighted mean gas temperature.", - "general", - "FMantissa9", - False, - ["PartType0/Temperatures"], - True, - 0, - ), - "Tgas_no_agn": ( - "GasTemperatureWithoutRecentAGNHeating", - 1, - np.float32, - "snap_temperature", - "Mass-weighted mean gas temperature, excluding gas that was recently heated by AGN.", - "general", - "FMantissa9", - False, - ["PartType0/Temperatures", "PartType0/LastAGNFeedbackScaleFactors"], - True, - 0, - ), - "Tgas_no_cool": ( - "GasTemperatureWithoutCoolGas", - 1, - np.float32, - "snap_temperature", - "Mass-weighted mean gas temperature, excluding cool gas with a temperature below 1e5 K.", - "general", - "FMantissa9", - False, - ["PartType0/Temperatures"], - True, - 0, - ), - "Tgas_no_cool_no_agn": ( - "GasTemperatureWithoutCoolGasAndRecentAGNHeating", - 1, - np.float32, - "snap_temperature", - "Mass-weighted mean gas temperature, excluding cool gas with a temperature below 1e5 K and gas that was recently heated by AGN.", - "general", - "FMantissa9", - False, - ["PartType0/Temperatures", "PartType0/LastAGNFeedbackScaleFactors"], - True, - 0, - ), - "Tgas_cy_weighted": ( - "GasComptonYTemperature", - 1, - np.float32, - "snap_temperature", - "ComptonY-weighted mean gas temperature.", - "general", - "FMantissa9", - False, - ["PartType0/Temperatures", "PartType0/ComptonYParameters"], - True, - 0, - ), - "Tgas_cy_weighted_no_agn": ( - "GasComptonYTemperatureWithoutRecentAGNHeating", - 1, - np.float32, - "snap_temperature", - "ComptonY-weighted mean gas temperature, excluding gas that was recently heated by AGN.", - "general", - "FMantissa9", - False, - [ - "PartType0/Temperatures", - "PartType0/ComptonYParameters", - "PartType0/LastAGNFeedbackScaleFactors", - ], - True, - 0, - ), - "Tgas_cy_weighted_core_excision": ( - "GasComptonYTemperatureCoreExcision", - 1, - np.float32, - "snap_temperature", - "ComptonY-weighted mean gas temperature, excluding the inner {core_excision}.", - "general", - "FMantissa9", - False, - [ - "PartType0/Temperatures", - "PartType0/ComptonYParameters", - "PartType0/Coordinates", - ], - True, - 0, - ), - "Tgas_cy_weighted_core_excision_no_agn": ( - "GasComptonYTemperatureWithoutRecentAGNHeatingCoreExcision", - 1, - np.float32, - "snap_temperature", - "ComptonY-weighted mean gas temperature, excluding the inner {core_excision} and gas that was recently heated by AGN.", - "general", - "FMantissa9", - False, - [ - "PartType0/Temperatures", - "PartType0/ComptonYParameters", - "PartType0/Coordinates", - "PartType0/LastAGNFeedbackScaleFactors", - ], - True, - 0, - ), - "Tgas_core_excision": ( - "GasTemperatureCoreExcision", - 1, - np.float32, - "snap_temperature", - "Mass-weighted mean gas temperature, excluding the inner {core_excision}.", - "general", - "FMantissa9", - False, - ["PartType0/Temperatures", "PartType0/Masses", "PartType0/Coordinates"], - True, - 0, - ), - "Tgas_no_cool_core_excision": ( - "GasTemperatureWithoutCoolGasCoreExcision", - 1, - np.float32, - "snap_temperature", - "Mass-weighted mean gas temperature, excluding the inner {core_excision} and gas below 1e5 K.", - "general", - "FMantissa9", - False, - ["PartType0/Temperatures", "PartType0/Masses", "PartType0/Coordinates"], - True, - 0, - ), - "Tgas_no_agn_core_excision": ( - "GasTemperatureWithoutRecentAGNHeatingCoreExcision", - 1, - np.float32, - "snap_temperature", - "Mass-weighted mean gas temperature, excluding the inner {core_excision}, and gas that was recently heated by AGN.", - "general", - "FMantissa9", - False, - [ - "PartType0/Temperatures", - "PartType0/Masses", - "PartType0/Coordinates", - "PartType0/LastAGNFeedbackScaleFactors", - ], - True, - 0, - ), - "Tgas_no_cool_no_agn_core_excision": ( - "GasTemperatureWithoutCoolGasAndRecentAGNHeatingCoreExcision", - 1, - np.float32, - "snap_temperature", - "Mass-weighted mean gas temperature, excluding the inner {core_excision}, gas below 1e5 K and gas that was recently heated by AGN.", - "general", - "FMantissa9", - False, - [ - "PartType0/Temperatures", - "PartType0/Masses", - "PartType0/Coordinates", - "PartType0/LastAGNFeedbackScaleFactors", - ], - True, - 0, - ), - "TotalInertiaTensor": ( - "TotalInertiaTensor", - 6, - np.float32, - "snap_length**2", - "3D inertia tensor computed iteratively from the total mass distribution, relative to the halo centre. Diagonal components and one off-diagonal triangle as (1,1), (2,2), (3,3), (1,2), (1,3), (2,3). Only calculated when we have more than 20 particles.", - "general", - "FMantissa9", - True, - [ - "PartType0/Coordinates", - "PartType0/Masses", - "PartType1/Coordinates", - "PartType1/Masses", - "PartType4/Coordinates", - "PartType4/Masses", - "PartType5/Coordinates", - "PartType5/DynamicalMasses", - ], - True, - 2, - ), - "TotalInertiaTensorReduced": ( - "TotalInertiaTensorReduced", - 6, - np.float32, - "dimensionless", - "Reduced 3D inertia tensor computed iteratively from the total mass distribution, relative to the halo centre. Diagonal components and one off-diagonal triangle as (1,1), (2,2), (3,3), (1,2), (1,3), (2,3). Only calculated when we have more than 20 particles.", - "general", - "FMantissa9", - True, - [ - "PartType0/Coordinates", - "PartType0/Masses", - "PartType1/Coordinates", - "PartType1/Masses", - "PartType4/Coordinates", - "PartType4/Masses", - "PartType5/Coordinates", - "PartType5/DynamicalMasses", - ], - True, - 0, - ), - "TotalInertiaTensorNoniterative": ( - "TotalInertiaTensorNoniterative", - 6, - np.float32, - "snap_length**2", - "3D inertia tensor computed in a single iteration from the total mass distribution, relative to the halo centre. Diagonal components and one off-diagonal triangle as (1,1), (2,2), (3,3), (1,2), (1,3), (2,3). Only calculated when we have more than 20 particles.", - "general", - "FMantissa9", - True, - [ - "PartType0/Coordinates", - "PartType0/Masses", - "PartType1/Coordinates", - "PartType1/Masses", - "PartType4/Coordinates", - "PartType4/Masses", - "PartType5/Coordinates", - "PartType5/DynamicalMasses", - ], - True, - 2, - ), - "TotalInertiaTensorReducedNoniterative": ( - "TotalInertiaTensorReducedNoniterative", - 6, - np.float32, - "dimensionless", - "Reduced 3D inertia tensor computed in a single iteration from the total mass distribution, relative to the halo centre. Diagonal components and one off-diagonal triangle as (1,1), (2,2), (3,3), (1,2), (1,3), (2,3). Only calculated when we have more than 20 particles.", - "general", - "FMantissa9", - True, - [ - "PartType0/Coordinates", - "PartType0/Masses", - "PartType1/Coordinates", - "PartType1/Masses", - "PartType4/Coordinates", - "PartType4/Masses", - "PartType5/Coordinates", - "PartType5/DynamicalMasses", - ], - True, - 0, - ), - "TotalSNIaRate": ( - "TotalSNIaRate", - 1, - np.float32, - "1/snap_time", - "Total SNIa rate.", - "star", - "FMantissa9", - False, - ["PartType4/SNIaRates"], - True, - None, - ), - "R_vmax_unsoft": ( - "MaximumCircularVelocityRadiusUnsoftened", - 1, - np.float32, - "snap_length", - "Radius at which MaximumCircularVelocityUnsoftened is reached.", - "basic", - "FMantissa9", - True, - [ - "PartType0/Coordinates", - "PartType0/Masses", - "PartType1/Coordinates", - "PartType1/Masses", - "PartType4/Coordinates", - "PartType4/Masses", - "PartType5/Coordinates", - "PartType5/DynamicalMasses", - ], - False, - 1, - ), - "Vmax_unsoft": ( - "MaximumCircularVelocityUnsoftened", - 1, - np.float32, - "snap_length/snap_time", - "Maximum circular velocity when not accounting for particle softening lengths.", - "basic", - "FMantissa9", - True, - [ - "PartType0/Coordinates", - "PartType0/Masses", - "PartType1/Coordinates", - "PartType1/Masses", - "PartType4/Coordinates", - "PartType4/Masses", - "PartType5/Coordinates", - "PartType5/DynamicalMasses", - ], - True, - 0, - ), - "Vmax_soft": ( - "MaximumCircularVelocity", - 1, - np.float32, - "snap_length/snap_time", - "Maximum circular velocity when accounting for particle softening lengths.", - "basic", - "FMantissa9", - True, - [ - "PartType0/Coordinates", - "PartType0/Masses", - "PartType1/Coordinates", - "PartType1/Masses", - "PartType4/Coordinates", - "PartType4/Masses", - "PartType5/Coordinates", - "PartType5/DynamicalMasses", - ], - True, - 0, - ), - "DM_R_vmax_soft": ( - "MaximumDarkMatterCircularVelocityRadius", - 1, - np.float32, - "snap_length", - "Radius at which MaximumDarkMatterCircularVelocity is reached.", - "dm", - "FMantissa9", - False, - ["PartType1/Coordinates", "PartType1/Masses"], - False, - 1, - ), - "DM_Vmax_soft": ( - "MaximumDarkMatterCircularVelocity", - 1, - np.float32, - "snap_length/snap_time", - "Maximum circular velocity calculated using dark matter particles when accounting for particle softening lengths..", - "dm", - "FMantissa9", - False, - ["PartType1/Coordinates", "PartType1/Masses"], - True, - 0, - ), - "Xraylum": ( - "XRayLuminosity", - 3, - np.float64, - "snap_mass*snap_length**2/snap_time**3", - "Total observer-frame Xray luminosity in three bands.", - "general", - "DMantissa9", - False, - ["PartType0/XrayLuminosities"], - True, - None, - ), - "Xraylum_restframe": ( - "XRayLuminosityInRestframe", - 3, - np.float64, - "snap_mass*snap_length**2/snap_time**3", - "Total rest-frame Xray luminosity in three bands.", - "general", - "DMantissa9", - False, - ["PartType0/XrayLuminositiesRestframe"], - True, - 0, - ), - "Xraylum_no_agn": ( - "XRayLuminosityWithoutRecentAGNHeating", - 3, - np.float64, - "snap_mass*snap_length**2/snap_time**3", - "Total observer-frame Xray luminosity in three bands. Excludes gas that was recently heated by AGN.", - "general", - "DMantissa9", - False, - [ - "PartType0/XrayLuminosities", - "PartType0/LastAGNFeedbackScaleFactors", - "PartType0/Temperatures", - ], - True, - None, - ), - "Xraylum_restframe_no_agn": ( - "XRayLuminosityInRestframeWithoutRecentAGNHeating", - 3, - np.float64, - "snap_mass*snap_length**2/snap_time**3", - "Total rest-frame Xray luminosity in three bands. Excludes gas that was recently heated by AGN.", - "general", - "DMantissa9", - False, - [ - "PartType0/XrayLuminositiesRestframe", - "PartType0/LastAGNFeedbackScaleFactors", - "PartType0/Temperatures", - ], - True, - 0, - ), - "Xraylum_core_excision": ( - "XRayLuminosityCoreExcision", - 3, - np.float64, - "snap_mass*snap_length**2/snap_time**3", - "Total observer-frame Xray luminosity in three bands. Excludes gas in the inner {core_excision}", - "general", - "DMantissa9", - False, - ["PartType0/XrayLuminosities", "PartType0/Coordinates"], - True, - None, - ), - "Xraylum_restframe_core_excision": ( - "XRayLuminosityInRestframeCoreExcision", - 3, - np.float64, - "snap_mass*snap_length**2/snap_time**3", - "Total rest-frame Xray luminosity in three bands. Excludes gas in the inner {core_excision}", - "general", - "DMantissa9", - False, - ["PartType0/XrayLuminositiesRestframe", "PartType0/Coordinates"], - True, - 0, - ), - "Xraylum_no_agn_core_excision": ( - "XRayLuminosityWithoutRecentAGNHeatingCoreExcision", - 3, - np.float64, - "snap_mass*snap_length**2/snap_time**3", - "Total observer-frame Xray luminosity in three bands. Excludes gas that was recently heated by AGN. Excludes gas in the inner {core_excision}", - "general", - "DMantissa9", - False, - [ - "PartType0/XrayLuminosities", - "PartType0/LastAGNFeedbackScaleFactors", - "PartType0/Temperatures", - "PartType0/Coordinates", - ], - True, - None, - ), - "Xraylum_restframe_no_agn_core_excision": ( - "XRayLuminosityInRestframeWithoutRecentAGNHeatingCoreExcision", - 3, - np.float64, - "snap_mass*snap_length**2/snap_time**3", - "Total rest-frame Xray luminosity in three bands. Excludes gas that was recently heated by AGN. Excludes gas in the inner {core_excision}", - "general", - "DMantissa9", - False, - [ - "PartType0/XrayLuminositiesRestframe", - "PartType0/LastAGNFeedbackScaleFactors", - "PartType0/Temperatures", - "PartType0/Coordinates", - ], - True, - 0, - ), - "Xrayphlum": ( - "XRayPhotonLuminosity", - 3, - np.float64, - "1/snap_time", - "Total observer-frame Xray photon luminosity in three bands.", - "general", - "DMantissa9", - False, - ["PartType0/XrayPhotonLuminosities"], - True, - None, - ), - "Xrayphlum_restframe": ( - "XRayPhotonLuminosityInRestframe", - 3, - np.float64, - "1/snap_time", - "Total rest-frame Xray photon luminosity in three bands.", - "general", - "DMantissa9", - False, - ["PartType0/XrayPhotonLuminositiesRestframe"], - True, - 0, - ), - "Xrayphlum_no_agn": ( - "XRayPhotonLuminosityWithoutRecentAGNHeating", - 3, - np.float64, - "1/snap_time", - "Total observer-frame Xray photon luminosity in three bands. Exclude gas that was recently heated by AGN.", - "general", - "DMantissa9", - False, - [ - "PartType0/XrayPhotonLuminosities", - "PartType0/LastAGNFeedbackScaleFactors", - "PartType0/Temperatures", - ], - True, - None, - ), - "Xrayphlum_restframe_no_agn": ( - "XRayPhotonLuminosityInRestframeWithoutRecentAGNHeating", - 3, - np.float64, - "1/snap_time", - "Total rest-frame Xray photon luminosity in three bands. Exclude gas that was recently heated by AGN.", - "general", - "DMantissa9", - False, - [ - "PartType0/XrayPhotonLuminositiesRestframe", - "PartType0/LastAGNFeedbackScaleFactors", - "PartType0/Temperatures", - ], - True, - 0, - ), - "Xrayphlum_core_excision": ( - "XRayPhotonLuminosityCoreExcision", - 3, - np.float64, - "1/snap_time", - "Total observer-frame Xray photon luminosity in three bands. Excludes gas in the inner {core_excision}", - "general", - "DMantissa9", - False, - [ - "PartType0/XrayPhotonLuminosities", - "PartType0/Temperatures", - "PartType0/Coordinates", - ], - True, - None, - ), - "Xrayphlum_restframe_core_excision": ( - "XRayPhotonLuminosityInRestframeCoreExcision", - 3, - np.float64, - "1/snap_time", - "Total rest-frame Xray photon luminosity in three bands. Excludes gas in the inner {core_excision}", - "general", - "DMantissa9", - False, - [ - "PartType0/XrayPhotonLuminositiesRestframe", - "PartType0/ElementMassFractions", - "PartType0/Coordinates", - ], - True, - 0, - ), - "Xrayphlum_no_agn_core_excision": ( - "XRayPhotonLuminosityWithoutRecentAGNHeatingCoreExcision", - 3, - np.float64, - "1/snap_time", - "Total observer-frame Xray photon luminosity in three bands. Exclude gas that was recently heated by AGN. Excludes gas in the inner {core_excision}", - "general", - "DMantissa9", - False, - [ - "PartType0/XrayPhotonLuminosities", - "PartType0/Temperatures", - "PartType0/LastAGNFeedbackScaleFactors", - "PartType0/Coordinates", - ], - True, - None, - ), - "Xrayphlum_restframe_no_agn_core_excision": ( - "XRayPhotonLuminosityInRestframeWithoutRecentAGNHeatingCoreExcision", - 3, - np.float64, - "1/snap_time", - "Total rest-frame Xray photon luminosity in three bands. Exclude gas that was recently heated by AGN. Excludes gas in the inner {core_excision}", - "general", - "DMantissa9", - False, - [ - "PartType0/XrayPhotonLuminositiesRestframe", - "PartType0/Temperatures", - "PartType0/LastAGNFeedbackScaleFactors", - "PartType0/Coordinates", - ], - True, - 0, - ), - "SpectroscopicLikeTemperature": ( - "SpectroscopicLikeTemperature", - 1, - np.float32, - "snap_temperature", - "Spectroscopic-like gas temperature.", - "general", - "FMantissa9", - False, - ["PartType0/Temperatures", "PartType0/Densities", "PartType0/Masses"], - True, - 0, - ), - "SpectroscopicLikeTemperature_no_agn": ( - "SpectroscopicLikeTemperatureWithoutRecentAGNHeating", - 1, - np.float32, - "snap_temperature", - "Spectroscopic-like gas temperature. Exclude gas that was recently heated by AGN", - "general", - "FMantissa9", - False, - [ - "PartType0/Temperatures", - "PartType0/Densities", - "PartType0/LastAGNFeedbackScaleFactors", - "PartType0/Masses", - ], - True, - 0, - ), - "SpectroscopicLikeTemperature_core_excision": ( - "SpectroscopicLikeTemperatureCoreExcision", - 1, - np.float32, - "snap_temperature", - "Spectroscopic-like gas temperature. Excludes gas in the inner {core_excision}", - "general", - "FMantissa9", - False, - [ - "PartType0/Temperatures", - "PartType0/Densities", - "PartType0/LastAGNFeedbackScaleFactors", - "PartType0/Masses", - "PartType0/Coordinates", - ], - True, - 0, - ), - "SpectroscopicLikeTemperature_no_agn_core_excision": ( - "SpectroscopicLikeTemperatureWithoutRecentAGNHeatingCoreExcision", - 1, - np.float32, - "snap_temperature", - "Spectroscopic-like gas temperature. Exclude gas that was recently heated by AGN. Excludes gas in the inner {core_excision}", - "general", - "FMantissa9", - False, - [ - "PartType0/Temperatures", - "PartType0/Densities", - "PartType0/LastAGNFeedbackScaleFactors", - "PartType0/Masses", - "PartType0/Coordinates", - "PartType0/LastAGNFeedbackScaleFactors", - ], - True, - 0, - ), - "concentration_unsoft": ( - "ConcentrationUnsoftened", - 1, - np.float32, - "dimensionless", - "Halo concentration assuming an NFW profile. No particle softening.", - "basic", - "FMantissa9", - True, - [ - "PartType0/Coordinates", - "PartType0/Masses", - "PartType1/Coordinates", - "PartType1/Masses", - "PartType4/Coordinates", - "PartType4/Masses", - "PartType5/Coordinates", - "PartType5/DynamicalMasses", - ], - True, - 0, - ), - "concentration_soft": ( - "Concentration", - 1, - np.float32, - "dimensionless", - "Halo concentration assuming an NFW profile. Minimum particle radius set to softening length", - "basic", - "FMantissa9", - True, - [ - "PartType0/Coordinates", - "PartType0/Masses", - "PartType1/Coordinates", - "PartType1/Masses", - "PartType4/Coordinates", - "PartType4/Masses", - "PartType5/Coordinates", - "PartType5/DynamicalMasses", - ], - True, - 0, - ), - "concentration_dmo_unsoft": ( - "DarkMatterConcentrationUnsoftened", - 1, - np.float32, - "dimensionless", - "Concentration of dark matter particles assuming an NFW profile. No particle softening", - "basic", - "FMantissa9", - False, - ["PartType1/Coordinates", "PartType1/Masses"], - True, - 0, - ), - "concentration_dmo_soft": ( - "DarkMatterConcentration", - 1, - np.float32, - "dimensionless", - "Concentration of dark matter particles assuming an NFW profile. Minimum particle radius set to softening length", - "basic", - "FMantissa9", - False, - ["PartType1/Coordinates", "PartType1/Masses"], - True, - 0, - ), - "DarkMatterMassFlowRate": ( - "DarkMatterMassFlowRate", - 6, - np.float32, - "snap_mass / snap_time", - "Mass flow rate of dark matter particles through spherical shells. Contains 6 entries: inflow rate at 0.1R, 0.3R, R, outflow rate at 0.1R, 0.3R, R.", - "general", - "FMantissa9", - True, - ["PartType1/Coordinates", "PartType1/Masses", "PartType1/Velocities"], - True, - 0, - ), - "ColdGasMassFlowRate": ( - "ColdGasMassFlowRate", - 9, - np.float32, - "snap_mass / snap_time", - "Mass flow rate of cold gas particles (log T < 3) through spherical shells. Contains 9 entries: inflow rate at 0.1R, 0.3R, R, outflow rate at 0.1R, 0.3R, R, fast outflow rate at 0.1R, 0.3R, R.", - "general", - "FMantissa9", - False, - [ - "PartType0/Coordinates", - "PartType0/Masses", - "PartType0/Velocities", - "PartType0/Temperatures", - ], - True, - 0, - ), - "CoolGasMassFlowRate": ( - "CoolGasMassFlowRate", - 9, - np.float32, - "snap_mass / snap_time", - "Mass flow rate of cool gas particles (3 < log T < 5) through spherical shells. Contains 9 entries: inflow rate at 0.1R, 0.3R, R, outflow rate at 0.1R, 0.3R, R, fast outflow rate at 0.1R, 0.3R, R.", - "general", - "FMantissa9", - False, - [ - "PartType0/Coordinates", - "PartType0/Masses", - "PartType0/Velocities", - "PartType0/Temperatures", - ], - True, - 0, - ), - "WarmGasMassFlowRate": ( - "WarmGasMassFlowRate", - 9, - np.float32, - "snap_mass / snap_time", - "Mass flow rate of warm gas particles (5 < log T < 7) through spherical shells. Contains 9 entries: inflow rate at 0.1R, 0.3R, R, outflow rate at 0.1R, 0.3R, R, fast outflow rate at 0.1R, 0.3R, R.", - "general", - "FMantissa9", - False, - [ - "PartType0/Coordinates", - "PartType0/Masses", - "PartType0/Velocities", - "PartType0/Temperatures", - ], - True, - 0, - ), - "HotGasMassFlowRate": ( - "HotGasMassFlowRate", - 9, - np.float32, - "snap_mass / snap_time", - "Mass flow rate of hot gas particles (7 < log T) through spherical shells. Contains 9 entries: inflow rate at 0.1R, 0.3R, R, outflow rate at 0.1R, 0.3R, R, fast outflow rate at 0.1R, 0.3R, R.", - "general", - "FMantissa9", - False, - [ - "PartType0/Coordinates", - "PartType0/Masses", - "PartType0/Velocities", - "PartType0/Temperatures", - ], - True, - 0, - ), - "HIMassFlowRate": ( - "HIMassFlowRate", - 6, - np.float32, - "snap_mass / snap_time", - "Mass flow rate of gas particles through spherical shells weighted by HI fraction. Contains 6 entries: inflow rate at 0.1R, 0.3R, R, outflow rate at 0.1R, 0.3R, R.", - "general", - "FMantissa9", - False, - [ - "PartType0/Coordinates", - "PartType0/Masses", - "PartType0/Velocities", - "PartType0/SpeciesFractions", - "PartType0/ElementMassFractions", - ], - True, - 0, - ), - "H2MassFlowRate": ( - "H2MassFlowRate", - 6, - np.float32, - "snap_mass / snap_time", - "Mass flow rate of gas particles through spherical shells weighted by H2 fraction. Does not include Helium. Contains 6 entries: inflow rate at 0.1R, 0.3R, R, outflow rate at 0.1R, 0.3R, R.", - "general", - "FMantissa9", - False, - [ - "PartType0/Coordinates", - "PartType0/Masses", - "PartType0/Velocities", - "PartType0/SpeciesFractions", - "PartType0/ElementMassFractions", - ], - True, - 0, - ), - "MetalMassFlowRate": ( - "MetalMassFlowRate", - 6, - np.float32, - "snap_mass / snap_time", - "Mass flow rate of gas particles through spherical shells weighted by metal fraction. Contains 6 entries: inflow rate at 0.1R, 0.3R, R, outflow rate at 0.1R, 0.3R, R.", - "general", - "FMantissa9", - False, - [ - "PartType0/Coordinates", - "PartType0/Masses", - "PartType0/Velocities", - "PartType0/MetalMassFractions", - ], - True, - 0, - ), - "StellarMassFlowRate": ( - "StellarMassFlowRate", - 6, - np.float32, - "snap_mass / snap_time", - "Mass flow rate of star particles through spherical shells. Contains 6 entries: inflow rate at 0.1R, 0.3R, R, outflow rate at 0.1R, 0.3R, R.", - "general", - "FMantissa9", - False, - ["PartType4/Coordinates", "PartType4/Masses", "PartType4/Velocities"], - True, - 0, - ), - "ColdGasEnergyFlowRate": ( - "ColdGasEnergyFlowRate", - 9, - np.float32, - "snap_mass*snap_length**2/snap_time**3", - "Energy flow rate of cold gas particles (log T < 3) through spherical shells. Contains 9 entries: inflow rate at 0.1R, 0.3R, R, outflow rate at 0.1R, 0.3R, R, fast outflow rate at 0.1R, 0.3R, R.", - "general", - "FMantissa9", - False, - [ - "PartType0/Coordinates", - "PartType0/Masses", - "PartType0/Velocities", - "PartType0/InternalEnergies", - ], - True, - 0, - ), - "CoolGasEnergyFlowRate": ( - "CoolGasEnergyFlowRate", - 9, - np.float32, - "snap_mass*snap_length**2/snap_time**3", - "Energy flow rate of cool gas particles (3 < log T < 5) through spherical shells. Contains 9 entries: inflow rate at 0.1R, 0.3R, R, outflow rate at 0.1R, 0.3R, R, fast outflow rate at 0.1R, 0.3R, R.", - "general", - "FMantissa9", - False, - [ - "PartType0/Coordinates", - "PartType0/Masses", - "PartType0/Velocities", - "PartType0/InternalEnergies", - ], - True, - 0, - ), - "WarmGasEnergyFlowRate": ( - "WarmGasEnergyFlowRate", - 9, - np.float32, - "snap_mass*snap_length**2/snap_time**3", - "Energy flow rate of warm gas particles (5 < log T < 7) through spherical shells. Contains 9 entries: inflow rate at 0.1R, 0.3R, R, outflow rate at 0.1R, 0.3R, R, fast outflow rate at 0.1R, 0.3R, R.", - "general", - "FMantissa9", - False, - [ - "PartType0/Coordinates", - "PartType0/Masses", - "PartType0/Velocities", - "PartType0/InternalEnergies", - ], - True, - 0, - ), - "HotGasEnergyFlowRate": ( - "HotGasEnergyFlowRate", - 9, - np.float32, - "snap_mass*snap_length**2/snap_time**3", - "Energy flow rate of hot gas particles (7 < log T) through spherical shells. Contains 9 entries: inflow rate at 0.1R, 0.3R, R, outflow rate at 0.1R, 0.3R, R, fast outflow rate at 0.1R, 0.3R, R.", - "general", - "FMantissa9", - False, - [ - "PartType0/Coordinates", - "PartType0/Masses", - "PartType0/Velocities", - "PartType0/InternalEnergies", - ], - True, - 0, - ), - "ColdGasMomentumFlowRate": ( - "ColdGasMomentumFlowRate", - 9, - np.float32, - "snap_mass*snap_length/snap_time**2", - "Momentum flow rate of cold gas particles (log T < 3) through spherical shells. Contains 9 entries: inflow rate at 0.1R, 0.3R, R, outflow rate at 0.1R, 0.3R, R, fast outflow rate at 0.1R, 0.3R, R.", - "general", - "FMantissa9", - False, - [ - "PartType0/Coordinates", - "PartType0/Masses", - "PartType0/Velocities", - "PartType0/InternalEnergies", - ], - True, - 0, - ), - "CoolGasMomentumFlowRate": ( - "CoolGasMomentumFlowRate", - 9, - np.float32, - "snap_mass*snap_length/snap_time**2", - "Momentum flow rate of cool gas particles (3 < log T < 5) through spherical shells. Contains 9 entries: inflow rate at 0.1R, 0.3R, R, outflow rate at 0.1R, 0.3R, R, fast outflow rate at 0.1R, 0.3R, R.", - "general", - "FMantissa9", - False, - [ - "PartType0/Coordinates", - "PartType0/Masses", - "PartType0/Velocities", - "PartType0/InternalEnergies", - ], - True, - 0, - ), - "WarmGasMomentumFlowRate": ( - "WarmGasMomentumFlowRate", - 9, - np.float32, - "snap_mass*snap_length/snap_time**2", - "Momentum flow rate of warm gas particles (5 < log T < 7) through spherical shells. Contains 9 entries: inflow rate at 0.1R, 0.3R, R, outflow rate at 0.1R, 0.3R, R, fast outflow rate at 0.1R, 0.3R, R.", - "general", - "FMantissa9", - False, - [ - "PartType0/Coordinates", - "PartType0/Masses", - "PartType0/Velocities", - "PartType0/InternalEnergies", - ], - True, - 0, - ), - "HotGasMomentumFlowRate": ( - "HotGasMomentumFlowRate", - 9, - np.float32, - "snap_mass*snap_length/snap_time**2", - "Momentum flow rate of hot gas particles (7 < log T) through spherical shells. Contains 9 entries: inflow rate at 0.1R, 0.3R, R, outflow rate at 0.1R, 0.3R, R, fast outflow rate at 0.1R, 0.3R, R.", - "general", - "FMantissa9", - False, - [ - "PartType0/Coordinates", - "PartType0/Masses", - "PartType0/Velocities", - "PartType0/InternalEnergies", - ], - True, - 0, - ), - "com": ( - "CentreOfMass", - 3, - np.float64, - "snap_length", - "Centre of mass.", - "basic", - "DScale6", - True, - [ - "PartType0/Coordinates", - "PartType0/Masses", - "PartType1/Coordinates", - "PartType1/Masses", - "PartType4/Coordinates", - "PartType4/Masses", - "PartType5/Coordinates", - "PartType5/DynamicalMasses", - ], - False, - 1, - ), - "com_gas": ( - "GasCentreOfMass", - 3, - np.float64, - "snap_length", - "Centre of mass of gas.", - "gas", - "DScale6", - False, - ["PartType0/Coordinates", "PartType0/Masses"], - False, - 1, - ), - "com_star": ( - "StellarCentreOfMass", - 3, - np.float64, - "snap_length", - "Centre of mass of stars.", - "star", - "DScale6", - False, - ["PartType4/Coordinates", "PartType4/Masses"], - False, - 1, - ), - "compY": ( - "ComptonY", - 1, - np.float64, - "snap_length**2", - "Total Compton y parameter.", - "general", - "DMantissa9", - False, - ["PartType0/ComptonYParameters"], - True, - 0, - ), - "compY_no_agn": ( - "ComptonYWithoutRecentAGNHeating", - 1, - np.float64, - "snap_length**2", - "Total Compton y parameter. Excludes gas that was recently heated by AGN.", - "general", - "DMantissa9", - False, - [ - "PartType0/ComptonYParameters", - "PartType0/LastAGNFeedbackScaleFactors", - "PartType0/Temperatures", - ], - True, - 0, - ), - "gasFefrac": ( - "GasMassFractionInIron", - 1, - np.float32, - "dimensionless", - "Total gas mass fraction in iron.", - "general", - "FMantissa9", - False, - ["PartType0/Masses", "PartType0/ElementMassFractions"], - True, - 0, - ), - "gasFefrac_SF": ( - "StarFormingGasMassFractionInIron", - 1, - np.float32, - "dimensionless", - "Total gas mass fraction in iron for gas that is star-forming.", - "general", - "FMantissa9", - False, - [ - "PartType0/Masses", - "PartType0/ElementMassFractions", - "PartType0/StarFormationRates", - ], - True, - 0, - ), - "gasOfrac": ( - "GasMassFractionInOxygen", - 1, - np.float32, - "dimensionless", - "Total gas mass in oxygen.", - "general", - "FMantissa9", - False, - ["PartType0/Masses", "PartType0/ElementMassFractions"], - True, - 0, - ), - "gasOfrac_SF": ( - "StarFormingGasMassFractionInOxygen", - 1, - np.float32, - "dimensionless", - "Total gas mass fraction in oxygen for gas that is star-forming.", - "general", - "FMantissa9", - False, - [ - "PartType0/Masses", - "PartType0/ElementMassFractions", - "PartType0/StarFormationRates", - ], - True, - 0, - ), - "gasmetalfrac": ( - "GasMassFractionInMetals", - 1, - np.float32, - "dimensionless", - "Total gas mass fraction in metals.", - "basic", - "FMantissa9", - False, - ["PartType0/Masses", "PartType0/MetalMassFractions"], - True, - 0, - ), - "gasmetalfrac_SF": ( - "StarFormingGasMassFractionInMetals", - 1, - np.float32, - "dimensionless", - "Total gas mass fraction in metals for gas that is star-forming.", - "basic", - "FMantissa9", - False, - [ - "PartType0/Masses", - "PartType0/MetalMassFractions", - "PartType0/StarFormationRates", - ], - True, - 0, - ), - "kappa_corot_baryons": ( - "KappaCorotBaryons", - 1, - np.float32, - "dimensionless", - "Kappa-corot for baryons (gas and stars), relative to the centre of potential and the centre of mass velocity of the baryons.", - "baryon", - "FMantissa9", - False, - [ - "PartType0/Coordinates", - "PartType0/Masses", - "PartType0/Velocities", - "PartType4/Coordinates", - "PartType4/Masses", - "PartType4/Velocities", - ], - True, - 0, - ), - "kappa_corot_gas": ( - "KappaCorotGas", - 1, - np.float32, - "dimensionless", - "Kappa-corot for gas, relative to the centre of potential and the centre of mass velocity of the gas.", - "gas", - "FMantissa9", - False, - ["PartType0/Coordinates", "PartType0/Masses", "PartType0/Velocities"], - True, - 0, - ), - "kappa_corot_star": ( - "KappaCorotStars", - 1, - np.float32, - "dimensionless", - "Kappa-corot for stars, relative to the centre of potential and the centre of mass velocity of the stars.", - "star", - "FMantissa9", - False, - ["PartType4/Coordinates", "PartType4/Masses", "PartType4/Velocities"], - True, - 0, - ), - "proj_veldisp_dm": ( - "DarkMatterProjectedVelocityDispersion", - 1, - np.float32, - "snap_length/snap_time", - "Mass-weighted velocity dispersion of the DM along the projection axis, relative to the DM centre of mass velocity.", - "dm", - "FMantissa9", - True, - ["PartType1/Velocities"], - True, - 1, - ), - "proj_veldisp_gas": ( - "GasProjectedVelocityDispersion", - 1, - np.float32, - "snap_length/snap_time", - "Mass-weighted velocity dispersion of the gas along the projection axis, relative to the gas centre of mass velocity.", - "gas", - "FMantissa9", - False, - ["PartType0/Velocities"], - True, - 1, - ), - "proj_veldisp_star": ( - "StellarProjectedVelocityDispersion", - 1, - np.float32, - "snap_length/snap_time", - "Mass-weighted velocity dispersion of the stars along the projection axis, relative to the stellar centre of mass velocity.", - "star", - "FMantissa9", - False, - ["PartType4/Velocities"], - True, - 1, - ), - "r": ( - "SORadius", - 1, - np.float32, - "snap_length", - "Radius of a sphere {label}", - "basic", - "FMantissa9", - True, - [ - "PartType0/Coordinates", - "PartType0/Masses", - "PartType1/Coordinates", - "PartType1/Masses", - "PartType4/Coordinates", - "PartType4/Masses", - "PartType5/Coordinates", - "PartType5/DynamicalMasses", - "PartType6/Coordinates", - "PartType6/Masses", - "PartType6/Weights", - ], - False, - 1, - ), - "spin_parameter": ( - "SpinParameter", - 1, - np.float32, - "dimensionless", - "Bullock et al. (2001) spin parameter.", - "general", - "FMantissa9", - True, - [ - "PartType0/Coordinates", - "PartType0/Masses", - "PartType0/Velocities", - "PartType1/Coordinates", - "PartType1/Masses", - "PartType1/Velocities", - "PartType4/Coordinates", - "PartType4/Masses", - "PartType4/Velocities", - "PartType5/Coordinates", - "PartType5/DynamicalMasses", - "PartType5/Velocities", - ], - True, - 0, - ), - "starFefrac": ( - "StellarMassFractionInIron", - 1, - np.float32, - "dimensionless", - "Total stellar mass fraction in iron.", - "star", - "FMantissa9", - False, - ["PartType4/Masses", "PartType4/ElementMassFractions"], - True, - 0, - ), - "starMgfrac": ( - "StellarMassFractionInMagnesium", - 1, - np.float32, - "dimensionless", - "Total stellar mass fraction in magnesium.", - "star", - "FMantissa9", - False, - ["PartType4/Masses", "PartType4/ElementMassFractions"], - True, - 0, - ), - "starOfrac": ( - "StellarMassFractionInOxygen", - 1, - np.float32, - "dimensionless", - "Total stellar mass fraction in oxygen.", - "star", - "FMantissa9", - False, - ["PartType4/Masses", "PartType4/ElementMassFractions"], - True, - 0, - ), - "starmetalfrac": ( - "StellarMassFractionInMetals", - 1, - np.float32, - "dimensionless", - "Total stellar mass fraction in metals.", - "basic", - "FMantissa9", - False, - ["PartType4/Masses", "PartType4/MetalMassFractions"], - True, - 0, - ), - "stellar_age_lw": ( - "LuminosityWeightedMeanStellarAge", - 1, - np.float32, - "snap_time", - "Luminosity weighted mean stellar age. The weight is the r band luminosity.", - "star", - "FMantissa9", - False, - ["PartType4/Luminosities", "PartType4/BirthScaleFactors"], - True, - None, - ), - "stellar_age_mw": ( - "MassWeightedMeanStellarAge", - 1, - np.float32, - "snap_time", - "Mass weighted mean stellar age.", - "star", - "FMantissa9", - False, - ["PartType4/Masses", "PartType4/BirthScaleFactors"], - True, - None, - ), - "vcom": ( - "CentreOfMassVelocity", - 3, - np.float32, - "snap_length/snap_time", - "Centre of mass velocity.", - "basic", - "DScale1", - True, - [ - "PartType0/Masses", - "PartType0/Velocities", - "PartType1/Masses", - "PartType1/Velocities", - "PartType4/Masses", - "PartType4/Velocities", - "PartType5/DynamicalMasses", - "PartType5/Velocities", - ], - False, - 1, - ), - "vcom_gas": ( - "GasCentreOfMassVelocity", - 3, - np.float32, - "snap_length/snap_time", - "Centre of mass velocity of gas.", - "gas", - "DScale1", - False, - ["PartType0/Masses", "PartType0/Velocities"], - False, - 1, - ), - "vcom_star": ( - "StellarCentreOfMassVelocity", - 3, - np.float32, - "snap_length/snap_time", - "Centre of mass velocity of stars.", - "star", - "DScale1", - False, - ["PartType4/Masses", "PartType4/Velocities"], - False, - 1, - ), - "veldisp_matrix_dm": ( - "DarkMatterVelocityDispersionMatrix", - 6, - np.float32, - "snap_length**2/snap_time**2", - "Mass-weighted velocity dispersion of the dark matter. Measured relative to the DM centre of mass velocity. The order of the components of the dispersion tensor is XX YY ZZ XY XZ YZ.", - "dm", - "FMantissa9", - True, - ["PartType1/Masses", "PartType1/Velocities"], - True, - 2, - ), - "veldisp_matrix_gas": ( - "GasVelocityDispersionMatrix", - 6, - np.float32, - "snap_length**2/snap_time**2", - "Mass-weighted velocity dispersion of the gas. Measured relative to the gas centre of mass velocity. The order of the components of the dispersion tensor is XX YY ZZ XY XZ YZ.", - "gas", - "FMantissa9", - False, - ["PartType0/Masses", "PartType0/Velocities"], - True, - 2, - ), - "veldisp_matrix_star": ( - "StellarVelocityDispersionMatrix", - 6, - np.float32, - "snap_length**2/snap_time**2", - "Mass-weighted velocity dispersion of the stars. Measured relative to the stellar centre of mass velocity. The order of the components of the dispersion tensor is XX YY ZZ XY XZ YZ.", - "star", - "FMantissa9", - False, - ["PartType4/Masses", "PartType4/Velocities"], - True, - 2, - ), - "LinearMassWeightedOxygenOverHydrogenOfGas": ( - "LinearMassWeightedOxygenOverHydrogenOfGas", - 1, - np.float32, - "dimensionless", - "Linear sum of the oxygen over hydrogen ratio of gas, multiplied with the gas mass.", - "gas", - "FMantissa9", - False, - [ - "PartType0/Masses", - "PartType0/ElementMassFractions", - "PartType0/Temperatures", - "PartType0/Densities", - ], - True, - 0, - ), - "LinearMassWeightedNitrogenOverOxygenOfGas": ( - "LinearMassWeightedNitrogenOverOxygenOfGas", - 1, - np.float32, - "dimensionless", - "Linear sum of the total nitrogen over oxygen ratio of gas, multiplied with the gas mass.", - "gas", - "FMantissa9", - False, - [ - "PartType0/Masses", - "PartType0/ElementMassFractions", - "PartType0/Temperatures", - "PartType0/Densities", - ], - True, - 0, - ), - "LinearMassWeightedCarbonOverOxygenOfGas": ( - "LinearMassWeightedCarbonOverOxygenOfGas", - 1, - np.float32, - "dimensionless", - "Linear sum of the total carbon over oxygen ratio of gas, multiplied with the gas mass.", - "gas", - "FMantissa9", - False, - [ - "PartType0/Masses", - "PartType0/ElementMassFractions", - "PartType0/Temperatures", - "PartType0/Densities", - ], - True, - 0, - ), - "LinearMassWeightedDiffuseNitrogenOverOxygenOfGas": ( - "LinearMassWeightedDiffuseNitrogenOverOxygenOfGas", - 1, - np.float32, - "dimensionless", - "Linear sum of the diffuse nitrogen over oxygen ratio of gas, multiplied with the gas mass.", - "gas", - "FMantissa9", - False, - [ - "PartType0/Masses", - "PartType0/ElementMassFractionsDiffuse", - "PartType0/Temperatures", - "PartType0/Densities", - ], - True, - 0, - ), - "LinearMassWeightedDiffuseCarbonOverOxygenOfGas": ( - "LinearMassWeightedDiffuseCarbonOverOxygenOfGas", - 1, - np.float32, - "dimensionless", - "Linear sum of the diffuse carbon over oxygen ratio of gas, multiplied with the gas mass.", - "gas", - "FMantissa9", - False, - [ - "PartType0/Masses", - "PartType0/ElementMassFractionsDiffuse", - "PartType0/Temperatures", - "PartType0/Densities", - ], - True, - 0, - ), - "LinearMassWeightedDiffuseOxygenOverHydrogenOfGas": ( - "LinearMassWeightedDiffuseOxygenOverHydrogenOfGas", - 1, - np.float32, - "dimensionless", - "Linear sum of the diffuse oxygen over hydrogen ratio of gas, multiplied with the gas mass.", - "gas", - "FMantissa9", - False, - [ - "PartType0/Masses", - "PartType0/ElementMassFractionsDiffuse", - "PartType0/Temperatures", - "PartType0/Densities", - ], - True, - 0, - ), - "LogarithmicMassWeightedDiffuseNitrogenOverOxygenOfGasLowLimit": ( - "LogarithmicMassWeightedDiffuseNitrogenOverOxygenOfGasLowLimit", - 1, - np.float32, - "dimensionless", - "Logarithmic sum of the diffuse nitrogen over oxygen ratio of gas, multiplied with the gas mass. Imposes a lower limit of 1.e-4 times solar N/O.", - "gas", - "FMantissa9", - False, - [ - "PartType0/Masses", - "PartType0/ElementMassFractionsDiffuse", - "PartType0/Temperatures", - "PartType0/Densities", - ], - True, - 0, - ), - "LogarithmicMassWeightedDiffuseNitrogenOverOxygenOfGasHighLimit": ( - "LogarithmicMassWeightedDiffuseNitrogenOverOxygenOfGasHighLimit", - 1, - np.float32, - "dimensionless", - "Logarithmic sum of the diffuse nitrogen over oxygen ratio of gas, multiplied with the gas mass. Imposes a lower limit of 1.e-3 times solar N/O.", - "gas", - "FMantissa9", - False, - [ - "PartType0/Masses", - "PartType0/ElementMassFractionsDiffuse", - "PartType0/Temperatures", - "PartType0/Densities", - ], - True, - 0, - ), - "LogarithmicMassWeightedDiffuseCarbonOverOxygenOfGasLowLimit": ( - "LogarithmicMassWeightedDiffuseCarbonOverOxygenOfGasLowLimit", - 1, - np.float32, - "dimensionless", - "Logarithmic sum of the diffuse carbon over oxygen ratio of gas, multiplied with the gas mass. Imposes a lower limit of 1.e-4 times solar C/O.", - "gas", - "FMantissa9", - False, - [ - "PartType0/Masses", - "PartType0/ElementMassFractionsDiffuse", - "PartType0/Temperatures", - "PartType0/Densities", - ], - True, - 0, - ), - "LogarithmicMassWeightedDiffuseCarbonOverOxygenOfGasHighLimit": ( - "LogarithmicMassWeightedDiffuseCarbonOverOxygenOfGasHighLimit", - 1, - np.float32, - "dimensionless", - "Logarithmic sum of the diffuse carbon over oxygen ratio of gas, multiplied with the gas mass. Imposes a lower limit of 1.e-3 times solar C/O.", - "gas", - "FMantissa9", - False, - [ - "PartType0/Masses", - "PartType0/ElementMassFractionsDiffuse", - "PartType0/Temperatures", - "PartType0/Densities", - ], - True, - 0, - ), - "LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfGasLowLimit": ( - "LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfGasLowLimit", - 1, - np.float32, - "dimensionless", - "Logarithmic sum of the diffuse oxygen over hydrogen ratio of gas, multiplied with the gas mass. Imposes a lower limit of 1.e-4 times solar O/H.", - "gas", - "FMantissa9", - False, - [ - "PartType0/Masses", - "PartType0/ElementMassFractionsDiffuse", - "PartType0/Temperatures", - "PartType0/Densities", - ], - True, - 0, - ), - "LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfGasHighLimit": ( - "LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfGasHighLimit", - 1, - np.float32, - "dimensionless", - "Logarithmic sum of the diffuse oxygen over hydrogen ratio of gas, multiplied with the gas mass. Imposes a lower limit of 1.e-3 times solar O/H.", - "gas", - "FMantissa9", - False, - [ - "PartType0/Masses", - "PartType0/ElementMassFractionsDiffuse", - "PartType0/Temperatures", - "PartType0/Densities", - ], - True, - 0, - ), - "LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfAtomicGasLowLimit": ( - "LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfAtomicGasLowLimit", - 1, - np.float32, - "dimensionless", - "Logarithmic sum of the diffuse oxygen over hydrogen ratio of atomic gas, multiplied with the gas mass. Imposes a lower limit of 1.e-4 times solar O/H.", - "gas", - "FMantissa9", - False, - [ - "PartType0/Masses", - "PartType0/ElementMassFractionsDiffuse", - "PartType0/ElementMassFractions", - "PartType0/SpeciesFractions", - "PartType0/Temperatures", - "PartType0/Densities", - ], - True, - 0, - ), - "LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfAtomicGasHighLimit": ( - "LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfAtomicGasHighLimit", - 1, - np.float32, - "dimensionless", - "Logarithmic sum of the diffuse oxygen over hydrogen ratio of atomic gas, multiplied with the gas mass. Imposes a lower limit of 1.e-3 times solar O/H.", - "gas", - "FMantissa9", - False, - [ - "PartType0/Masses", - "PartType0/ElementMassFractionsDiffuse", - "PartType0/ElementMassFractions", - "PartType0/SpeciesFractions", - "PartType0/Temperatures", - "PartType0/Densities", - ], - True, - 0, - ), - "LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfMolecularGasLowLimit": ( - "LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfMolecularGasLowLimit", - 1, - np.float32, - "dimensionless", - "Logarithmic sum of the diffuse oxygen over hydrogen ratio of molecular gas, multiplied with the gas mass. Imposes a lower limit of 1.e-4 times solar O/H.", - "gas", - "FMantissa9", - False, - [ - "PartType0/Masses", - "PartType0/ElementMassFractionsDiffuse", - "PartType0/ElementMassFractions", - "PartType0/SpeciesFractions", - "PartType0/Temperatures", - "PartType0/Densities", - ], - True, - 0, - ), - "LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfMolecularGasHighLimit": ( - "LogarithmicMassWeightedDiffuseOxygenOverHydrogenOfMolecularGasHighLimit", - 1, - np.float32, - "dimensionless", - "Logarithmic sum of the diffuse oxygen over hydrogen ratio of molecular gas, multiplied with the gas mass. Imposes a lower limit of 1.e-3 times solar O/H.", - "gas", - "FMantissa9", - False, - [ - "PartType0/Masses", - "PartType0/ElementMassFractionsDiffuse", - "PartType0/ElementMassFractions", - "PartType0/SpeciesFractions", - "PartType0/Temperatures", - "PartType0/Densities", - ], - True, - 0, - ), - "LinearMassWeightedIronOverHydrogenOfStars": ( - "LinearMassWeightedIronOverHydrogenOfStars", - 1, - np.float32, - "dimensionless", - "Linear sum of the iron over hydrogen ratio of stars, multiplied with the stellar mass.", - "star", - "FMantissa9", - False, - ["PartType4/Masses", "PartType4/ElementMassFractions"], - True, - 0, - ), - "LogarithmicMassWeightedIronOverHydrogenOfStarsLowLimit": ( - "LogarithmicMassWeightedIronOverHydrogenOfStarsLowLimit", - 1, - np.float32, - "dimensionless", - "Logarithmic sum of the iron over hydrogen ratio of stars, multiplied with the stellar mass. Imposes a lower limit of 1.e-4 times solar Fe/H.", - "star", - "FMantissa9", - False, - ["PartType4/Masses", "PartType4/ElementMassFractions"], - True, - 0, - ), - "LogarithmicMassWeightedIronOverHydrogenOfStarsHighLimit": ( - "LogarithmicMassWeightedIronOverHydrogenOfStarsHighLimit", - 1, - np.float32, - "dimensionless", - "Logarithmic sum of the iron over hydrogen ratio of stars, multiplied with the stellar mass. Imposes a lower limit of 1.e-3 times solar Fe/H.", - "star", - "FMantissa9", - False, - ["PartType4/Masses", "PartType4/ElementMassFractions"], - True, - 0, - ), - "LinearMassWeightedMagnesiumOverHydrogenOfStars": ( - "LinearMassWeightedMagnesiumOverHydrogenOfStars", - 1, - np.float32, - "dimensionless", - "Linear sum of the magnesium over hydrogen ratio of stars, multiplied with the stellar mass.", - "star", - "FMantissa9", - False, - ["PartType4/Masses", "PartType4/ElementMassFractions"], - True, - 0, - ), - "LogarithmicMassWeightedMagnesiumOverHydrogenOfStarsLowLimit": ( - "LogarithmicMassWeightedMagnesiumOverHydrogenOfStarsLowLimit", - 1, - np.float32, - "dimensionless", - "Logarithmic sum of the magnesium over hydrogen ratio of stars, multiplied with the stellar mass. Imposes a lower limit of 1.e-4 times solar Fe/H.", - "star", - "FMantissa9", - False, - ["PartType4/Masses", "PartType4/ElementMassFractions"], - True, - 0, - ), - "LogarithmicMassWeightedMagnesiumOverHydrogenOfStarsHighLimit": ( - "LogarithmicMassWeightedMagnesiumOverHydrogenOfStarsHighLimit", - 1, - np.float32, - "dimensionless", - "Logarithmic sum of the magnesium over hydrogen ratio of stars, multiplied with the stellar mass. Imposes a lower limit of 1.e-3 times solar Fe/H.", - "star", - "FMantissa9", - False, - ["PartType4/Masses", "PartType4/ElementMassFractions"], - True, - 0, - ), - "GasMassInColdDenseDiffuseMetals": ( - "GasMassInColdDenseDiffuseMetals", - 1, - np.float32, - "snap_mass", - "Sum of the diffuse metal mass in cold, dense gas.", - "gas", - "FMantissa9", - False, - [ - "PartType0/Masses", - "PartType0/MetalMassFractions", - "PartType0/DustMassFractions", - "PartType0/Temperatures", - "PartType0/Densities", - ], - True, - 0, - ), - "LogarithmicMassWeightedIronFromSNIaOverHydrogenOfStarsLowLimit": ( - "LogarithmicMassWeightedIronFromSNIaOverHydrogenOfStarsLowLimit", - 1, - np.float32, - "dimensionless", - "Logarithmic sum of the iron over hydrogen ratio of stars, multiplied with the stellar mass, where only iron from SNIa is included. Imposes a lower limit of 1.e-4 times solar Fe/H.", - "star", - "FMantissa9", - False, - [ - "PartType4/Masses", - "PartType4/ElementMassFractions", - "PartType4/IronMassFractionsFromSNIa", - ], - True, - 0, - ), - "LinearMassWeightedIronFromSNIaOverHydrogenOfStars": ( - "LinearMassWeightedIronFromSNIaOverHydrogenOfStars", - 1, - np.float32, - "dimensionless", - "Sum of the iron over hydrogen ratio of stars, multiplied with the stellar mass, where only iron from SNIa is included.", - "star", - "FMantissa9", - False, - [ - "PartType4/Masses", - "PartType4/ElementMassFractions", - "PartType4/IronMassFractionsFromSNIa", - ], - True, - 0, - ), - # InputHalo properties - "cofp": ( - "HaloCentre", - 3, - np.float64, - "snap_length", - "The centre of the subhalo as given by the halo finder. Used as reference for all relative positions. For VR and HBTplus this is equal to the position of the most bound particle in the subhalo.", - "Input", - "DScale6", - True, - [], - False, - 1, - ), - "index": ( - "HaloCatalogueIndex", - 1, - np.int64, - "dimensionless", - "Index of this halo in the original halo finder catalogue (first halo has index=0).", - "Input", - "None", - True, - [], - True, - None, - ), - "is_central": ( - "IsCentral", - 1, - np.int64, - "dimensionless", - "Whether the halo finder flagged the halo as central (1) or satellite (0).", - "Input", - "None", - True, - [], - True, - None, - ), - "nr_bound_part": ( - "NumberOfBoundParticles", - 1, - np.int64, - "dimensionless", - "Total number of particles bound to the subhalo.", - "Input", - "None", - True, - [], - True, - None, - ), - # Velociraptor properties - "VR/ID": ( - "ID", - 1, - np.uint64, - "dimensionless", - "ID assigned to this halo by VR.", - "VR", - "None", - True, - [], - True, - None, - ), - "VR/Parent_halo_ID": ( - "ParentHaloID", - 1, - np.int64, - "dimensionless", - "VR/ID of the direct parent of this halo. -1 for field halos.", - "VR", - "None", - True, - [], - True, - None, - ), - "VR/Structuretype": ( - "StructureType", - 1, - np.int32, - "dimensionless", - "Structure type identified by VR. Field halos are 10, higher numbers are for satellites.", - "VR", - "None", - True, - [], - True, - None, - ), - "VR/hostHaloID": ( - "HostHaloID", - 1, - np.int64, - "dimensionless", - "VR/ID of the top level parent of this halo. -1 for field halos.", - "VR", - "None", - True, - [], - True, - None, - ), - "VR/numSubStruct": ( - "NumberOfSubstructures", - 1, - np.uint64, - "dimensionless", - "Number of sub-structures within this halo.", - "VR", - "None", - True, - [], - True, - None, - ), - # HBT properties - "HBTplus/Depth": ( - "Depth", - 1, - np.uint64, - "dimensionless", - "Level of the subhalo in the merging hierarchy.", - "HBTplus", - "None", - True, - [], - True, - None, - ), - "HBTplus/HostHaloId": ( - "HostFOFId", - 1, - np.int64, - "dimensionless", - "ID of the host FOF halo of this subhalo. Hostless halos have HostFOFId == -1", - "HBTplus", - "None", - True, - [], - True, - None, - ), - "HBTplus/TrackId": ( - "TrackId", - 1, - np.uint64, - "dimensionless", - "Unique ID for this subhalo which is consistent across snapshots.", - "HBTplus", - "None", - True, - [], - True, - None, - ), - "HBTplus/SnapshotIndexOfBirth": ( - "SnapshotIndexOfBirth", - 1, - np.int64, - "dimensionless", - "Snapshot when this subhalo was formed.", - "HBTplus", - "None", - True, - [], - True, - None, - ), - "HBTplus/NestedParentTrackId": ( - "NestedParentTrackId", - 1, - np.int64, - "dimensionless", - "TrackId of the parent of this subhalo.", - "HBTplus", - "None", - True, - [], - True, - None, - ), - "HBTplus/DescendantTrackId": ( - "DescendantTrackId", - 1, - np.int64, - "dimensionless", - "TrackId of the descendant of this subhalo.", - "HBTplus", - "None", - True, - [], - True, - None, - ), - "HBTplus/LastMaxMass": ( - "LastMaxMass", - 1, - np.float32, - "snap_mass", - "Maximum mass of this subhalo across its evolutionary history", - "HBTplus", - "FMantissa9", - True, - [], - True, - 0, - ), - "HBTplus/SnapshotIndexOfLastMaxMass": ( - "SnapshotIndexOfLastMaxMass", - 1, - np.uint64, - "dimensionless", - "Latest snapshot when this subhalo had its maximum mass.", - "HBTplus", - "None", - True, - [], - True, - None, - ), - "HBTplus/LastMaxVmaxPhysical": ( - "LastMaxVmaxPhysical", - 1, - np.float32, - "snap_length/snap_time", - "Largest value of maximum circular velocity of this subhalo across its evolutionary history", - "HBTplus", - "FMantissa9", - True, - [], - True, - None, - ), - "HBTplus/SnapshotIndexOfLastMaxVmax": ( - "SnapshotIndexOfLastMaxVmax", - 1, - np.uint64, - "dimensionless", - "Latest snapshot when this subhalo had its largest maximum circular velocity.", - "HBTplus", - "None", - True, - [], - True, - None, - ), - # FOF properties - "FOF/Centres": ( - "Centres", - 3, - np.float64, - "snap_length", - "Centre of mass of the host FOF halo of this subhalo. Zero for satellite and hostless subhalos.", - "FOF", - "DScale6", - True, - [], - False, - 1, - ), - "FOF/Masses": ( - "Masses", - 1, - np.float32, - "snap_mass", - "Mass of the host FOF halo of this subhalo. Zero for satellite and hostless subhalos.", - "FOF", - "FMantissa9", - True, - [], - True, - 0, - ), - "FOF/Sizes": ( - "Sizes", - 1, - np.uint64, - "dimensionless", - "Number of particles in the host FOF halo of this subhalo. Zero for satellite and hostless subhalos.", - "FOF", - "None", - True, - [], - True, - None, - ), - # SOAP properties - "SubhaloRankByBoundMass": ( - "SubhaloRankByBoundMass", - 1, - np.int32, - "dimensionless", - "Ranking by mass of the halo within its parent field halo. Zero for the most massive halo in the field halo.", - "SOAP", - "None", - True, - [], - True, - None, - ), - "HostHaloIndex": ( - "HostHaloIndex", - 1, - np.int64, - "dimensionless", - "Index (within the SOAP arrays) of the top level parent of this subhalo. -1 for central subhalos.", - "SOAP", - "None", - True, - [], - True, - None, - ), - "IncludedInReducedSnapshot": ( - "IncludedInReducedSnapshot", - 1, - np.int32, - "dimensionless", - "Whether this halo is included in the reduced snapshot.", - "SOAP", - "None", - True, - [], - True, - None, - ), - } - - # halo properties derived from other properties by SOAP - soap_properties = [ - name for name, info in full_property_list.items() if info[5] == "SOAP" - ] - - # object member variables - properties: Dict[str, Dict] - footnotes: List[str] - - def get_footnotes(self, name: str): - """ - List all of the footnotes for a particular property. Returns an empty - string for properties that have no footnotes. - """ - footnotes = [] - for fnote in self.explanation.keys(): - names = self.explanation[fnote] - if name in names: - try: - i = self.footnotes.index(fnote) - except ValueError: - i = len(self.footnotes) - self.footnotes.append(fnote) - footnotes.append(i + 1) - if len(footnotes) > 0: - return f'$^{{{",".join([f"{i}" for i in footnotes])}}}$' - else: - return "" - - def __init__(self, parameters, snipshot_parameters): - """ - Constructor. - """ - self.properties = {} - self.footnotes = [] - self.parameters = parameters - self.snipshot_parameters = snipshot_parameters - - def add_properties(self, halo_property: HaloProperty, halo_type: str): - """ - Add all the properties calculated for a particular halo type to the - internal dictionary. - """ - # Get the property_mask dict, which says whether a property should be included - props = halo_property.property_list - base_halo_type = halo_type - if halo_type in ["ExclusiveSphereProperties", "InclusiveSphereProperties"]: - base_halo_type = "ApertureProperties" - property_mask = self.parameters.get_property_mask( - base_halo_type, [prop[1] for prop in props] - ) - snipshot_mask = self.snipshot_parameters.get_property_mask( - base_halo_type, [prop[1] for prop in props] - ) - - # Loop through all possible properties for this halo type and add them to the - # table, skipping those that we shouldn't calculate according to the parameter file - for ( - i, - ( - prop_name, - prop_outputname, - prop_shape, - prop_dtype, - prop_units, - prop_description, - prop_cat, - prop_comp, - prop_dmo, - prop_partprops, - prop_physical, - prop_a_exponent, - ), - ) in enumerate(props): - if not property_mask[prop_outputname]: - continue - - units = unyt.unyt_quantity(1, units=prop_units) - if not prop_physical: - units = units * unyt.Unit("a") ** prop_a_exponent - prop_units = units.units.latex_repr.replace( - "\\rm{km} \\cdot \\rm{kpc}", "\\rm{kpc} \\cdot \\rm{km}" - ).replace("\\frac{\\rm{km}^{2}}{\\rm{s}^{2}}", "\\rm{km}^{2} / \\rm{s}^{2}") - - prop_dtype = prop_dtype.__name__ - if prop_name in self.properties: - # run some checks - if prop_shape != self.properties[prop_name]["shape"]: - print("Shape mismatch!") - print(halo_type, prop_name, prop_shape, self.properties[prop_name]) - exit() - if prop_dtype != self.properties[prop_name]["dtype"]: - print("dtype mismatch!") - print(halo_type, prop_name, prop_dtype, self.properties[prop_name]) - exit() - if prop_units != self.properties[prop_name]["units"]: - print("Unit mismatch!") - print(halo_type, prop_name, prop_units, self.properties[prop_name]) - exit() - if prop_description != self.properties[prop_name]["description"]: - print("Description mismatch!") - print( - halo_type, - prop_name, - prop_description, - self.properties[prop_name], - ) - exit() - if prop_cat != self.properties[prop_name]["category"]: - print("Category mismatch!") - print(halo_type, prop_name, prop_cat, self.properties[prop_name]) - exit() - assert prop_outputname == self.properties[prop_name]["name"] - - if not snipshot_mask[prop_outputname]: - self.properties[prop_name]["types"].append( - "SnapshotOnly" + halo_type - ) - else: - self.properties[prop_name]["types"].append(halo_type) - else: - self.properties[prop_name] = { - "name": prop_outputname, - "shape": prop_shape, - "dtype": prop_dtype, - "units": prop_units, - "description": prop_description, - "category": prop_cat, - "compression": prop_comp, - "dmo": prop_dmo, - "raw": props[i], - } - if not snipshot_mask[prop_outputname]: - self.properties[prop_name]["types"] = ["SnapshotOnly" + halo_type] - else: - self.properties[prop_name]["types"] = [halo_type] - - def print_dictionary(self): - """ - Print the internal list of properties. Useful for regenerating the - property dictionary with additional information for each property. - - Note that his will sort the dictionary alphabetically. - """ - names = sorted(list(self.properties.keys())) - print("full_property_list = {") - for name in names: - ( - raw_name, - raw_outputname, - raw_shape, - raw_dtype, - raw_units, - raw_description, - raw_cat, - raw_comp, - raw_dmo, - raw_partprops, - ) = self.properties[name]["raw"] - raw_dtype = f"np.{raw_dtype.__name__}" - print( - f' "{raw_name}": ("{raw_outputname}", {raw_shape}, {raw_dtype}, "{raw_units}", "{raw_description}", "{raw_cat}", "{raw_comp}", {raw_dmo}, {raw_partprops}),' - ) - print("}") - - def generate_tex_files(self, output_dir: str): - """ - Outputs all the .tex files required to generate the documentation. - - The documentation consists of - - a hand-written SOAP.tex file. - - a table.tex - - a footnotes.tex file which will contain the contents of - the various hand-written footnote*.tex files - - a version and time stamp file, called timestamp.tex - - a filters.tex which contains the threshold value of each filter - - a variations.tex file which contains a table of all the halo type - variations present in the parameter file, and the filter for each - - This function regenerates the last 5 files, based on the contents of - the internal property dictionary and the parameter file passed. - """ - - # sort the properties by category and then alphabetically within each - # category - category_order = [ - "basic", - "general", - "gas", - "dm", - "star", - "baryon", - "Input", - "VR", - "HBTplus", - "FOF", - "SOAP", - ] - prop_names = sorted( - self.properties.keys(), - key=lambda key: ( - category_order.index(self.properties[key]["category"]), - self.properties[key]["name"].lower(), - ), - ) - - # generate the LaTeX header for a standalone table file - headstr = """\\documentclass{article} -\\usepackage{amsmath} -\\usepackage{amssymb} -\\usepackage{longtable} -\\usepackage{pifont} -\\usepackage{pdflscape} -\\usepackage{a4wide} -\\usepackage{multirow} -\\usepackage{xcolor} - -\\begin{document}""" - - # property table string: table header - tablestr = """\\begin{landscape} -\\begin{longtable}{p{15em}llllllllll} -Name & Shape & Type & Units & SH & ES & IS & EP & SO & Category & Compression\\\\ -\\multicolumn{11}{l}{\\rule{30pt}{0pt}Description}\\\\ -\\hline{}\\endhead{}""" - # keep track of the previous category to draw a line when a category - # is finished - prev_cat = None - for prop_name in prop_names: - prop = self.properties[prop_name] - footnotes = self.get_footnotes(prop_name) - prop_outputname = f"{prop['name']}{footnotes}" - prop_outputname = word_wrap_name(prop_outputname) - prop_shape = f'{prop["shape"]}' - prop_dtype = prop["dtype"] - prop_units = ( - f'${prop["units"]}$' if prop["units"] != "" else "dimensionless" - ) - prop_cat = prop["category"] - prop_comp = self.compression_description[prop["compression"]] - prop_description = prop["description"].format( - label="satisfying a spherical overdensity criterion.", - core_excision="excised core", - ) - - checkmark = "\\ding{51}" - xmark = "\\ding{53}" - scissor = "\\ding{36}" - prop_subhalo = checkmark if "SubhaloProperties" in prop["types"] else xmark - prop_subhalo = ( - scissor - if "SnapshotOnlySubhaloProperties" in prop["types"] - else prop_subhalo - ) - prop_exclusive = ( - checkmark if "ExclusiveSphereProperties" in prop["types"] else xmark - ) - prop_exclusive = ( - scissor - if "SnapshotOnlyExclusiveSphereProperties" in prop["types"] - else prop_exclusive - ) - prop_inclusive = ( - checkmark if "InclusiveSphereProperties" in prop["types"] else xmark - ) - prop_inclusive = ( - scissor - if "SnapshotOnlyInclusiveSphereProperties" in prop["types"] - else prop_inclusive - ) - prop_projected = ( - checkmark if "ProjectedApertureProperties" in prop["types"] else xmark - ) - prop_projected = ( - scissor - if "SnapshotOnlyProjectedApertureProperties" in prop["types"] - else prop_projected - ) - prop_SO = checkmark if "SOProperties" in prop["types"] else xmark - prop_SO = ( - scissor if "SnapshotOnlySOProperties" in prop["types"] else prop_SO - ) - table_props = [ - prop_outputname, - prop_shape, - prop_dtype, - prop_units, - prop_subhalo, - prop_exclusive, - prop_inclusive, - prop_projected, - prop_SO, - prop_cat, - prop_comp, - ] - if prop["dmo"]: - print_table_props = [f"{{\\color{{violet}}{v}}}" for v in table_props] - prop_description = f"{{\\color{{violet}}{prop_description}}}" - else: - print_table_props = list(table_props) - if prev_cat is None: - prev_cat = prop_cat - if prop_cat != prev_cat: - prev_cat = prop_cat - tablestr += "\\hline{}" - tablestr += "\\rule{0pt}{4ex}" - tablestr += " & ".join([v for v in print_table_props]) + "\\\\*\n" - tablestr += f"\\multicolumn{{11}}{{p{{15cm}}}}{{\\rule{{30pt}}{{0pt}}{prop_description}}}\\\\\n" - tablestr += """\\end{longtable} -\\end{landscape}""" - # standalone table file footer - tailstr = "\\end{document}" - - # generate the documentation files - with open(f"{output_dir}/timestamp.tex", "w") as ofile: - ofile.write(get_version_string()) - with open(f"{output_dir}/table.tex", "w") as ofile: - ofile.write(tablestr) - with open(f"{output_dir}/footnotes.tex", "w") as ofile: - for i, fnote in enumerate(self.footnotes): - with open(f"documentation/{fnote}", "r") as ifile: - fnstr = ifile.read() - fnstr = fnstr.replace("$FOOTNOTE_NUMBER$", f"{i+1}") - ofile.write(f"{fnstr}\n\n") - - # Particle limits for each filter - with open(f"{output_dir}/filters.tex", "w") as ofile: - for name, filter_info in self.parameters.parameters["filters"].items(): - value = filter_info["limit"] - ofile.write(f'\\newcommand{{\\{name.lower()}filter}}{{{value}}}\n') - - # Create table of variations of each halo type, always add BoundSubhalo - tablestr = """\\pagebreak -\\begin{adjustbox}{tabular=llcl,center} -Group name (HDF5) & Group name (swiftsimio) & Inclusive? & Filter \\\\ -\\hline -\\verb+BoundSubhalo+ & \\verb+bound_subhalo+ & \\ding{53} & - \\\\*\n""" - # Add SO apertures to table - apertures = self.parameters.parameters.get("SOProperties", {}) - for _, variation in apertures.get("variations", {}).items(): - name = "" - if "radius_multiple" in variation: - name += f'{int(variation["radius_multiple"])}xR_' - if variation["type"] == "BN98": - name += f"BN98" - else: - name += f'{variation["value"]:.0f}_{variation["type"]}' - filter = variation.get("filter", "basic") - filter = "-" if filter == "basic" else filter - tablestr += f"\\verb+SO/{name}+ & \\verb+spherical_overdensity_{name.lower()}+& \\ding{{51}} & {filter} \\\\*\n" - # Determine which ExclusiveSphere and InclusiveSphere apertures are present - variations_ES, variations_IS = {}, {} - apertures = self.parameters.parameters.get("ApertureProperties", {}) - for _, variation in apertures.get("variations", {}).items(): - if variation["inclusive"]: - variations_IS[int(variation["radius_in_kpc"])] = variation.get( - "filter", "basic" - ) - else: - variations_ES[int(variation["radius_in_kpc"])] = variation.get( - "filter", "basic" - ) - # Add ExclusiveSphere apertures to table in sorted order - for radius in sorted(variations_ES.keys()): - filter = "-" if variations_ES[radius] == "basic" else variations_ES[radius] - tablestr += f"\\verb+ExclusiveSphere/{radius}kpc+ & \\verb+exclusive_sphere_{radius}kpc+ & \\ding{{53}} & {filter} \\\\*\n" - # Add InclusiveSphere apertures to table in sorted order - for radius in sorted(variations_IS.keys()): - filter = "-" if variations_IS[radius] == "basic" else variations_IS[radius] - tablestr += f"\\verb+InclusiveSphere/{radius}kpc+ & \\verb+inclusive_sphere_{radius}kpc+ & \\ding{{51}} & {filter} \\\\*\n" - # Determine which projected apertures are present - variations_proj = {} - apertures = self.parameters.parameters.get("ProjectedApertureProperties", {}) - for _, variation in apertures.get("variations", {}).items(): - variations_proj[int(variation["radius_in_kpc"])] = variation.get( - "filter", "basic" - ) - # Add ProjectedApertures to table in sorted order - for radius in sorted(variations_proj.keys()): - filter = ( - "-" if variations_proj[radius] == "basic" else variations_proj[radius] - ) - tablestr += f"\\verb+ProjectedAperture/{radius}kpc/projP+ & \\verb+projected_aperture_{radius}kpc_projP+ & \\ding{{53}} & {filter} \\\\*\n" - # Add others groups - tablestr += f"\\verb+SOAP+ & \\verb+soap+ & - & - \\\\*\n" - tablestr += f"\\verb+InputHalos+ & \\verb+input_halos+ & - & - \\\\*\n" - halo_finder = self.parameters.parameters["HaloFinder"]["type"] - tablestr += f"\\verb+InputHalos/{halo_finder}+ & \\verb+input_halos_{halo_finder.lower()}+ & - & - \\\\*\n" - if halo_finder == "HBTplus": - tablestr += ( - f"\\verb+InputHalos/FOF+ & \\verb+input_halos_fof+ & - & - \\\\*\n" - ) - # Finish table and output to file - tablestr += "\\end{adjustbox}\n\\newpage" - with open(f"{output_dir}/variations_table.tex", "w") as ofile: - ofile.write(tablestr) - - -class DummyProperties: - """ - Dummy HaloProperty object used to include properties which are not computed - for any halo type (e.g. the 'VR' properties). - """ - - def __init__(self, halo_finder): - categories = ["SOAP", "Input", halo_finder] - # Currently FOF properties are only stored for HBT - if halo_finder == "HBTplus": - categories += ["FOF"] - self.property_list = [ - (prop, *info) - for prop, info in PropertyTable.full_property_list.items() - if info[5] in categories - ] - - -if __name__ == "__main__": - """ - Standalone script execution: - Create a PropertyTable object will all the properties from all the halo - types and print the property table or the documentation. The latter is the - default; the former can be achieved by changing the boolean in the condition - below. - - You must pass a parameter file and a snapshot to run this script - """ - - from parameter_file import ParameterFile - import sys - import h5py - - # get all the halo types - # we only import them here to avoid circular imports when this script is - # imported from another script - from aperture_properties import ExclusiveSphereProperties, InclusiveSphereProperties - from projected_aperture_properties import ProjectedApertureProperties - from SO_properties import SOProperties, CoreExcisedSOProperties - from subhalo_properties import SubhaloProperties - - # Parse parameter file - try: - parameters = ParameterFile(file_name=sys.argv[1], snipshot=False) - snipshot_parameters = ParameterFile(file_name=sys.argv[1], snipshot=True) - except IndexError: - print("A valid parameter file was not passed.") - exit() - - # Parse snapshot file to extract base units - try: - with h5py.File(sys.argv[2]) as snap: - units_cgs = { - name: float(value[0]) for name, value in snap["Units"].attrs.items() - } - unyt.define_unit( - "snap_length", - units_cgs["Unit length in cgs (U_L)"] * unyt.cm, - tex_repr="\\rm{L}", - ) - unyt.define_unit( - "snap_mass", - units_cgs["Unit mass in cgs (U_M)"] * unyt.g, - tex_repr="\\rm{M}", - ) - unyt.define_unit( - "snap_time", - units_cgs["Unit time in cgs (U_t)"] * unyt.s, - tex_repr="\\rm{t}", - ) - unyt.define_unit( - "snap_temperature", - units_cgs["Unit temperature in cgs (U_T)"] * unyt.K, - tex_repr="\\rm{T}", - ) - except IndexError: - print("No snapshot file passed.") - exit() - # Define scale factor unit - unyt.define_unit("a", 1 * unyt.dimensionless, tex_repr="\\rm{a}") - - table = PropertyTable(parameters, snipshot_parameters) - # Add standard halo definitions - table.add_properties(SubhaloProperties, "SubhaloProperties") - table.add_properties(ExclusiveSphereProperties, "ExclusiveSphereProperties") - table.add_properties(InclusiveSphereProperties, "InclusiveSphereProperties") - table.add_properties(ProjectedApertureProperties, "ProjectedApertureProperties") - # Decide whether to add core excised properties - for _, variation in parameters.parameters["SOProperties"]["variations"].items(): - if variation.get("core_excision_fraction", 0): - table.add_properties(CoreExcisedSOProperties, "SOProperties") - break - else: - table.add_properties(SOProperties, "SOProperties") - # Add InputHalos and SOAP properties - table.add_properties( - DummyProperties(parameters.parameters["HaloFinder"]["type"]), "" - ) - - table.generate_tex_files("documentation") - - # Output base units - with open("documentation/units.tex", "w") as ofile: - for name, symbol in [ - ("length", "L"), - ("mass", "M"), - ("time", "t"), - ("temperature", "T"), - ]: - value = units_cgs[f"Unit {name} in cgs (U_{symbol})"] - ofile.write(f'\\newcommand{{\\{name}baseunit}}{{{value:.4g}}}\n') - vel_kms = (1 * unyt.snap_length / unyt.snap_time).to("km/s").value - ofile.write(f'\\newcommand{{\\velbaseunit}}{{{vel_kms:.4g}}}\n') diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..50f5b9c1 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,42 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] + +[project] +name = "SOAP" +version = "0.1" +description = "MPI parallel Python code to compute properties of halos in SWIFT n-body simulations" +readme = "README.md" +requires-python = ">=3.10" +license = {text = "GNU LGPLv3+"} +authors = [ + {name = "Rob McGibbon", email = "mcgibbon@strw.leidenuniv.nl"} +] +classifiers = [ + "License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", +] +dependencies = [ + "mpi4py", + "h5py", + "numpy>=2", + "unyt>=3", + "astropy>=6", + "scipy", + "matplotlib", + "psutil", + "virgodc", +] + +[project.optional-dependencies] +test = [ + "pytest-mpi", + "numba", +] + +[project.urls] +"Homepage" = "https://github.com/SWIFTSIM/SOAP" +"Bug Tracker" = "https://github.com/SWIFTSIM/SOAP/issues" diff --git a/requirements.txt b/requirements.txt index 715bca6e..461cebf0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,11 @@ +mpi4py h5py -numpy +numpy>=2 unyt>=3 -astropy -numba +astropy>=6 scipy -pandas -pytest-mpi matplotlib psutil virgodc +numba +pytest-mpi diff --git a/scripts/COLIBRE/calculate_fof_radii.sh b/scripts/COLIBRE/calculate_fof_radii.sh new file mode 100755 index 00000000..3126ab1a --- /dev/null +++ b/scripts/COLIBRE/calculate_fof_radii.sh @@ -0,0 +1,50 @@ +#!/bin/bash -l +# +# Create FOF catalogues containing radii. Note this links to the old +# catalogues for the other FOF properties, THEY ARE NOT COPIED. +# +# Run by passing the snapshots to process as the job array, and the +# simulation as the job name. E.g. +# +# sbatch -J L0025N0188/Thermal --array=0-127%4 ./scripts/COLIBRE/calculate_fof_radii.sh +# +# N3008 runs need 8 nodes for z>1, 16 for z<1 +# N1504 runs need 1 node for z>1, 2 for z<1 +# Other runs need 1 node for all snapshots +# +#SBATCH --nodes=1 +#SBATCH --cpus-per-task=1 +#SBATCH -o ./logs/fof_radii_%a.%A.out +#SBATCH -J fof_radii +#SBATCH -p cosma8 +#SBATCH -A dp004 +#SBATCH --exclusive +#SBATCH -t 01:00:00 +# + +set -e + +module purge +module load python/3.12.4 gnu_comp/14.1.0 openmpi/5.0.3 parallel_hdf5/1.12.3 +source openmpi-5.0.3-hdf5-1.12.3-env/bin/activate + +# Which snapshot to do +snapnum=$(printf '%04d' ${SLURM_ARRAY_TASK_ID}) + +# Which simulation to do +sim="${SLURM_JOB_NAME}" + +colibre_dir="/cosma8/data/dp004/dc-mcgi1/COLIBRE/fof_radii" +snap_basename="${colibre_dir}/${sim}/snapshots/colibre_${snapnum}/colibre_${snapnum}" +fof_basename="${colibre_dir}/${sim}/fof/fof_output_${snapnum}/fof_output_${snapnum}" +output_basename="${colibre_dir}/${sim}/fof_radii/fof_output_${snapnum}/fof_output_${snapnum}" + +mpirun -- python -u misc/calculate_fof_radii.py \ + --snap-basename "${snap_basename}" \ + --fof-basename "${fof_basename}" \ + --output-basename "${output_basename}" \ + +echo "Setting files to read-only" +chmod a=r "${output_basename}"* + +echo "Job complete!" diff --git a/scripts/COLIBRE/compress_group_membership.sh b/scripts/COLIBRE/compress_group_membership.sh index f49d4ce5..ee34965f 100644 --- a/scripts/COLIBRE/compress_group_membership.sh +++ b/scripts/COLIBRE/compress_group_membership.sh @@ -15,7 +15,7 @@ # #SBATCH --ntasks=128 #SBATCH --cpus-per-task=1 -#SBATCH -o ./logs/compress_membership.%a.%j.out +#SBATCH -o ./logs/compress_membership.%a.%A.out #SBATCH -p cosma8 #SBATCH -A dp004 #SBATCH --exclusive @@ -60,7 +60,6 @@ outbase="${output_dir}/${sim}/SOAP/" # Create the output folder if it does not exist outdir="${outbase}/membership_${snapnum}" mkdir -p "${outdir}" -lfs setstripe --stripe-count=-1 --stripe-size=32M "${outdir}" # Uncompressed membership file basename input_filename="${inbase}/membership_${snapnum}/membership_${snapnum}" @@ -91,7 +90,7 @@ echo "Creating virtual snapshot" snapshot="${output_dir}/${sim}/snapshots/colibre_${snapnum}/colibre_${snapnum}.hdf5" membership="${output_filename}.{file_nr}.hdf5" virtual="${outbase}/colibre_with_SOAP_membership_${snapnum}.hdf5" -python make_virtual_snapshot.py $snapshot $membership $virtual +python compression/make_virtual_snapshot.py $snapshot $membership $virtual echo "Setting virtual file to be read-only" chmod a=r "${virtual}" diff --git a/scripts/COLIBRE/compress_halo_properties.sh b/scripts/COLIBRE/compress_halo_properties.sh index 475d96f9..b75a7632 100755 --- a/scripts/COLIBRE/compress_halo_properties.sh +++ b/scripts/COLIBRE/compress_halo_properties.sh @@ -2,48 +2,34 @@ # # Compress SOAP catalogues. # -# Output locations are specified by enviroment variables. E.g. -# -# export COLIBRE_SCRATCH_DIR=/snap8/scratch/dp004/${USER}/COLIBRE/ScienceRuns/ -# export COLIBRE_OUTPUT_DIR=/cosma8/data/dp004/${USER}/COLIBRE/ScienceRuns/ -# -# To run: +# Before running set the location of the input/output/scratch directories. +# Pass the simulation name and snapshots to process when running, e.g. # # cd SOAP # mkdir logs # sbatch -J L0025N0376/Thermal_fiducial --array=0-127%4 ./scripts/COLIBRE/compress_halo_properties.sh # -#SBATCH --ntasks=128 +#SBATCH --nodes=1 #SBATCH --cpus-per-task=1 -#SBATCH -o ./logs/compress_properties.%a.%j.out +#SBATCH -o ./logs/compress_properties.%a.%A.out #SBATCH -p cosma8 #SBATCH -A dp004 #SBATCH --exclusive #SBATCH --no-requeue #SBATCH -t 01:00:00 -#SBATCH --reservation=colibre -# set -e module purge module load python/3.12.4 -# Get location for temporary output -if [[ "${COLIBRE_SCRATCH_DIR}" ]] ; then - scratch_dir="${COLIBRE_SCRATCH_DIR}" -else - echo Please set COLIBRE_SCRATCH_DIR - exit 1 -fi +# TODO: Set these locations +input_dir="/snap8/scratch/dp004/dc-mcgi1/COLIBRE/Runs" +output_dir="/cosma8/data/dp004/dc-mcgi1/COLIBRE/Runs" +scratch_dir="/snap8/scratch/dp004/dc-mcgi1/COLIBRE/Runs" -# Get location for final output -if [[ "${COLIBRE_OUTPUT_DIR}" ]] ; then - output_dir="${COLIBRE_OUTPUT_DIR}" -else - echo Please set COLIBRE_OUTPUT_DIR - exit 1 -fi +# compression script +script="./compression/compress_soap_catalogue.py" # Which snapshot to do snapnum=`printf '%04d' ${SLURM_ARRAY_TASK_ID}` @@ -51,29 +37,22 @@ snapnum=`printf '%04d' ${SLURM_ARRAY_TASK_ID}` # Which simulation to do sim="${SLURM_JOB_NAME}" -# compression script -script="./compression/compress_fast_metadata.py" - -# Location of the input to compress -inbase="${scratch_dir}/${sim}/SOAP_uncompressed/" - -# Location of the compressed output -outbase="${output_dir}/${sim}/SOAP/" -mkdir -p $outbase - # Name of the input SOAP catalogue -input_filename="${inbase}/halo_properties_${snapnum}.hdf5" +input_filename="${input_dir}/${sim}/SOAP_uncompressed/halo_properties_${snapnum}.hdf5" -# name of the output SOAP catalogue +# Location and name of the output SOAP catalogue +outbase="${output_dir}/${sim}/SOAP" +mkdir -p $outbase output_filename="${outbase}/halo_properties_${snapnum}.hdf5" # directory used to store temporary files (preferably a /snap8 directory for # faster writing and reading) -scratch_dir="${scratch_dir}/${sim}/SOAP_compression_tmp/" +tmp_dir="${scratch_dir}/${sim}/SOAP_compression_tmp/" # run the script using all available threads on the node -python3 -u ${script} --nproc 128 ${input_filename} ${output_filename} ${scratch_dir} +mpirun -- python -u ${script} ${input_filename} ${output_filename} ${tmp_dir} +# set the output file to be read-only chmod a=r ${output_filename} echo "Job complete!" diff --git a/scripts/COLIBRE/group_membership_hybrid.sh b/scripts/COLIBRE/group_membership_hybrid.sh index 9cc3f49d..fc97a4ee 100644 --- a/scripts/COLIBRE/group_membership_hybrid.sh +++ b/scripts/COLIBRE/group_membership_hybrid.sh @@ -13,7 +13,7 @@ # #SBATCH --nodes=1 #SBATCH --cpus-per-task=1 -#SBATCH -o ./logs/colibre_membership.%a.%j.out +#SBATCH -o ./logs/colibre_membership.%a.%A.out #SBATCH -J group_membership_colibre #SBATCH -p cosma8 #SBATCH -A dp004 @@ -34,7 +34,7 @@ snapnum=`printf '%04d' ${SLURM_ARRAY_TASK_ID}` sim="${SLURM_JOB_NAME}" # Run the code -mpirun -- python3 -u -m mpi4py ./group_membership.py \ +mpirun -- python3 -u -m mpi4py SOAP/group_membership.py \ parameter_files/COLIBRE_HYBRID.yml \ --sim-name=${sim} --snap-nr=${snapnum} diff --git a/scripts/COLIBRE/group_membership_thermal.sh b/scripts/COLIBRE/group_membership_thermal.sh index 1a9de017..a4efca3d 100644 --- a/scripts/COLIBRE/group_membership_thermal.sh +++ b/scripts/COLIBRE/group_membership_thermal.sh @@ -13,7 +13,7 @@ # #SBATCH --nodes=1 #SBATCH --cpus-per-task=1 -#SBATCH -o ./logs/colibre_membership.%a.%j.out +#SBATCH -o ./logs/colibre_membership.%a.%A.out #SBATCH -J group_membership_colibre #SBATCH -p cosma8 #SBATCH -A dp004 @@ -34,7 +34,7 @@ snapnum=`printf '%04d' ${SLURM_ARRAY_TASK_ID}` sim="${SLURM_JOB_NAME}" # Run the code -mpirun -- python3 -u -m mpi4py ./group_membership.py \ +mpirun -- python3 -u -m mpi4py SOAP/group_membership.py \ parameter_files/COLIBRE_THERMAL.yml \ --sim-name=${sim} --snap-nr=${snapnum} diff --git a/scripts/COLIBRE/halo_properties_hybrid.sh b/scripts/COLIBRE/halo_properties_hybrid.sh index e4f53999..486dbf0e 100644 --- a/scripts/COLIBRE/halo_properties_hybrid.sh +++ b/scripts/COLIBRE/halo_properties_hybrid.sh @@ -14,7 +14,7 @@ # #SBATCH --nodes=1 #SBATCH --cpus-per-task=1 -#SBATCH -o ./logs/colibre_properties_%a.%j.out +#SBATCH -o ./logs/colibre_properties_%a.%A.out #SBATCH -J halo_properties_colibre #SBATCH -p cosma8 #SBATCH -A dp004 @@ -37,7 +37,7 @@ sim="${SLURM_JOB_NAME}" dmo_flag="" #TODO: Set nodes and chunks -mpirun -- python3 -u -m mpi4py ./compute_halo_properties.py \ +mpirun -- python3 -u -m mpi4py SOAP/compute_halo_properties.py \ parameter_files/COLIBRE_HYBRID.yml \ --sim-name=${sim} --snap-nr=${snapnum} --chunks=1 ${dmo_flag} diff --git a/scripts/COLIBRE/halo_properties_thermal.sh b/scripts/COLIBRE/halo_properties_thermal.sh index 5c8543e7..28120f98 100644 --- a/scripts/COLIBRE/halo_properties_thermal.sh +++ b/scripts/COLIBRE/halo_properties_thermal.sh @@ -15,7 +15,7 @@ # #SBATCH --nodes=1 #SBATCH --cpus-per-task=1 -#SBATCH -o ./logs/colibre_properties_%a.%j.out +#SBATCH -o ./logs/colibre_properties_%a.%A.out #SBATCH -J halo_properties_colibre #SBATCH -p cosma8 #SBATCH -A dp004 @@ -38,7 +38,7 @@ sim="${SLURM_JOB_NAME}" dmo_flag="" #TODO: Set nodes and chunks -mpirun -- python3 -u -m mpi4py ./compute_halo_properties.py \ +mpirun -- python3 -u -m mpi4py SOAP/compute_halo_properties.py \ parameter_files/COLIBRE_THERMAL.yml \ --sim-name=${sim} --snap-nr=${snapnum} --chunks=1 ${dmo_flag} diff --git a/scripts/COLIBRE/match_colibre.sh b/scripts/COLIBRE/match_colibre.sh deleted file mode 100644 index 672f5a02..00000000 --- a/scripts/COLIBRE/match_colibre.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/bash -l -# -# Match halos between different colibre runs with the -# same box size and resolution. Pass the snapshots to -# run when calling this script, e.g. -# -# cd SOAP -# mkdir logs -# sbatch -array=92,127%1 ./scripts/COLIBRE/match_colibre.sh -# -#SBATCH --nodes=1 -#SBATCH --cpus-per-task=1 -#SBATCH -J match_colibre -#SBATCH -o ./logs/%x_%a.out -#SBATCH -p cosma8 -#SBATCH -A dp004 -#SBATCH -t 1:00:00 -# - -set -e - -module purge -module load python/3.12.4 gnu_comp/14.1.0 openmpi/5.0.3 parallel_hdf5/1.12.3 -source openmpi-5.0.3-hdf5-1.12.3-env/bin/activate - -# Snapshot to do -snapnum=`printf '%03d' ${SLURM_ARRAY_TASK_ID}` - -# Where to put the output files -outdir="/snap8/scratch/dp004/${USER}/COLIBRE/matching/" - -# Sims to match -sims=( - "L100_m7/THERMAL_AGN_m7 L100_m7/HYBRID_AGN_m7" - "L100_m7/THERMAL_AGN_m7 L100_m7/DMO" -) -for sim in "${sims[@]}"; do - set -- $sim - sim1=$1 - sim2=$2 - - # Location of the HBT catalogues - basedir="/cosma8/data/dp004/colibre/Runs/" - hbt_basename1="${basedir}/${sim1}/HBTplus/${snapnum}/SubSnap_${snapnum}" - hbt_basename2="${basedir}/${sim2}/HBTplus/${snapnum}/SubSnap_${snapnum}" - nr_particles=50 - - # Name of output file - mkdir -p ${outdir} - sim1=$(echo $sim1 | tr '/' '_') - sim2=$(echo $sim2 | tr '/' '_') - outfile="${outdir}/match_${sim1}_${sim2}_${snapnum}.${nr_particles}.hdf5" - - echo - echo Matching $sim1 to $sim2, snapshot ${snapnum} - mpirun -- python -u \ - ./match_hbt_halos.py ${hbt_basename1} ${hbt_basename2} ${nr_particles} ${outfile} - -done - -echo "Job complete!" diff --git a/scripts/COLIBRE/match_fof.sh b/scripts/COLIBRE/match_fof.sh new file mode 100755 index 00000000..93c249fd --- /dev/null +++ b/scripts/COLIBRE/match_fof.sh @@ -0,0 +1,74 @@ +#!/bin/bash -l +# +# Match halos between different colibre runs with the +# same box size and resolution. Pass the snapshots to +# run when calling this script, e.g. +# +# cd SOAP +# mkdir logs +# sbatch -array=92,127%1 ./scripts/COLIBRE/match_colibre.sh +# +#SBATCH --nodes=1 +#SBATCH --cpus-per-task=1 +#SBATCH -J match_colibre +#SBATCH -o ./logs/%x_%a.out +#SBATCH -p cosma8 +#SBATCH -A dp004 +#SBATCH -t 1:00:00 +# + +set -e + +module purge +module load python/3.12.4 gnu_comp/14.1.0 openmpi/5.0.3 parallel_hdf5/1.12.3 +source openmpi-5.0.3-hdf5-1.12.3-env/bin/activate + +# Snapshot to do +snapnum=`printf '%04d' ${SLURM_ARRAY_TASK_ID}` + +# Where to put the output files +outdir="/snap8/scratch/dp004/${USER}/COLIBRE/matching/" + +# Sims to match +sims=( + "L0025N0188/Thermal L0025N0188/DMO" +) +for sim in "${sims[@]}"; do + set -- $sim + sim1=$1 + sim2=$2 + + # Location of the input files + basedir="/cosma8/data/dp004/colibre/Runs" + snap_basename1="${basedir}/${sim1}/snapshots/colibre_${snapnum}/colibre_${snapnum}" + snap_basename2="${basedir}/${sim2}/snapshots/colibre_${snapnum}/colibre_${snapnum}" + membership_basename1="${basedir}/${sim1}/snapshots/colibre_${snapnum}/colibre_${snapnum}" + membership_basename2="${basedir}/${sim2}/snapshots/colibre_${snapnum}/colibre_${snapnum}" + fof_filename1="${basedir}/${sim1}/fof/fof_output_${snapnum}/fof_output_${snapnum}.{file_nr}.hdf5" + fof_filename2="${basedir}/${sim2}/fof/fof_output_${snapnum}/fof_output_${snapnum}.{file_nr}.hdf5" + + # Matching parameters + nr_particles=50 + + # Name of output file + mkdir -p ${outdir} + sim1=$(echo $sim1 | tr '/' '_') + sim2=$(echo $sim2 | tr '/' '_') + output_filename="${outdir}/match_fof_${sim1}_${sim2}_${snapnum}.${nr_particles}.hdf5" + + echo + echo Matching $sim1 to $sim2, snapshot ${snapnum} + mpirun -- python -u misc/match_group_membership.py \ + --snap-basename1 ${snap_basename1}\ + --snap-basename2 ${snap_basename2}\ + --membership-basename1 ${membership_basename1}\ + --membership-basename2 ${membership_basename2}\ + --catalogue-filename1 ${fof_filename1} \ + --catalogue-filename2 ${fof_filename2} \ + --output-filename ${output_filename}\ + --nr-particles ${nr_particles} \ + --match-fof + +done + +echo "Job complete!" diff --git a/scripts/COLIBRE/match_soap.sh b/scripts/COLIBRE/match_soap.sh new file mode 100755 index 00000000..89d8c261 --- /dev/null +++ b/scripts/COLIBRE/match_soap.sh @@ -0,0 +1,73 @@ +#!/bin/bash -l +# +# Match halos between different colibre runs with the +# same box size and resolution. Pass the snapshots to +# run when calling this script, e.g. +# +# cd SOAP +# mkdir logs +# sbatch -array=92,127%1 ./scripts/COLIBRE/match_colibre.sh +# +#SBATCH --nodes=1 +#SBATCH --cpus-per-task=1 +#SBATCH -J match_colibre +#SBATCH -o ./logs/%x_%a.out +#SBATCH -p cosma8 +#SBATCH -A dp004 +#SBATCH -t 1:00:00 +# + +set -e + +module purge +module load python/3.12.4 gnu_comp/14.1.0 openmpi/5.0.3 parallel_hdf5/1.12.3 +source openmpi-5.0.3-hdf5-1.12.3-env/bin/activate + +# Snapshot to do +snapnum=`printf '%04d' ${SLURM_ARRAY_TASK_ID}` + +# Where to put the output files +outdir="/snap8/scratch/dp004/${USER}/COLIBRE/matching/" + +# Sims to match +sims=( + "L0025N0188/Thermal L0025N0188/DMO" +) +for sim in "${sims[@]}"; do + set -- $sim + sim1=$1 + sim2=$2 + + # Location of the input files + basedir="/cosma8/data/dp004/colibre/Runs" + snap_basename1="${basedir}/${sim1}/snapshots/colibre_${snapnum}/colibre_${snapnum}" + snap_basename2="${basedir}/${sim2}/snapshots/colibre_${snapnum}/colibre_${snapnum}" + membership_basename1="${basedir}/${sim1}/SOAP-HBT/membership_${snapnum}/membership_${snapnum}" + membership_basename2="${basedir}/${sim2}/SOAP-HBT/membership_${snapnum}/membership_${snapnum}" + soap_filename1="${basedir}/${sim1}/SOAP-HBT/halo_properties_${snapnum}.hdf5" + soap_filename2="${basedir}/${sim2}/SOAP-HBT/halo_properties_${snapnum}.hdf5" + + # Matching parameters + nr_particles=50 + + # Name of output file + mkdir -p ${outdir} + sim1=$(echo $sim1 | tr '/' '_') + sim2=$(echo $sim2 | tr '/' '_') + output_filename="${outdir}/match_${sim1}_${sim2}_${snapnum}.${nr_particles}.hdf5" + + echo + echo Matching $sim1 to $sim2, snapshot ${snapnum} + mpirun -- python -u misc/match_group_membership.py \ + --snap-basename1 ${snap_basename1}\ + --snap-basename2 ${snap_basename2}\ + --membership-basename1 ${membership_basename1}\ + --membership-basename2 ${membership_basename2}\ + --catalogue-filename1 ${soap_filename1} \ + --catalogue-filename2 ${soap_filename2} \ + --output-filename ${output_filename}\ + --nr-particles ${nr_particles} \ + +done + +echo "Job complete!" diff --git a/scripts/EAGLE.sh b/scripts/EAGLE.sh new file mode 100755 index 00000000..9d5f5130 --- /dev/null +++ b/scripts/EAGLE.sh @@ -0,0 +1,84 @@ +#!/bin/bash -l +# +#SBATCH --ntasks=256 +#SBATCH --cpus-per-task=1 +#SBATCH -o ./logs/eagle_%a.%j.out +#SBATCH -J soap_eagle +#SBATCH -p cosma7 +#SBATCH -A dp004 +#SBATCH --exclusive +#SBATCH -t 02:00:00 +# + +sim_name='L0100N1504' +snap_nr="028" +z_suffix="z000p000" + +module purge +module load python/3.12.4 gnu_comp/14.1.0 openmpi/5.0.3 parallel_hdf5/1.12.3 +source openmpi-5.0.3-hdf5-1.12.3-env/bin/activate +pip install git+ssh://git@github.com/kyleaoman/Hdecompose.git + +######## Link files to snap (to remove awful z suffix) + +sim_dir="/cosma7/data/Eagle/ScienceRuns/Planck1/${sim_name}/PE/REFERENCE/data" +output_dir="/snap7/scratch/dp004/dc-mcgi1/SOAP_EAGLE/${sim_name}" + +sim_snap_dir="${sim_dir}/particledata_${snap_nr}_${z_suffix}" +output_snap_dir="${output_dir}/gadget_snapshots/snapshot_${snap_nr}" +mkdir -p $output_snap_dir +i=0 +while [[ -e "${sim_snap_dir}/eagle_subfind_particles_${snap_nr}_${z_suffix}.${i}.hdf5" ]]; do + old_name="${sim_snap_dir}/eagle_subfind_particles_${snap_nr}_${z_suffix}.${i}.hdf5" + new_name="${output_snap_dir}/snap_${snap_nr}.${i}.hdf5" + ln -s $old_name $new_name + ((i++)) +done + +sim_group_dir="${sim_dir}/groups_${snap_nr}_${z_suffix}" +output_group_dir="${output_dir}/subfind/groups_${snap_nr}" +mkdir -p $output_group_dir +i=0 +while [[ -e "${sim_group_dir}/eagle_subfind_tab_${snap_nr}_${z_suffix}.${i}.hdf5" ]]; do + old_name="${sim_group_dir}/eagle_subfind_tab_${snap_nr}_${z_suffix}.${i}.hdf5" + new_name="${output_group_dir}/subfind_tab_${snap_nr}.${i}.hdf5" + ln -s $old_name $new_name + ((i++)) +done + +######### Create SWIFT snapshot + +set -e + +mpirun -- python -u misc/convert_eagle.py \ + --snap-basename "${output_dir}/gadget_snapshots/snapshot_${snap_nr}/snap_${snap_nr}" \ + --subfind-basename "${output_group_dir}/subfind_tab_${snap_nr}" \ + --output-basename "${output_dir}/swift_snapshots/swift_${snap_nr}/snap_${snap_nr}" \ + --membership-basename "${output_dir}/SOAP_uncompressed/membership_${snap_nr}/membership_${snap_nr}" + +######### Estimate SpeciesFraction of hydrogen + +mpirun -- python -u misc/hdecompose_hydrogen_fractions.py \ + --snap-basename "${output_dir}/swift_snapshots/swift_${snap_nr}/snap_${snap_nr}" \ + --output-basename "${output_dir}/species_fractions/swift_${snap_nr}/snap_${snap_nr}" + +######### Create virtual snapshot +# Must be run from the snapshot directory itself or there will be issues with paths + +cd "${output_dir}/swift_snapshots/swift_${snap_nr}" +wget https://gitlab.cosma.dur.ac.uk/swift/swiftsim/-/raw/master/tools/create_virtual_snapshot.py +python create_virtual_snapshot.py "snap_${snap_nr}.0.hdf5" +rm create_virtual_snapshot.py* +cd - + +######### Run SOAP + +chunks=10 + +mpirun -- python3 -u -m mpi4py SOAP/compute_halo_properties.py \ + parameter_files/EAGLE.yml \ + --sim-name=${sim_name} --snap-nr=${snap_nr} --chunks=${chunks} + +############## + +echo "Job complete!" diff --git a/scripts/FLAMINGO/L1000N0900/compress_group_membership_L1000N0900.sh b/scripts/FLAMINGO/L1000N0900/compress_group_membership_L1000N0900.sh index 5e8f3317..ddf02a36 100644 --- a/scripts/FLAMINGO/L1000N0900/compress_group_membership_L1000N0900.sh +++ b/scripts/FLAMINGO/L1000N0900/compress_group_membership_L1000N0900.sh @@ -69,7 +69,6 @@ outbase="${output_dir}/${sim}/SOAP/${halo_finder}/" # Create the output folder if it does not exist outdir="${outbase}/membership_${snapnum}" mkdir -p "${outdir}" -lfs setstripe --stripe-count=-1 --stripe-size=32M "${outdir}" # Uncompressed membership file basename input_filename="${inbase}/membership_${snapnum}/membership_${snapnum}" diff --git a/scripts/FLAMINGO/L1000N0900/compress_halo_properties_L1000N0900.sh b/scripts/FLAMINGO/L1000N0900/compress_halo_properties_L1000N0900.sh deleted file mode 100644 index 0d79280e..00000000 --- a/scripts/FLAMINGO/L1000N0900/compress_halo_properties_L1000N0900.sh +++ /dev/null @@ -1,87 +0,0 @@ -#!/bin/bash -# -# Compress SOAP catalogues. -# -# Output locations are specified by enviroment variables. E.g. -# -# export FLAMINGO_SCRATCH_DIR=/snap8/scratch/dp004/${USER}/FLAMINGO/ScienceRuns/ -# export FLAMINGO_OUTPUT_DIR=/cosma8/data/dp004/${USER}/FLAMINGO/ScienceRuns/ -# export HALO_FINDER=HBTplus -# -# To run: -# -# cd SOAP -# mkdir logs -# sbatch -J HYDRO_FIDUCIAL --array=0-77%4 ./scripts/FLAMINGO/L1000N0900/compress_halo_properties_L1000N0900.sh -# -#SBATCH --ntasks=128 -#SBATCH --cpus-per-task=1 -#SBATCH -o ./logs/compress_properties_L1000N0900_%x.%a.%j.out -#SBATCH -p cosma8 -#SBATCH -A dp004 -#SBATCH --exclusive -#SBATCH --no-requeue -#SBATCH -t 01:00:00 -# - -set -e - -module purge -module load python/3.12.4 gnu_comp/14.1.0 openmpi/5.0.3 parallel_hdf5/1.12.3 -source openmpi-5.0.3-hdf5-1.12.3-env/bin/activate - -# Get location for temporary output -if [[ "${FLAMINGO_SCRATCH_DIR}" ]] ; then - scratch_dir="${FLAMINGO_SCRATCH_DIR}" -else - echo Please set FLAMINGO_SCRATCH_DIR - exit 1 -fi - -# Get location for final output -if [[ "${FLAMINGO_OUTPUT_DIR}" ]] ; then - output_dir="${FLAMINGO_OUTPUT_DIR}" -else - echo Please set FLAMINGO_OUTPUT_DIR - exit 1 -fi - -# Get halo finder used -if [[ "${HALO_FINDER}" ]] ; then - halo_finder="${HALO_FINDER}" -else - echo Please set HALO_FINDER - exit 1 -fi - -# Which snapshot to do -snapnum=`printf '%04d' ${SLURM_ARRAY_TASK_ID}` - -# Which simulation to do -sim="L1000N0900/${SLURM_JOB_NAME}" - -# compression script -script="./compression/compress_fast_metadata.py" - -# Location of the input to compress -inbase="${output_dir}/${sim}/SOAP_uncompressed/${halo_finder}/" - -# Location of the compressed output -outbase="${output_dir}/${sim}/SOAP/${halo_finder}/" - -# Name of the input SOAP catalogue -input_filename="${inbase}/halo_properties_${snapnum}.hdf5" - -# name of the output SOAP catalogue -output_filename="${outbase}/halo_properties_${snapnum}.hdf5" - -# directory used to store temporary files (preferably a /snap8 directory for -# faster writing and reading) -scratch_dir="${scratch_dir}/${sim}/SOAP_compression_tmp/" - -# run the script using all available threads on the node -python3 ${script} --nproc 128 ${input_filename} ${output_filename} ${scratch_dir} - -chmod a=r ${output_filename} - -echo "Job complete!" diff --git a/scripts/FLAMINGO/L1000N0900/group_membership_L1000N0900.sh b/scripts/FLAMINGO/L1000N0900/group_membership_L1000N0900.sh index 2d1a8de5..de5505b7 100644 --- a/scripts/FLAMINGO/L1000N0900/group_membership_L1000N0900.sh +++ b/scripts/FLAMINGO/L1000N0900/group_membership_L1000N0900.sh @@ -35,7 +35,7 @@ snapnum=`printf '%04d' ${SLURM_ARRAY_TASK_ID}` sim="L1000N0900/${SLURM_JOB_NAME}" # Run the code -mpirun -- python3 -u -m mpi4py ./group_membership.py \ +mpirun -- python3 -u -m mpi4py SOAP/group_membership.py \ --sim-name=${sim} --snap-nr=${snapnum} \ parameter_files/FLAMINGO.yml diff --git a/scripts/FLAMINGO/L1000N0900/halo_properties_L1000N0900.sh b/scripts/FLAMINGO/L1000N0900/halo_properties_L1000N0900.sh index c2bdf97a..6bc8d42b 100644 --- a/scripts/FLAMINGO/L1000N0900/halo_properties_L1000N0900.sh +++ b/scripts/FLAMINGO/L1000N0900/halo_properties_L1000N0900.sh @@ -41,10 +41,10 @@ else swift_filename="/cosma8/data/dp004/flamingo/Runs/${sim}/snapshots/flamingo_{snap_nr:04}/flamingo_{snap_nr:04}.{file_nr}.hdf5" xray_filename="/snap8/scratch/dp004/${USER}/flamingo/Runs/${sim}/xray/xray_{snap_nr:04}/xray_{snap_nr:04}.{file_nr}.hdf5" xray_table_path='/cosma8/data/dp004/flamingo/Tables/Xray/X_Ray_table_metals_full.hdf5' - mpirun -- python recalculate_xrays.py $swift_filename $xray_filename $xray_table_path --snap-nr=$snapnum + mpirun -- python misc/recalculate_xrays.py $swift_filename $xray_filename $xray_table_path --snap-nr=$snapnum fi -mpirun -- python3 -u -m mpi4py ./compute_halo_properties.py \ +mpirun -- python3 -u -m mpi4py SOAP/compute_halo_properties.py \ --sim-name=${sim} --snap-nr=${snapnum} --reference-snapshot=77 \ --chunks=1 ${dmo_flag} parameter_files/FLAMINGO.yml diff --git a/scripts/FLAMINGO/L1000N1800/compress_group_membership_L1000N1800.sh b/scripts/FLAMINGO/L1000N1800/compress_group_membership_L1000N1800.sh index 53f6ff4b..9b3de6ca 100644 --- a/scripts/FLAMINGO/L1000N1800/compress_group_membership_L1000N1800.sh +++ b/scripts/FLAMINGO/L1000N1800/compress_group_membership_L1000N1800.sh @@ -69,7 +69,6 @@ outbase="${output_dir}/${sim}/SOAP/${halo_finder}/" # Create the output folder if it does not exist outdir="${outbase}/membership_${snapnum}" mkdir -p "${outdir}" -lfs setstripe --stripe-count=-1 --stripe-size=32M "${outdir}" # Uncompressed membership file basename input_filename="${inbase}/membership_${snapnum}/membership_${snapnum}" diff --git a/scripts/FLAMINGO/L1000N1800/compress_halo_properties_L1000N1800.sh b/scripts/FLAMINGO/L1000N1800/compress_halo_properties_L1000N1800.sh old mode 100755 new mode 100644 index 5caa6455..60d202d0 --- a/scripts/FLAMINGO/L1000N1800/compress_halo_properties_L1000N1800.sh +++ b/scripts/FLAMINGO/L1000N1800/compress_halo_properties_L1000N1800.sh @@ -2,56 +2,34 @@ # # Compress SOAP catalogues. # -# Output locations are specified by enviroment variables. E.g. -# -# export FLAMINGO_SCRATCH_DIR=/snap8/scratch/dp004/${USER}/FLAMINGO/ScienceRuns/ -# export FLAMINGO_OUTPUT_DIR=/cosma8/data/dp004/${USER}/FLAMINGO/ScienceRuns/ -# export HALO_FINDER=HBTplus -# -# To run: +# Before running set the location of the input/output/scratch directories. +# Pass the simulation variation and snapshots to process when running, e.g. # # cd SOAP # mkdir logs # sbatch -J HYDRO_FIDUCIAL --array=0-77%4 ./scripts/FLAMINGO/L1000N1800/compress_halo_properties_L1000N1800.sh # -#SBATCH --ntasks=128 +#SBATCH --nodes=1 #SBATCH --cpus-per-task=1 -#SBATCH -o ./logs/compress_properties_L1000N1800_%x.%a.%j.out +#SBATCH -o ./logs/compress_properties_%x.%a.%j.out #SBATCH -p cosma8 #SBATCH -A dp004 #SBATCH --exclusive #SBATCH --no-requeue #SBATCH -t 01:00:00 -# set -e module purge module load python/3.12.4 -# Get location for temporary output -if [[ "${FLAMINGO_SCRATCH_DIR}" ]] ; then - scratch_dir="${FLAMINGO_SCRATCH_DIR}" -else - echo Please set FLAMINGO_SCRATCH_DIR - exit 1 -fi +# TODO: Set these locations +input_dir="/snap8/scratch/dp004/dc-mcgi1/FLAMINGO/Runs" +output_dir="/cosma8/data/dp004/dc-mcgi1/FLAMINGO/Runs" +scratch_dir="/snap8/scratch/dp004/dc-mcgi1/FLAMINGO/Runs" -# Get location for final output -if [[ "${FLAMINGO_OUTPUT_DIR}" ]] ; then - output_dir="${FLAMINGO_OUTPUT_DIR}" -else - echo Please set FLAMINGO_OUTPUT_DIR - exit 1 -fi - -# Get halo finder used -if [[ "${HALO_FINDER}" ]] ; then - halo_finder="${HALO_FINDER}" -else - echo Please set HALO_FINDER - exit 1 -fi +# compression script +script="./compression/compress_soap_catalogue.py" # Which snapshot to do snapnum=`printf '%04d' ${SLURM_ARRAY_TASK_ID}` @@ -59,29 +37,19 @@ snapnum=`printf '%04d' ${SLURM_ARRAY_TASK_ID}` # Which simulation to do sim="L1000N1800/${SLURM_JOB_NAME}" -# compression script -script="./compression/compress_fast_metadata.py" - -# Location of the input to compress -inbase="${scratch_dir}/${sim}/SOAP_uncompressed/${halo_finder}/" - -# Location of the compressed output -outbase="${output_dir}/${sim}/SOAP-HBT/" -mkdir -p $outbase - # Name of the input SOAP catalogue -input_filename="${inbase}/halo_properties_${snapnum}.hdf5" +input_filename="${input_dir}/${sim}/SOAP_uncompressed/halo_properties_${snapnum}.hdf5" -# name of the output SOAP catalogue +# Location and name of the output SOAP catalogue +outbase="${output_dir}/${sim}/SOAP" +mkdir -p $outbase output_filename="${outbase}/halo_properties_${snapnum}.hdf5" # directory used to store temporary files (preferably a /snap8 directory for # faster writing and reading) -scratch_dir="${scratch_dir}/${sim}/SOAP_compression_tmp/" +tmp_dir="${scratch_dir}/${sim}/SOAP_compression_tmp/" # run the script using all available threads on the node -python3 ${script} --nproc 128 ${input_filename} ${output_filename} ${scratch_dir} - -chmod a=r ${output_filename} +mpirun -- python -u ${script} ${input_filename} ${output_filename} ${tmp_dir} echo "Job complete!" diff --git a/scripts/FLAMINGO/L1000N1800/group_membership_L1000N1800.sh b/scripts/FLAMINGO/L1000N1800/group_membership_L1000N1800.sh index 5924b611..65ec3224 100644 --- a/scripts/FLAMINGO/L1000N1800/group_membership_L1000N1800.sh +++ b/scripts/FLAMINGO/L1000N1800/group_membership_L1000N1800.sh @@ -35,7 +35,7 @@ snapnum=`printf '%04d' ${SLURM_ARRAY_TASK_ID}` sim="L1000N1800/${SLURM_JOB_NAME}" # Run the code -mpirun -- python3 -u -m mpi4py ./group_membership.py \ +mpirun -- python3 -u -m mpi4py SOAP/group_membership.py \ --sim-name=${sim} --snap-nr=${snapnum} \ parameter_files/FLAMINGO.yml diff --git a/scripts/FLAMINGO/L1000N1800/halo_properties_L1000N1800.sh b/scripts/FLAMINGO/L1000N1800/halo_properties_L1000N1800.sh index 9807c91f..520db302 100644 --- a/scripts/FLAMINGO/L1000N1800/halo_properties_L1000N1800.sh +++ b/scripts/FLAMINGO/L1000N1800/halo_properties_L1000N1800.sh @@ -41,10 +41,10 @@ else swift_filename="/cosma8/data/dp004/flamingo/Runs/${sim}/snapshots/flamingo_{snap_nr:04}/flamingo_{snap_nr:04}.{file_nr}.hdf5" xray_filename="/snap8/scratch/dp004/${USER}/flamingo/Runs/${sim}/xray/xray_{snap_nr:04}/xray_{snap_nr:04}.{file_nr}.hdf5" xray_table_path='/cosma8/data/dp004/flamingo/Tables/Xray/X_Ray_table_metals_full.hdf5' - mpirun -- python recalculate_xrays.py $swift_filename $xray_filename $xray_table_path --snap-nr=$snapnum + mpirun -- python misc/recalculate_xrays.py $swift_filename $xray_filename $xray_table_path --snap-nr=$snapnum fi -mpirun -- python3 -u -m mpi4py ./compute_halo_properties.py \ +mpirun -- python3 -u -m mpi4py SOAP/compute_halo_properties.py \ --sim-name=${sim} --snap-nr=${snapnum} --reference-snapshot=77 \ --chunks=4 ${dmo_flag} parameter_files/FLAMINGO.yml diff --git a/scripts/FLAMINGO/L1000N3600/group_membership_L1000N3600.sh b/scripts/FLAMINGO/L1000N3600/group_membership_L1000N3600.sh index 5778a6f6..9cdf8358 100644 --- a/scripts/FLAMINGO/L1000N3600/group_membership_L1000N3600.sh +++ b/scripts/FLAMINGO/L1000N3600/group_membership_L1000N3600.sh @@ -35,7 +35,7 @@ snapnum=`printf '%04d' ${SLURM_ARRAY_TASK_ID}` sim="L1000N3600/${SLURM_JOB_NAME}" # Run the code -mpirun -- python3 -u -m mpi4py ./group_membership.py \ +mpirun -- python3 -u -m mpi4py SOAP/group_membership.py \ --sim-name=${sim} --snap-nr=${snapnum} \ parameter_files/FLAMINGO.yml diff --git a/scripts/FLAMINGO/L1000N3600/halo_properties_L1000N3600.sh b/scripts/FLAMINGO/L1000N3600/halo_properties_L1000N3600.sh index f9d42991..e47d26cc 100644 --- a/scripts/FLAMINGO/L1000N3600/halo_properties_L1000N3600.sh +++ b/scripts/FLAMINGO/L1000N3600/halo_properties_L1000N3600.sh @@ -41,10 +41,10 @@ else swift_filename="/cosma8/data/dp004/flamingo/Runs/${sim}/snapshots/flamingo_{snap_nr:04}/flamingo_{snap_nr:04}.{file_nr}.hdf5" xray_filename="/snap8/scratch/dp004/${USER}/flamingo/Runs/${sim}/xray/xray_{snap_nr:04}/xray_{snap_nr:04}.{file_nr}.hdf5" xray_table_path='/cosma8/data/dp004/flamingo/Tables/Xray/X_Ray_table_metals_full.hdf5' - mpirun -- python recalculate_xrays.py $swift_filename $xray_filename $xray_table_path --snap-nr=$snapnum + mpirun -- python misc/recalculate_xrays.py $swift_filename $xray_filename $xray_table_path --snap-nr=$snapnum fi -mpirun -- python3 -u -m mpi4py ./compute_halo_properties.py \ +mpirun -- python3 -u -m mpi4py SOAP/compute_halo_properties.py \ --sim-name=${sim} --snap-nr=${snapnum} --reference-snapshot=78 \ --chunks=80 ${dmo_flag} parameter_files/FLAMINGO.yml diff --git a/scripts/FLAMINGO/L5600N5040/halo_properties_L5600N5040.sh b/scripts/FLAMINGO/L5600N5040/halo_properties_L5600N5040.sh deleted file mode 100644 index 18a44d05..00000000 --- a/scripts/FLAMINGO/L5600N5040/halo_properties_L5600N5040.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash -l -# -# Compute halo properties for a snapshot. Must run the group_membership -# script first. -# -# Job name determines which of the L5600N5040 runs we process. -# Array job index is the snapshot number to do. Submit with (for example): -# -# cd SOAP -# mkdir logs -# sbatch -J DMO_FIDUCIAL --array=0-78%4 ./scripts/FLAMINGO/L5600N5040/halo_properties_L5600N5040.sh -# -#SBATCH --nodes=32 -#SBATCH --cpus-per-task=1 -#SBATCH -o ./logs/halo_properties_L5600N5040_%x.%a.%A.out -#SBATCH -p cosma8 -#SBATCH -A dp004 -#SBATCH --exclusive -#SBATCH -t 08:00:00 -# - -module purge -module load python/3.12.4 gnu_comp/14.1.0 openmpi/5.0.3 parallel_hdf5/1.12.3 -source openmpi-5.0.3-hdf5-1.12.3-env/bin/activate - -set -e - -# Which snapshot to do -snapnum=${SLURM_ARRAY_TASK_ID} - -# Which simulation to do -sim="L5600N5040/${SLURM_JOB_NAME}" - -# Check for DMO run -dmo_flag="" -if [[ $sim == *DMO_* ]] ; then - dmo_flag="--dmo" -fi - -mpirun -- python3 -u -m mpi4py ./compute_halo_properties.py \ - --sim-name=${sim} --snap-nr=${snapnum} --reference-snapshot=78 \ - --chunks=128 ${dmo_flag} parameter_files/FLAMINGO.yml - -echo "Job complete!" diff --git a/scripts/cosma_python_env.sh b/scripts/cosma_python_env.sh index bf263267..7998dfd7 100755 --- a/scripts/cosma_python_env.sh +++ b/scripts/cosma_python_env.sh @@ -42,3 +42,6 @@ ln -s "${mpirun}" "${venv_name}"/bin/mpirun pip install -r requirements.txt git clone https://github.com/jchelly/VirgoDC.git "${venv_name}/VirgoDC" pip install "${venv_name}/VirgoDC/python" + +# Install SOAP +pip install -e . diff --git a/scripts/strw_python_env.sh b/scripts/strw_python_env.sh new file mode 100755 index 00000000..4503cd0a --- /dev/null +++ b/scripts/strw_python_env.sh @@ -0,0 +1,22 @@ +module purge +module load localhosts +module load HDF5 +module load Python/3.11.5-GCCcore-13.2.0 + +if [ -d "venv" ]; then + source venv/bin/activate +else + python -m venv venv + source venv/bin/activate + + python -m pip cache purge + pip install mpi4py + export CC=mpicc + export HDF5_MPI="ON" + pip install --no-binary=h5py h5py + + pip install -r requirements.txt + + pip install -e . +fi + diff --git a/tests/COLIBRE/find_halo_ids.py b/tests/COLIBRE/find_halo_ids.py index 38e10cb9..7d36bf3f 100755 --- a/tests/COLIBRE/find_halo_ids.py +++ b/tests/COLIBRE/find_halo_ids.py @@ -2,7 +2,7 @@ # # Find IDs of halos in a corner of the simulation box # -# Run with e.g. `python3 ./find_halo_ids.py L0025N0376/Thermal_fiducial 123 1` +# Run with e.g. `python ./find_halo_ids.py L0050N0376/Thermal 127 3` # import sys import numpy as np @@ -10,14 +10,16 @@ def find_halo_indices(sim, snap_nr, boxsize): - soap_file = f"/cosma8/data/dp004/jlvc76/COLIBRE/ScienceRuns/{sim}/SOAP/SOAP_uncompressed/halo_properties_{snap_nr:04d}.hdf5" + soap_file = f"/cosma8/data/dp004/colibre/Runs//{sim}/SOAP-HBT/halo_properties_{snap_nr:04d}.hdf5" with h5py.File(soap_file, "r") as f: pos = f["InputHalos/HaloCentre"][()] mask = np.all(pos < boxsize, axis=1) index = f["InputHalos/HaloCatalogueIndex"][()] is_central = f["InputHalos/IsCentral"][()] + nstar = f['BoundSubhalo/NumberOfStarParticles'][:] if np.sum(is_central[mask]) == 0: print('No centrals loaded') + print(f'Max number of stars: {np.max(nstar[mask])}') return index[mask] diff --git a/tests/COLIBRE/parameters.yml b/tests/COLIBRE/parameters.yml index 5b91f4c5..d0a3d6fa 100644 --- a/tests/COLIBRE/parameters.yml +++ b/tests/COLIBRE/parameters.yml @@ -1,22 +1,22 @@ # Values in this section are substituted into the other sections Parameters: - sim_dir: /cosma8/data/dp004/jlvc76/COLIBRE/ScienceRuns + sim_dir: /cosma8/data/dp004/colibre/Runs output_dir: output scratch_dir: output # Location of the Swift snapshots: Snapshots: - filename: "{sim_dir}/{sim_name}/colibre_{snap_nr:04d}/colibre_{snap_nr:04d}.{file_nr}.hdf5" + filename: "{sim_dir}/{sim_name}/snapshots/colibre_{snap_nr:04d}/colibre_{snap_nr:04d}.{file_nr}.hdf5" # Which halo finder we're using, and base name for halo finder output files HaloFinder: type: HBTplus - filename: "{sim_dir}/{sim_name}/HBTplus/{snap_nr:03d}/SubSnap_{snap_nr:03d}" - fof_filename: "{sim_dir}/{sim_name}/fof_output_{snap_nr:04d}/fof_output_{snap_nr:04d}.{file_nr}.hdf5" + filename: "{sim_dir}/{sim_name}/HBT-HERONS/sorted_catalogues/OrderedSubSnap_{snap_nr:03d}.hdf5" + fof_filename: "{sim_dir}/{sim_name}/fof/fof_output_{snap_nr:04d}/fof_output_{snap_nr:04d}.{file_nr}.hdf5" # Where to find the group membership files GroupMembership: - filename: "{sim_dir}/{sim_name}/SOAP/SOAP_uncompressed/membership_{snap_nr:04d}/membership_{snap_nr:04d}.{file_nr}.hdf5" + filename: "{sim_dir}/{sim_name}/SOAP-HBT/membership_{snap_nr:04d}/membership_{snap_nr:04d}.{file_nr}.hdf5" HaloProperties: # Where to write the halo properties file diff --git a/tests/COLIBRE/run_L0025N0188_Thermal.sh b/tests/COLIBRE/run_L0025N0188_Thermal.sh new file mode 100755 index 00000000..27a43e1e --- /dev/null +++ b/tests/COLIBRE/run_L0025N0188_Thermal.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# +# This runs SOAP on a few halos in the L0025N0376/Thermal_fiducial +# box on Cosma8. It can be used as a quick test of new halo property +# code. Takes ~2 minutes to run. +# +# Should be run from the SOAP source directory. E.g.: +# +# cd SOAP +# ./tests/FLAMINGO/run_L0025N0376_Thermal_fiducial.sh +# + +module purge +module load python/3.12.4 gnu_comp/14.1.0 openmpi/5.0.3 parallel_hdf5/1.12.3 +source openmpi-5.0.3-hdf5-1.12.3-env/bin/activate + +# Which simulation to do +sim="L0025N0188/Thermal" + +# Snapshot number to do +snapnum=0092 + +# Halo indices to do: all halos with x<5, y<5, and z<5 cMpc in snap 92 +halo_indices="17079 20065 22326 27035 27037 34951 36275 39305 40463 40495 44619 45938 45939 48451 49639 49646 49657 51919 51938 53031 56226 57389 60474 61533 62532 64282 67777 67801 68350 69437 69897 70019 70400 71432 71975 72461 72943 73459 73932 73939 73962 74036 74440 75916 7 819 1684 1689 2295 3231 3232 5123 5928 5954 6828 6853 6863 7859 8930 8932 10191 11482 11496 11499 11507 12942 12951 14444 16066 16096 16097 19060 19077 20095 21208 21212 21218 23496 25868 25870 25872 26584 27042 28291 29610 30883 30942 32234 32268 33601 34967 34986 36282 37688 37689 37699 37710 37711 39117 41890 41892 43234 44574 44576 44589 44598 44599 45947 45948 47198 47200 47208 48449 48461 49654 50787 50788 50798 51921 51967 51974 54152 54699 55224 55227 56283 56284 57383 57385 57386 57392 59473 59476 62536 62546 62547 63591 64274 64917 66673 66675 66697 67779 67781 68342 68911 69426 69430 69432 70953 70955 72942 72948 72950 73452 73931 73933 74460 75423 75424 39078 47222 61547 73961 821 6842 12933 14454 14459 16071 27065 28285 29595 29612 33595 36311 39103 43263 45950 45954 53065 54100 56277 60517 61497 62524 62554 63595 63987 64286 64920 64921 65527 66694 67786 70404 71417 71420 71968 73941 74448 74945 75903 3380 3381 5116 5128 5133 9138 14494 14732 16055 16241 18094 19219 19222 24819 24846 28564 28566 28569 31201 32240 33597 34944 37694 43240 45927 47203 48440 48446 49637 50805 53029 53033 54146 55183 55186 55189 55195 55405 56258 60703 61536 61759 63763 64294 64307 65631 66120 66122 67241 68344 68905 69436 69438 70394 70954 70957 71430 72453 72454 72962 72963 73935 73945 73963 74438 74441 75415 75918 5939 18100 22345 32291 55169 55194 71418 72951 30929 57361 62589 67803 36297 50822 67788 68347 69899" + +# Create parameters files +python tests/COLIBRE/create_parameters_file.py + +# Remove tmp directory (so we don't load chunks if they already exist) +rm -r output/SOAP-tmp + +# Run SOAP on eight cores processing the selected halos. Use 'python3 -m pdb' to start in the debugger. +mpirun -np 8 python3 -u -m mpi4py SOAP/compute_halo_properties.py \ + ./tests/COLIBRE/test_parameters.yml \ + --halo-indices ${halo_indices} \ + --sim-name=${sim} --snap-nr=${snapnum} --chunks=1 + diff --git a/tests/COLIBRE/run_L0025N0376_Thermal_fiducial.sh b/tests/COLIBRE/snipshot.sh similarity index 73% rename from tests/COLIBRE/run_L0025N0376_Thermal_fiducial.sh rename to tests/COLIBRE/snipshot.sh index 7418c5ef..324925fb 100755 --- a/tests/COLIBRE/run_L0025N0376_Thermal_fiducial.sh +++ b/tests/COLIBRE/snipshot.sh @@ -15,19 +15,22 @@ module load python/3.12.4 gnu_comp/14.1.0 openmpi/5.0.3 parallel_hdf5/1.12.3 source openmpi-5.0.3-hdf5-1.12.3-env/bin/activate # Which simulation to do -sim="L0025N0376/Thermal_fiducial" +sim="L0100N0752/Thermal" # Snapshot number to do -snapnum=0123 +snapnum=0126 -# Halo indices to do: all halos with x<1, y<1, and z<1 cMpc in snap 123 -halo_indices="1938 1850 1907 1150 1947 1234 436621 1056 1900 1858 1988 1212 1564 1948 2574 1567 1331 1940" +# Most massive object +halo_indices="3771072" # Create parameters files python tests/COLIBRE/create_parameters_file.py +# Remove tmp directory (so we don't load chunks if they already exist) +rm -r output/SOAP-tmp + # Run SOAP on eight cores processing the selected halos. Use 'python3 -m pdb' to start in the debugger. -mpirun -np 8 python3 -u -m mpi4py ./compute_halo_properties.py \ +mpirun -np 8 python3 -u -m mpi4py SOAP/compute_halo_properties.py \ ./tests/COLIBRE/test_parameters.yml \ --halo-indices ${halo_indices} \ --sim-name=${sim} --snap-nr=${snapnum} --chunks=1 diff --git a/tests/FLAMINGO/parameters_HYDRO.yml b/tests/FLAMINGO/parameters_HYDRO.yml index aa040e03..cf7fa252 100644 --- a/tests/FLAMINGO/parameters_HYDRO.yml +++ b/tests/FLAMINGO/parameters_HYDRO.yml @@ -9,14 +9,14 @@ Snapshots: HaloFinder: type: HBTplus filename: "{sim_dir}/{sim_name}/HBT/{snap_nr:03d}/SubSnap_{snap_nr:03d}" - fof_filename: "/cosma8/data/dp004/jlvc76/FLAMINGO/FOF/{sim_name}/fof_catalog/fof_output_{snap_nr:04d}/fof_output_{snap_nr:04d}.{file_nr}.hdf5" + fof_filename: "{sim_dir}/{sim_name}/fof_HBT/fof_output_{snap_nr:04d}/fof_output_{snap_nr:04d}.{file_nr}.hdf5" GroupMembership: # Use the pre-existing membership files filename: "{sim_dir}/{sim_name}/SOAP-HBT/membership_{snap_nr:04d}/membership_{snap_nr:04d}.{file_nr}.hdf5" ExtraInput: - xrays: "/cosma8/data/dp004/dc-mcgi1/FLAMINGO/Xray/L1000N1800/HYDRO_FIDUCIAL/xray/flamingo_{snap_nr:04}/xray_{snap_nr:04}.{file_nr}.hdf5" + xrays: "/cosma8/data/dp004/dc-mcgi1/FLAMINGO/Xray/{sim_name}/xray/flamingo_{snap_nr:04}/xray_{snap_nr:04}.{file_nr}.hdf5" HaloProperties: # Where to write the halo properties file diff --git a/tests/FLAMINGO/parameters_VR.yml b/tests/FLAMINGO/parameters_VR.yml new file mode 100644 index 00000000..b60df5ed --- /dev/null +++ b/tests/FLAMINGO/parameters_VR.yml @@ -0,0 +1,24 @@ +Parameters: + sim_dir: /cosma8/data/dp004/flamingo/Runs/ + output_dir: output/ + scratch_dir: output/ + +Snapshots: + filename: "{sim_dir}/{sim_name}/snapshots/flamingo_{snap_nr:04d}/flamingo_{snap_nr:04d}.{file_nr}.hdf5" + +HaloFinder: + type: VR + filename: "{sim_dir}/{sim_name}/VR/catalogue_{snap_nr:04d}/vr_catalogue_{snap_nr:04d}" + +GroupMembership: + # Use the pre-existing membership files + filename: "{sim_dir}/{sim_name}/SOAP-VR/membership_{snap_nr:04d}/membership_{snap_nr:04d}.{file_nr}.hdf5" + +ExtraInput: + xrays: "/cosma8/data/dp004/dc-mcgi1/FLAMINGO/Xray/{sim_name}/xray/flamingo_{snap_nr:04}/xray_{snap_nr:04}.{file_nr}.hdf5" + +HaloProperties: + # Where to write the halo properties file + filename: "{output_dir}/halo_properties_{snap_nr:04d}.hdf5" + # Where to write temporary chunk output + chunk_dir: "{scratch_dir}/SOAP-tmp/{halo_finder}" diff --git a/tests/FLAMINGO/run_L1000N1800_DMO.sh b/tests/FLAMINGO/run_L1000N1800_DMO.sh index 8cca91fe..083cf3ac 100755 --- a/tests/FLAMINGO/run_L1000N1800_DMO.sh +++ b/tests/FLAMINGO/run_L1000N1800_DMO.sh @@ -26,9 +26,14 @@ halo_indices="188656 187627 38142 14580600 159179 182418 214329 226243 187624 17 # Create parameters files python tests/FLAMINGO/create_parameters_file.py tests/FLAMINGO/parameters_DMO.yml +# Remove tmp directory (so we don't load chunks if they already exist) +rm -r output/SOAP-tmp + # Run SOAP on eight cores processing the selected halos. Use 'python3 -m pdb' to start in the debugger. -mpirun -np 8 python3 -u -m mpi4py ./compute_halo_properties.py \ +mpirun -np 8 python3 -u -m mpi4py SOAP/compute_halo_properties.py \ ./tests/FLAMINGO/test_parameters.yml \ --halo-indices ${halo_indices} \ --dmo \ + --record-halo-timings \ + --record-property-timings \ --sim-name=${sim} --snap-nr=${snapnum} --chunks=1 diff --git a/tests/FLAMINGO/run_L1000N1800_HYDRO.sh b/tests/FLAMINGO/run_L1000N1800_HYDRO.sh index e0b3ef68..85cb28ae 100755 --- a/tests/FLAMINGO/run_L1000N1800_HYDRO.sh +++ b/tests/FLAMINGO/run_L1000N1800_HYDRO.sh @@ -26,8 +26,11 @@ halo_indices="17254 18469 22841 42946 42950 63135 76467 93390 95879 109700 11015 # Create parameter file python tests/FLAMINGO/create_parameters_file.py tests/FLAMINGO/parameters_HYDRO.yml +# Remove tmp directory (so we don't load chunks if they already exist) +rm -r output/SOAP-tmp + # Run SOAP on eight cores processing the selected halos. Use 'python3 -m pdb' to start in the debugger. -mpirun -np 8 python3 -u -m mpi4py ./compute_halo_properties.py \ +mpirun -np 8 python3 -u -m mpi4py SOAP/compute_halo_properties.py \ ./tests/FLAMINGO/test_parameters.yml \ --halo-indices ${halo_indices} \ --sim-name=${sim} --snap-nr=${snapnum} --chunks=1 diff --git a/tests/FLAMINGO/run_L1000N1800_VR.sh b/tests/FLAMINGO/run_L1000N1800_VR.sh new file mode 100755 index 00000000..52584d4f --- /dev/null +++ b/tests/FLAMINGO/run_L1000N1800_VR.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# +# This runs SOAP on a few halos in the L1000N1800/HYDRO_FIDUCIAL +# box on Cosma8. It can be used as a quick test of new halo property +# code. Takes ~2 minutes to run. +# +# Should be run from the SOAP source directory. E.g.: +# +# cd SOAP +# ./tests/FLAMINGO/run_L1000N1800_VR.sh +# + +module purge +module load python/3.12.4 gnu_comp/14.1.0 openmpi/5.0.3 parallel_hdf5/1.12.3 +source openmpi-5.0.3-hdf5-1.12.3-env/bin/activate + +# Which simulation to do +sim="L1000N1800/HYDRO_FIDUCIAL" + +# Snapshot number to do +snapnum=0057 + +# Halo indices to do: all halos with x<10, y<10, and z<10Mpc in snap 57 +halo_indices="2358 2881 3633 9537 9732 14097 14639 18078 34414 38651 43121 43643 44887 45957 49461 51256 67007 68228 71121 72153 79479 80461 82525 83482 83955 89427 93408 96685 98640" + +# Create parameter file +python tests/FLAMINGO/create_parameters_file.py tests/FLAMINGO/parameters_VR.yml + +# Remove tmp directory (so we don't load chunks if they already exist) +rm -r output/SOAP-tmp + +# Run SOAP on eight cores processing the selected halos. Use 'python3 -m pdb' to start in the debugger. +mpirun -np 8 python3 -u -m mpi4py SOAP/compute_halo_properties.py \ + ./tests/FLAMINGO/test_parameters.yml \ + --halo-indices ${halo_indices} \ + --sim-name=${sim} --snap-nr=${snapnum} --chunks=1 + diff --git a/tests/cosma_run_tests.sh b/tests/cosma_run_tests.sh new file mode 100755 index 00000000..da8a1928 --- /dev/null +++ b/tests/cosma_run_tests.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -e + +module purge +module load python/3.12.4 gnu_comp/14.1.0 openmpi/5.0.3 parallel_hdf5/1.12.3 +source openmpi-5.0.3-hdf5-1.12.3-env/bin/activate + +pytest -m "not mpi" -W error +rm test_SO_radius_*.png + +mpirun -np 8 pytest -m mpi --with-mpi -W error + + diff --git a/dummy_halo_generator.py b/tests/dummy_halo_generator.py similarity index 94% rename from dummy_halo_generator.py rename to tests/dummy_halo_generator.py index b2c63b2e..73e36aa8 100644 --- a/dummy_halo_generator.py +++ b/tests/dummy_halo_generator.py @@ -15,14 +15,15 @@ import h5py import numpy as np import scipy -import unyt import types -from swift_units import unit_registry_from_snapshot -from snapshot_datasets import SnapshotDatasets from typing import Dict, Union, List, Tuple -from property_table import PropertyTable -from recently_heated_gas_filter import RecentlyHeatedGasFilter -from cold_dense_gas_filter import ColdDenseGasFilter +import unyt + +from SOAP.core.swift_units import unit_registry_from_snapshot +from SOAP.core.snapshot_datasets import SnapshotDatasets +from SOAP.property_table import PropertyTable +from SOAP.particle_filter.recently_heated_gas_filter import RecentlyHeatedGasFilter +from SOAP.particle_filter.cold_dense_gas_filter import ColdDenseGasFilter class DummySnapshot: @@ -166,6 +167,8 @@ def __init__(self): "Masses", "Velocities", "FOFGroupIDs", + "SpecificPotentialEnergies", + "GroupNr_bound", "MetalMassFractions", "Temperatures", "InternalEnergies", @@ -180,17 +183,27 @@ def __init__(self): "ElectronNumberDensities", "SpeciesFractions", "DustMassFractions", + "TotalDustMassFractions", "LastSNIIKineticFeedbackDensities", "LastSNIIThermalFeedbackDensities", "ElementMassFractionsDiffuse", "SmoothedElementMassFractions", ], - "PartType1": ["Coordinates", "Masses", "Velocities", "FOFGroupIDs"], + "PartType1": [ + "Coordinates", + "GroupNr_bound", + "Masses", + "Velocities", + "FOFGroupIDs", + "SpecificPotentialEnergies", + ], "PartType4": [ "Coordinates", "Masses", "Velocities", "FOFGroupIDs", + "GroupNr_bound", + "SpecificPotentialEnergies", "InitialMasses", "Luminosities", "MetalMassFractions", @@ -206,6 +219,8 @@ def __init__(self): "DynamicalMasses", "Velocities", "FOFGroupIDs", + "GroupNr_bound", + "SpecificPotentialEnergies", "SubgridMasses", "LastAGNFeedbackScaleFactors", "ParticleIDs", @@ -239,8 +254,8 @@ def __init__(self): } self.named_columns = { - "Luminosities": {"GAMA_r": 2}, - "SmoothedElementMassFractions": { + "PartType4/Luminosities": {"GAMA_r": 2}, + "PartType0/SmoothedElementMassFractions": { "Hydrogen": 0, "Helium": 1, "Carbon": 2, @@ -251,7 +266,7 @@ def __init__(self): "Silicon": 7, "Iron": 8, }, - "SpeciesFractions": { + "PartType0/SpeciesFractions": { "elec": 0, "HI": 1, "HII": 2, @@ -263,7 +278,7 @@ def __init__(self): "H2p": 8, "H3p": 9, }, - "DustMassFractions": { + "PartType0/DustMassFractions": { "GraphiteLarge": 0, "MgSilicatesLarge": 1, "FeSilicatesLarge": 2, @@ -272,6 +287,9 @@ def __init__(self): "FeSilicatesSmall": 5, }, } + self.named_columns["PartType4/ElementMassFractions"] = self.named_columns[ + "PartType0/SmoothedElementMassFractions" + ] self.dust_grain_composition = np.array( [ @@ -445,10 +463,10 @@ def __init__(self, reg: unyt.UnitRegistry, snap: h5py.File): Omega_k = self.cosmology["Omega_k"] Omega_Lambda = self.cosmology["Omega_lambda"] Omega_m = self.cosmology["Omega_m"] - bnx = -(Omega_k / self.a ** 2 + Omega_Lambda) / ( - Omega_k / self.a ** 2 + Omega_m / self.a ** 3 + Omega_Lambda + bnx = -(Omega_k / self.a**2 + Omega_Lambda) / ( + Omega_k / self.a**2 + Omega_m / self.a**3 + Omega_Lambda ) - self.virBN98 = 18.0 * np.pi ** 2 + 82.0 * bnx - 39.0 * bnx ** 2 + self.virBN98 = 18.0 * np.pi**2 + 82.0 * bnx - 39.0 * bnx**2 if self.virBN98 < 50.0 or self.virBN98 > 1000.0: raise RuntimeError("Invalid value for virBN98!") @@ -513,13 +531,14 @@ def get_recently_heated_gas_filter(self): self.dummy_cellgrid, 0 * unyt.Myr, False, + True, delta_logT_min=-1.0, delta_logT_max=0.3, ) @staticmethod def get_cold_dense_gas_filter(): - return ColdDenseGasFilter(3.16e4 * unyt.K, 0.1 / unyt.cm ** 3, True) + return ColdDenseGasFilter(3.16e4 * unyt.K, 0.1 / unyt.cm**3, True) @staticmethod def get_halo_result_template(particle_numbers): @@ -527,46 +546,54 @@ def get_halo_result_template(particle_numbers): Return a halo_result object which only contains the number of each particle type. """ return { - f"BoundSubhalo/{PropertyTable.full_property_list['Ngas'][0]}": ( + f"BoundSubhalo/{PropertyTable.full_property_list['Ngas'].name}": ( unyt.unyt_array( particle_numbers["PartType0"], - dtype=PropertyTable.full_property_list["Ngas"][2], + dtype=PropertyTable.full_property_list["Ngas"].dtype, units="dimensionless", ), "Dummy Ngas for filter", ), - f"BoundSubhalo/{PropertyTable.full_property_list['Ndm'][0]}": ( + f"BoundSubhalo/{PropertyTable.full_property_list['Ndm'].name}": ( unyt.unyt_array( particle_numbers["PartType1"], - dtype=PropertyTable.full_property_list["Ndm"][2], + dtype=PropertyTable.full_property_list["Ndm"].dtype, units="dimensionless", ), "Dummy Ndm for filter", ), - f"BoundSubhalo/{PropertyTable.full_property_list['Nstar'][0]}": ( + f"BoundSubhalo/{PropertyTable.full_property_list['Nstar'].name}": ( unyt.unyt_array( particle_numbers["PartType4"], - dtype=PropertyTable.full_property_list["Nstar"][2], + dtype=PropertyTable.full_property_list["Nstar"].dtype, units="dimensionless", ), "Dummy Nstar for filter", ), - f"BoundSubhalo/{PropertyTable.full_property_list['Nbh'][0]}": ( + f"BoundSubhalo/{PropertyTable.full_property_list['Nbh'].name}": ( unyt.unyt_array( particle_numbers["PartType5"], - dtype=PropertyTable.full_property_list["Nbh"][2], + dtype=PropertyTable.full_property_list["Nbh"].dtype, units="dimensionless", ), "Dummy Nbh for filter", ), - f"SO/200_crit/{PropertyTable.full_property_list['Ngas'][0]}": ( + f"SO/200_crit/{PropertyTable.full_property_list['Ngas'].name}": ( unyt.unyt_array( particle_numbers["PartType0"], - dtype=PropertyTable.full_property_list["Ngas"][2], + dtype=PropertyTable.full_property_list["Ngas"].dtype, units="dimensionless", ), "Dummy SO Ngas for filter", ), + f"BoundSubhalo/EncloseRadius": ( + unyt.unyt_array( + 100, + dtype=np.float32, + units=unyt.kpc, + ), + "Dummy enclose radius for ExclusiveSphere", + ), } @staticmethod @@ -936,6 +963,12 @@ def get_random_halo( units="snap_mass/(a**3*snap_length**3)", registry=reg, ) + data["PartType0"]["TotalDustMassFractions"] = unyt.unyt_array( + np.random.random(Ngas), + dtype=np.float32, + units=unyt.dimensionless, + registry=reg, + ) dmf = np.zeros((Ngas, 6)) dmf[:, 0] = 6.7e-3 * np.random.random(Ngas) dmf[:, 1] = 5.3e-3 * np.random.random(Ngas) @@ -957,6 +990,12 @@ def get_random_halo( data["PartType0"]["GroupNr_all"] = groupnr_all[gas_mask] data["PartType0"]["GroupNr_bound"] = groupnr_bound[gas_mask] data["PartType0"]["FOFGroupIDs"] = fof_group_ids[gas_mask] + data["PartType0"]["SpecificPotentialEnergies"] = unyt.unyt_array( + -1000 * np.random.random(Ngas), + dtype=np.float32, + units=unyt.Unit("km/s", registry=reg) ** 2, + registry=reg, + ) # we assume a fixed "snapshot" redshift of 0.1, so we make sure # the random values span a range of scale factors that is lower data["PartType0"]["LastAGNFeedbackScaleFactors"] = unyt.unyt_array( @@ -1097,6 +1136,12 @@ def get_random_halo( data["PartType1"]["Masses"] = mass[dm_mask] Mtot += data["PartType1"]["Masses"].sum() data["PartType1"]["Velocities"] = vs[dm_mask] + data["PartType1"]["SpecificPotentialEnergies"] = unyt.unyt_array( + -1000 * np.random.random(Ndm), + dtype=np.float32, + units=unyt.Unit("km/s", registry=reg) ** 2, + registry=reg, + ) # star properties star_mask = types == "PartType4" @@ -1146,6 +1191,12 @@ def get_random_halo( units=unyt.dimensionless, registry=reg, ) + data["PartType4"]["SpecificPotentialEnergies"] = unyt.unyt_array( + -1000 * np.random.random(Nstar), + dtype=np.float32, + units=unyt.Unit("km/s", registry=reg) ** 2, + registry=reg, + ) # all entries in the element mass fractions have their own limits, # so we need to generate those separately if we want a realistic # sample @@ -1312,6 +1363,12 @@ def get_random_halo( registry=reg, ) data["PartType5"]["Velocities"] = vs[bh_mask] + data["PartType5"]["SpecificPotentialEnergies"] = unyt.unyt_array( + -1000 * np.random.random(Nbh), + dtype=np.float32, + units=unyt.Unit("km/s", registry=reg) ** 2, + registry=reg, + ) # Neutrino properties nu_mask = types == "PartType6" @@ -1346,10 +1403,10 @@ def get_random_halo( / self.dummy_cellgrid.cosmology["H [internal units]"] ) ** 2 - / self.dummy_cellgrid.a ** 3 + / self.dummy_cellgrid.a**3 ) - Mtot += nu_density * 4.0 * np.pi / 3.0 * rmax ** 3 + Mtot += nu_density * 4.0 * np.pi / 3.0 * rmax**3 particle_numbers = { "PartType0": Ngas, diff --git a/tests/generate_test_data/HBT_config.txt b/tests/generate_test_data/HBT_config.txt new file mode 100644 index 00000000..f967d04c --- /dev/null +++ b/tests/generate_test_data/HBT_config.txt @@ -0,0 +1,26 @@ +# Configuration file used to run on small DMO simulation + +# Compulsary Params +SnapshotPath ./ # Location of snapshots +SnapshotFileBase snap # Basename of snapshot files +HaloPath ./ # Location of FOF ID files (normal snapshot for SWIFT) +SubhaloPath HBT_output # Output directory +ParticlesSplit 0 # Do we have gas particles that split +MergeTrappedSubhalos 1 # Allow subhalos to merge +# For the swiftsim reader these will be read in from snapshots automatically +BoxSize -1 +SofteningHalo -1 +MaxPhysicalSofteningHalo -1 + +# Reader +SnapshotFormat swiftsim +GroupFileFormat swiftsim_particle_index +MinSnapshotIndex 0 # First snapshot to use +MaxSnapshotIndex 18 # Final snapshot to use +MaxConcurrentIO 8 # Number of cores for IO +MinNumPartOfSub 20 # Minimum number of particles in a subhalo + +# Units +MassInMsunh 6.81e9 # Removes h factors from the final output +LengthInMpch 0.681 # Removes h factors from the final output +VelInKmS 1 diff --git a/tests/generate_test_data/README.md b/tests/generate_test_data/README.md new file mode 100644 index 00000000..a9eac917 --- /dev/null +++ b/tests/generate_test_data/README.md @@ -0,0 +1,6 @@ +This directory contains a number of scripts to generate halo catalogues from the test simulation. + +- `./download_sim.sh` - Downloads a small DMO simulation taken from [this notebook](https://github.com/robjmcgibbon/SWIFTCON2024/) +- `./run_HBT.sh` - Runs HBT on the test simulation +- `./run_VR.sh` - Runs VR on the test simulation + diff --git a/tests/generate_test_data/VR_config.cfg b/tests/generate_test_data/VR_config.cfg new file mode 100644 index 00000000..c78e73a0 --- /dev/null +++ b/tests/generate_test_data/VR_config.cfg @@ -0,0 +1,211 @@ +#suggested configuration file for cosmological dm and subhalo catalog +#Configuration file for analysing all particles +#runs 3DFOF algorithm, calculates many properties +#Units currently set to take in as input, Mpc, 1e10 solar masses, km/s, output in same units +#To set temporally unique halo ids, alter Snapshot_value=SNAP to appropriate value. Ie: for snapshot 12, change SNAP to 12 +#Script calculates several aperture quantities, also several spherical overensity. Currently does NOT +#write all the particles within the lowest spherical density. + +################################ +#input options +#set up to use SWIFT HDF input, load dark matter only +################################ +HDF_name_convention=6 #HDF SWIFT naming convention +Input_includes_dm_particle=1 #include dark matter particles in hydro input +Input_includes_gas_particle=0 #include gas particles in hydro input +Input_includes_star_particle=0 #include star particles in hydro input +Input_includes_bh_particle=0 #include bh particles in hydro input +Input_includes_wind_particle=0 #include wind particles in hydro input (used by Illustris and moves particle type 0 to particle type 3 when decoupled from hydro forces). Here shown as example +Input_includes_tracer_particle=0 #include tracer particles in hydro input (used by Illustris). Here shown as example +Input_includes_extradm_particle=0 #include extra dm particles stored in particle type 2 and type 3, useful for zooms + +#cosmological run +Cosmological_input=1 + +################################ +#unit options, should always be provided +################################ + +#units conversion from input input to desired internal unit +Length_input_unit_conversion_to_output_unit=1.0 #default code unit, +Velocity_input_unit_conversion_to_output_unit=1.0 #default velocity unit, +Mass_input_unit_conversion_to_output_unit=1.0 #default mass unit, +#assumes input is in 1e10 msun, Mpc and km/s and output units are the same +#converting hydro quantities +Stellar_age_input_is_cosmological_scalefactor=1 +Metallicity_input_unit_conversion_to_output_unit=1.0 +Stellar_age_input_unit_conversion_to_output_unit=1.0 +Star_formation_rate_input_unit_conversion_to_output_unit=1.0 + +#set the units of the output by providing conversion to a defined unit +#conversion of output length units to kpc +Length_unit_to_kpc=1000.0 +#conversion of output velocity units to km/s +Velocity_to_kms=1.0 +#conversion of output mass units to solar masses +Mass_to_solarmass=1.0e10 +Metallicity_to_solarmetallicity=1.0 +Star_formation_rate_to_solarmassperyear=1.0 +Stellar_age_to_yr=1.0 +#ensures that output is physical and not comoving distances per little h +Comoving_units=0 + +#sets the total buffer size in bytes used to store temporary particle information +#of mpi read threads before they are broadcast to the appropriate waiting non-read threads +#if not set, default value is equivalent to 1e6 particles per mpi process, quite large +#but significantly minimises the number of send/receives +#in this example the buffer size is roughly that for a send/receive of 10000 particles +#for 100 mpi processes +MPI_particle_total_buf_size=100000000 + +################################ +#search related options +################################ + +#how to search a simulation +Particle_search_type=2 #search all particles, see allvars for other types +#for baryon search +Baryon_searchflag=0 #if 1 search for baryons separately using phase-space search when identifying substructures, 2 allows special treatment in field FOF linking and phase-space substructure search, 0 treat the same as dark matter particles +#for search for substruture +Search_for_substructure=1 #if 0, end search once field objects are found +#also useful for zoom simulations or simulations of individual objects, setting this flag means no field structure search is run +Singlehalo_search=0 #if file is single halo in which one wishes to search for substructure +#additional option for field haloes +Keep_FOF=0 #if field 6DFOF search is done, allows to keep structures found in 3DFOF (can be interpreted as the inter halo stellar mass when only stellar search is used).\n + +#minimum size for structures +Minimum_size=20 #min 20 particles +Minimum_halo_size=35 #if field halos have different minimum sizes, otherwise set to -1. + +#for field fof halo search +FoF_Field_search_type=5 #5 3DFOF search for field halos, 4 for 6DFOF clean up of field halos, 3 for 6DFOF with velocity scale distinct for each halo +Halo_3D_linking_length=0.20 #3DFOF linking length in interparticle spacing + +#for mean field estimates and local velocity density distribution funciton estimator related quantiites, rarely need to change this +Cell_fraction = 0.01 #fraction of field fof halo used to determine mean velocity distribution function. Typical values are ~0.005-0.02 +Grid_type=1 #normal entropy based grid, shouldn't have to change +Nsearch_velocity=32 #number of velocity neighbours used to calculate local velocity distribution function. Typial values are ~32 +Nsearch_physical=256 #numerof physical neighbours from which the nearest velocity neighbour set is based. Typical values are 128-512 +Local_velocity_density_approximate_calculation=2 #approximative and mpi local calculation of density, less accurate much faster. + +#for substructure search, rarely ever need to change this +FoF_search_type=1 #default phase-space FOF search. Don't really need to change +Iterative_searchflag=1 #iterative substructure search, for substructure find initial candidate substructures with smaller linking lengths then expand search region +Outlier_threshold=2.5 #outlier threshold for a particle to be considered residing in substructure, that is how dynamically distinct a particle is. Typical values are >2 +Substructure_physical_linking_length=0.10 #physical linking length. IF reading periodic volumes in gadget/hdf/ramses, in units of the effective inter-particle spacing. Otherwise in user defined code units. Here set to 0.10 as iterative flag one, values of 0.1-0.3 are typical. +Velocity_ratio=2.0 #ratio of speeds used in phase-space FOF +Velocity_opening_angle=0.10 #angle between velocities. 18 degrees here, typical values are ~10-30 +Velocity_linking_length=0.20 #where scaled by structure dispersion +Significance_level=1.0 #how significant a substructure is relative to Poisson noise. Values >= 1 are fine. + +#for iterative substructure search, rarely ever need to change this +Iterative_threshold_factor=1.0 #change in threshold value when using iterative search. Here no increase in threshold if iterative or not +Iterative_linking_length_factor=2.0 #increase in final linking final iterative substructure search will be sqrt(2.25)*this factor +Iterative_Vratio_factor=1.0 #change in Vratio when using iterative search. no change in vratio +Iterative_ThetaOp_factor=1.0 #change in velocity opening angle. no change in velocity opening angle + +#for checking for halo merger remnants, which are defined as large, well separated phase-space density maxima +Halo_core_search=2 # searches for separate 6dfof cores in field haloes, and then more than just flags halo as merging, assigns particles to each merging "halo". 2 is full separation, 1 is flagging, 0 is off +#if searching for cores, linking lengths. likely does not need to change much +Use_adaptive_core_search=0 #calculate dispersions in configuration & vel space to determine linking lengths +Use_phase_tensor_core_growth=2 #use full stepped phase-space tensor assignment +Halo_core_ellx_fac=0.7 #how linking lengths are changed when searching for local 6DFOF cores, +Halo_core_ellv_fac=2.0 #how velocity lengths based on dispersions are changed when searching for local 6DFOF cores +Halo_core_ncellfac=0.005 #fraction of total halo particle number setting min size of a local 6DFOF core +Halo_core_num_loops=8 #number of loops to iteratively search for cores +Halo_core_loop_ellx_fac=0.75 #how much to change the configuration space linking per iteration +Halo_core_loop_ellv_fac=1.0 #how much to change the velocity space linking per iteration +Halo_core_loop_elln_fac=1.2 #how much to change the min number of particles per iteration +Halo_core_phase_significance=2.0 #how significant a core must be in terms of dispersions (sigma) significance + +#merge substructures if the overlap in phase-space by some fraction of their dispersion +#here distance has to be less than 0.25 sigma +Structure_phase_merge_dist=0.25 +#also merge structures with background if overlap heavily in phase-space based on dispersions. +Apply_phase_merge_to_host=1 + +################################ +#Unbinding options (VELOCIraptor is able to accurately identify tidal debris so particles need not be bound to a structure) +################################ + +#unbinding related items +Unbind_flag=1 #run unbinding +#objects must have particles that meet the allowed kinetic to potential ratio AND also have some total fraction that are completely bound. +Unbinding_type=0 +#alpha factor used to determine whether particle is "bound" alaph*T+W<0. For standard subhalo catalogues use >0.9 but if interested in tidal debris 0.2-0.5 +Allowed_kinetic_potential_ratio=0.95 +Min_bound_mass_frac=0.65 #minimum bound mass fraction +#run unbinding of field structures, aka halos. This is useful for sams and 6DFOF halos but may not be useful if interested in 3DFOF mass functions. +Bound_halos=0 +#don't keep background potential when unbinding +Keep_background_potential=1 +#use all particles to determine velocity frame for unbinding +Frac_pot_ref=1.0 +Min_npot_ref=20 +#reference frame only meaningful if calculating velocity frame using subset of particles in object. Can use radially sorted fraction of particles about minimum potential or centre of mass +Kinetic_reference_frame_type=0 + +################################ +#Cosmological parameters +#this is typically overwritten by information in the gadget/hdf header if those input file types are read +################################ +h_val=1.0 +Omega_m=0.3 +Omega_Lambda=0.7 +Critical_density=1.0 +Virial_density=200 #so-called virial overdensity value +Omega_b=0. #no baryons + +################################ +#Calculation of properties related options +################################ +Inclusive_halo_masses=3 #calculate inclusive masses for halos using full Spherical overdensity apertures once all substructures have been found (if substructures are searched for). +#when calculating properties, for field objects calculate inclusive masses +Iterate_cm_flag=0 #do not interatively find the centre-of-mass, giving bulk centre of mass and centre of mass velocity. +Sort_by_binding_energy=1 #sort by binding energy +Reference_frame_for_properties=2 #use the position of the particle with the minimum potential as the point about which properties should be calculated. +#calculate more (sub)halo properties (like angular momentum in spherical overdensity apertures, both inclusive and exclusive) +Extensive_halo_properties_output=1 + +#aperture related (list must be in increasing order and terminates with , ie: 1,2,3, ) +#calculate aperture masses +Calculate_aperture_quantities=1 +Number_of_apertures=6 +Aperture_values_in_kpc=3,5,10,30,50,100, +Number_of_projected_apertures=3 +Projected_aperture_values_in_kpc=10,50,100, + +#spherical overdensity related quantities +Virial_density=500 #user defined virial overdensity. Note that 200 rho_c, 200 rho_m and BN98 are already calculated. +#number of spherical overdensity thresholds +Number_of_overdensities=5 +Overdensity_values_in_critical_density=25,100,500,1000,2500, + +#calculate radial profiles +Calculate_radial_profiles=1 +Number_of_radial_profile_bin_edges=20 +#default radial normalisation log rad bins, in proper kpc +Radial_profile_norm=0 +Radial_profile_bin_edges=-2.,-1.87379263,-1.74758526,-1.62137789,-1.49517052,-1.36896316,-1.24275579,-1.11654842,-0.99034105,-0.86413368,-0.73792631,-0.61171894,-0.48551157,-0.3593042,-0.23309684,-0.10688947,0.0193179,0.14552527,0.27173264,0.39794001, + +################################ +#output related +################################ + +Write_group_array_file=0 #write a group array file +Separate_output_files=0 #separate output into field and substructure files similar to subfind +Binary_output=2 #binary output 1, ascii 0, and HDF 2 +#do not output particles residing in the spherical overdensity apertures of halos, only the particles exclusively belonging to halos +Spherical_overdensity_halo_particle_list_output=0 + +#halo ids are adjusted by this value * 1000000000000 (or 1000000 if code compiled with the LONGINTS option turned off) +#to ensure that halo ids are temporally unique. So if you had 100 snapshots, for snap 100 set this to 100 and 100*1000000000000 will +#be added to the halo id as set for this snapshot, so halo 1 becomes halo 100*1000000000000+1 and halo 1 of snap 0 would just have ID=1 + +#ALTER THIS as part of a script to get temporally unique ids +Snapshot_value=SNAP + +################################ +#other options +################################ +Verbose=0 #how talkative do you want the code to be, 0 not much, 1 a lot, 2 chatterbox diff --git a/tests/generate_test_data/download_sim.sh b/tests/generate_test_data/download_sim.sh new file mode 100755 index 00000000..218417b8 --- /dev/null +++ b/tests/generate_test_data/download_sim.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +wget https://ftp.strw.leidenuniv.nl/mcgibbon/SOAP/swift_output/fof_output_0018.hdf5 +for i in {0..18}; do + snap_nr=$(printf "%04d" $i) + wget "https://ftp.strw.leidenuniv.nl/mcgibbon/SOAP/swift_output/snap_${snap_nr}.hdf5" +done + diff --git a/tests/generate_test_data/run_HBT.sh b/tests/generate_test_data/run_HBT.sh new file mode 100755 index 00000000..be9477f2 --- /dev/null +++ b/tests/generate_test_data/run_HBT.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +# Modules for cosma +module purge +module load gnu_comp/13.1.0 hdf5/1.12.2 openmpi/4.1.4 + +# Clone and compile code +git clone https://github.com/SWIFTSIM/HBTplus.git +cd HBTplus +cmake -B$PWD/build -D HBT_USE_OPENMP=ON -D HBT_DM_ONLY=ON -D HBT_UNSIGNED_LONG_ID_OUTPUT=OFF +cd build +make -j 4 +cd ../.. + +# Run HBT +export OMP_NUM_THREADS=8 +mpirun -np 1 ./HBTplus/build/HBT HBT_config.txt + diff --git a/tests/generate_test_data/run_VR.sh b/tests/generate_test_data/run_VR.sh new file mode 100755 index 00000000..3f7bb00d --- /dev/null +++ b/tests/generate_test_data/run_VR.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +module purge +module load gnu_comp/13.1.0 hdf5/1.12.2 +module load gsl +# module load gnu_comp/14.1.0 openmpi/5.0.3 parallel_hdf5/1.12.3 + +git clone https://github.com/ICRAR/VELOCIraptor-STF.git +cd VELOCIraptor-STF +git submodule update --init --recursive +mkdir build +cd build +# cmake ../ -DVR_USE_GAS=ON -DVR_USE_STAR=ON -DV_USE_BH=ON +cmake ../ -DVR_USE_HYDRO=FALSE -DCMAKE_CXX_FLAGS="-fPIC" -DCMAKE_BUILD_TYPE=Release +make -j 4 +cd ../.. + +export OMP_NUM_THREADS=8 +./VELOCIraptor-STF/build/stf -i snap_0018 -o vr_018 -I 2 -C VR_config.cfg + +# Pretend we ran with mpi +mv vr_018.properties vr_018.properties.0 +mv vr_018.catalog_particles.unbound vr_018.catalog_particles.unbound.0 +mv vr_018.catalog_particles vr_018.catalog_particles.0 +mv vr_018.catalog_groups vr_018.catalog_groups.0 diff --git a/tests/helpers.py b/tests/helpers.py new file mode 100644 index 00000000..8e2f1d8c --- /dev/null +++ b/tests/helpers.py @@ -0,0 +1,94 @@ +""" +Contains helper functions for downloading test data +""" + +import os +import subprocess + +import pytest + +webstorage_location = "https://ftp.strw.leidenuniv.nl/mcgibbon/SOAP/" +test_output_location = "test_data/" + + +def requires(filepaths, comm=None): + """ + Use this as a decorator around tests that require data. + + Can be passed either a single filepath, or a list of filepaths + + If running with MPI then pass the comm so that only a single + rank will download the data. + """ + + # First check if the test data directory exists + if (comm is None) or (comm.Get_rank() == 1): + if not os.path.exists(test_output_location): + os.mkdir(test_output_location) + + # Handle case where we are passed a single path instead of a list + if isinstance(filepaths, str): + filepaths = [filepaths] + return_str = True + else: + return_str = False + + output_locations = [] + for filepath in filepaths: + filename = os.path.basename(filepath) + output_location = f"{test_output_location}{filename}" + output_locations.append(output_location) + + if (comm is not None) and (comm.Get_rank() != 0): + file_available = None + else: + if not os.path.exists(output_location): + # Download the file if it doesn't exist + ret = subprocess.call( + ["wget", f"{webstorage_location}{filepath}", "-O", output_location] + ) + + if ret != 0: + Warning(f"Unable to download file at {filepath}") + # It wrote an empty file, kill it. + subprocess.call(["rm", output_location]) + file_available = False + else: + file_available = True + else: + file_available = True + + if comm is not None: + file_available = comm.bcast(file_available) + + if not file_available: + + def dont_call_test(func): + def empty(*args, **kwargs): + return pytest.skip() + + return empty + + return dont_call_test + + # Return a single path if that's what we were passed + if return_str: + output_locations = output_locations[0] + + # We can do the test! + def do_call_test(func): + def final_test(): + return func(output_locations) + + return final_test + + return do_call_test + + +if __name__ == "__main__": + # Download the data required for run_small_volume.sh + # Call @requires by passing a dummy function + dummy = lambda x: x + requires("swift_output/fof_output_0018.hdf5")(dummy)() + requires("swift_output/snap_0018.hdf5")(dummy)() + requires("HBT_output/018/SubSnap_018.0.hdf5")(dummy)() diff --git a/tests/run_small_volume.sh b/tests/run_small_volume.sh new file mode 100755 index 00000000..4bd371d6 --- /dev/null +++ b/tests/run_small_volume.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# +# This runs SOAP on a small DMO box +# +set -e + +# Load the correct modules if we're running on cosma +if [[ $(hostname) == *cosma* ]] ; then + module purge + module load python/3.12.4 gnu_comp/14.1.0 openmpi/5.0.3 parallel_hdf5/1.12.3 + source openmpi-5.0.3-hdf5-1.12.3-env/bin/activate +fi + +# Download the required data +python tests/helpers.py + +# Run the group membership script +mpirun -np 8 python -u SOAP/group_membership.py \ + --sim-name=DM_test \ + --snap-nr=18 \ + tests/small_volume.yml + +# Calculate halo properties +mpirun -np 8 python -u SOAP/compute_halo_properties.py \ + --sim-name=DM_test \ + --snap-nr=18 \ + --chunks=1 \ + --dmo \ + tests/small_volume.yml + +# Generate documentation +python SOAP/property_table.py \ + tests/small_volume.yml \ + test_data/snap_0018.hdf5 +cd documentation +pdflatex -halt-on-error SOAP.tex +pdflatex -halt-on-error SOAP.tex +cd .. diff --git a/tests/run_tests.sh b/tests/run_tests.sh deleted file mode 100755 index f5315ba7..00000000 --- a/tests/run_tests.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -set -e - -module purge -module load python/3.12.4 gnu_comp/14.1.0 openmpi/5.0.3 parallel_hdf5/1.12.3 -source openmpi-5.0.3-hdf5-1.12.3-env/bin/activate - -pytest -W error aperture_properties.py -pytest -W error half_mass_radius.py -pytest -W error projected_aperture_properties.py -pytest -W error SO_properties.py -pytest -W error subhalo_properties.py -pytest -W error test_SO_radius_calculation.py -rm test_SO_radius_*.png - -mpirun -np 8 pytest -W error --with-mpi shared_mesh.py -mpirun -np 8 pytest -W error --with-mpi subhalo_rank.py -# Running without -W error since VirgoDC triggers a warning -mpirun -np 8 pytest --with-mpi read_vr.py -mpirun -np 8 pytest -W error --with-mpi io_test.py -rm io_test.png - -# TODO: Add persistent data for these tests -#mpirun -np 8 pytest -W error --with-mpi read_subfind.py -#mpirun -np 8 pytest -W error --with-mpi read_rockstar.py diff --git a/tests/small_volume.yml b/tests/small_volume.yml new file mode 100644 index 00000000..4180ea9a --- /dev/null +++ b/tests/small_volume.yml @@ -0,0 +1,100 @@ +Parameters: + sim_dir: ./test_data + output_dir: ./output + scratch_dir: ./output + +Snapshots: + filename: "{sim_dir}/snap_{snap_nr:04d}.hdf5" + +HaloFinder: + type: HBTplus + filename: "{sim_dir}/SubSnap_{snap_nr:03d}" + fof_filename: "{sim_dir}/fof_output_{snap_nr:04d}.hdf5" + # type: VR + # filename: "{sim_dir}/vr_{snap_nr:03d}" + +GroupMembership: + filename: "{output_dir}/membership_{snap_nr:04d}/membership_{snap_nr:04d}.hdf5" + +HaloProperties: + filename: "{output_dir}/halo_properties_{snap_nr:04d}.hdf5" + chunk_dir: "{scratch_dir}/SOAP-tmp/{halo_finder}/" + +ApertureProperties: + properties: + {} + variations: + {} +ProjectedApertureProperties: + properties: + {} + variations: + {} +SOProperties: + properties: + CentreOfMass: true + CentreOfMassVelocity: true + Concentration: true + ConcentrationUnsoftened: true + MassFractionSatellites: true + MassFractionExternal: true + MaximumCircularVelocity: true + MaximumCircularVelocityRadius: true + NumberOfDarkMatterParticles: true + SORadius: true + SpinParameter: true + TotalMass: true + variations: + 200_crit: + type: crit + value: 200.0 + 200_mean: + type: mean + value: 200.0 + 500_crit: + type: crit + value: 500.0 +SubhaloProperties: + properties: + CentreOfMass: true + CentreOfMassVelocity: true + EncloseRadius: true + NumberOfDarkMatterParticles: true + NumberOfGasParticles: true + NumberOfStarParticles: true + NumberOfBlackHoleParticles: true + MaximumCircularVelocity: true + MaximumCircularVelocityUnsoftened: true + MaximumCircularVelocityRadiusUnsoftened: true + SpinParameter: true + TotalMass: true +filters: + general: + limit: 100 + properties: + - BoundSubhalo/NumberOfGasParticles + - BoundSubhalo/NumberOfDarkMatterParticles + - BoundSubhalo/NumberOfStarParticles + - BoundSubhalo/NumberOfBlackHoleParticles + combine_properties: sum + baryon: + limit: 100 + properties: + - BoundSubhalo/NumberOfGasParticles + - BoundSubhalo/NumberOfStarParticles + combine_properties: sum + dm: + limit: 100 + properties: + - BoundSubhalo/NumberOfDarkMatterParticles + gas: + limit: 100 + properties: + - BoundSubhalo/NumberOfGasParticles + star: + limit: 100 + properties: + - BoundSubhalo/NumberOfStarParticles +calculations: + calculate_missing_properties: false + min_read_radius_cmpc: 5 diff --git a/tests/test_SO_properties.py b/tests/test_SO_properties.py new file mode 100644 index 00000000..1ba8c03c --- /dev/null +++ b/tests/test_SO_properties.py @@ -0,0 +1,457 @@ +import pytest +import numpy as np +import unyt + +from SOAP.core.category_filter import CategoryFilter +from SOAP.core.parameter_file import ParameterFile +from SOAP.particle_selection.halo_properties import SearchRadiusTooSmallError +from SOAP.particle_selection.SO_properties import ( + SOProperties, + RadiusMultipleSOProperties, +) + +from dummy_halo_generator import DummyHaloGenerator + + +def test_SO_properties_random_halo(): + """ + Unit test for the SO property calculation. + + We generate 100 random halos and check that the various SO halo + calculations return the expected results and do not lead to any + errors. + """ + from dummy_halo_generator import DummyHaloGenerator + + dummy_halos = DummyHaloGenerator(4251) + gas_filter = dummy_halos.get_recently_heated_gas_filter() + cat_filter = CategoryFilter(dummy_halos.get_filters({"general": 100})) + parameters = ParameterFile( + parameter_dictionary={ + "aliases": { + "PartType0/ElementMassFractions": "PartType0/SmoothedElementMassFractions", + "PartType4/ElementMassFractions": "PartType4/SmoothedElementMassFractions", + "PartType0/XrayLuminositiesRestframe": "PartType0/XrayLuminositiesRestframe", + "PartType0/XrayPhotonLuminositiesRestframe": "PartType0/XrayPhotonLuminositiesRestframe", + } + } + ) + dummy_halos.get_cell_grid().snapshot_datasets.setup_aliases( + parameters.get_aliases() + ) + parameters.get_halo_type_variations( + "SOProperties", + { + "50_kpc": {"value": 50.0, "type": "physical"}, + "2500_mean": {"value": 2500.0, "type": "mean"}, + "2500_crit": {"value": 2500.0, "type": "crit"}, + "BN98": {"value": 0.0, "type": "BN98"}, + "5xR2500_mean": {"value": 2500.0, "type": "mean", "radius_multiple": 5.0}, + }, + ) + + property_calculator_50kpc = SOProperties( + dummy_halos.get_cell_grid(), + parameters, + gas_filter, + cat_filter, + "basic", + 50.0, + "physical", + ) + property_calculator_2500mean = SOProperties( + dummy_halos.get_cell_grid(), + parameters, + gas_filter, + cat_filter, + "basic", + 2500.0, + "mean", + ) + property_calculator_2500crit = SOProperties( + dummy_halos.get_cell_grid(), + parameters, + gas_filter, + cat_filter, + "basic", + 2500.0, + "crit", + ) + property_calculator_BN98 = SOProperties( + dummy_halos.get_cell_grid(), + parameters, + gas_filter, + cat_filter, + "basic", + 0.0, + "BN98", + ) + property_calculator_5x2500mean = RadiusMultipleSOProperties( + dummy_halos.get_cell_grid(), + parameters, + gas_filter, + cat_filter, + "basic", + 2500.0, + 5.0, + "mean", + ) + + # Create a filter that no halos will satisfy + fail_filter = CategoryFilter(dummy_halos.get_filters({"general": 10000000})) + property_calculator_filter_test = SOProperties( + dummy_halos.get_cell_grid(), + parameters, + gas_filter, + fail_filter, + "general", + 200.0, + "crit", + ) + property_calculator_filter_test.SO_name = "filter_test" + property_calculator_filter_test.group_name = "SO/filter_test" + + for i in range(100): + ( + input_halo, + data, + rmax, + Mtot, + Npart, + particle_numbers, + ) = dummy_halos.get_random_halo([10, 100, 1000, 10000], has_neutrinos=True) + halo_result_template = dummy_halos.get_halo_result_template(particle_numbers) + rho_ref = Mtot / (4.0 / 3.0 * np.pi * rmax**3) + + # force the SO radius to be outside the search sphere and check that + # we get a SearchRadiusTooSmallError + property_calculator_2500mean.reference_density = 0.01 * rho_ref + property_calculator_2500crit.reference_density = 0.01 * rho_ref + property_calculator_BN98.reference_density = 0.01 * rho_ref + for prop_calc in [ + property_calculator_2500mean, + property_calculator_2500crit, + property_calculator_BN98, + ]: + fail = False + try: + halo_result = dict(halo_result_template) + prop_calc.calculate(input_halo, rmax, data, halo_result) + except SearchRadiusTooSmallError: + fail = True + # 1 particle halos don't fail, since we always assume that the first + # particle is at the centre of potential (which means we exclude it + # in the SO calculation) + # non-centrals don't fail, since we do not calculate any SO + # properties and simply return zeros in this case + + # TODO: This can fail due to how we calculate the SO if the + # first particle is a neutrino with negative mass. In that case + # we linearly interpolate the mass of the first non-negative particle + # outwards. + # TODO + # assert (Npart == 1) or input_halo["is_central"] == 0 or fail + + # force the radius multiple to trip over not having computed the + # required radius + fail = False + try: + halo_result = dict(halo_result_template) + property_calculator_5x2500mean.calculate( + input_halo, rmax, data, halo_result + ) + except RuntimeError: + fail = True + assert fail + + # force the radius multiple to trip over the search radius + fail = False + try: + halo_result = dict(halo_result_template) + halo_result.update( + { + f"SO/2500_mean/{property_calculator_5x2500mean.radius_name}": ( + 0.1 * rmax, + "Dummy value.", + ) + } + ) + property_calculator_5x2500mean.calculate( + input_halo, 0.2 * rmax, data, halo_result + ) + except SearchRadiusTooSmallError: + fail = True + assert fail + + # force the SO radius to be within the search sphere + property_calculator_2500mean.reference_density = 2.0 * rho_ref + property_calculator_2500crit.reference_density = 2.0 * rho_ref + property_calculator_BN98.reference_density = 2.0 * rho_ref + + for SO_name, prop_calc in [ + ("50_kpc", property_calculator_50kpc), + ("2500_mean", property_calculator_2500mean), + ("2500_crit", property_calculator_2500crit), + ("BN98", property_calculator_BN98), + ("5xR_2500_mean", property_calculator_5x2500mean), + ("filter_test", property_calculator_filter_test), + ]: + halo_result = dict(halo_result_template) + # make sure the radius multiple is found this time + if SO_name == "5xR_2500_mean": + halo_result[ + f"SO/2500_mean/{property_calculator_5x2500mean.radius_name}" + ] = (0.1 * rmax, "Dummy value to force correct behaviour") + input_data = {} + for ptype in prop_calc.particle_properties: + if ptype in data: + input_data[ptype] = {} + for dset in prop_calc.particle_properties[ptype]: + input_data[ptype][dset] = data[ptype][dset] + # TODO: remove this + # Adding Restframe luminosties as they are calculated in halo_tasks + if "PartType0" in input_data: + for dset in [ + "XrayLuminositiesRestframe", + "XrayPhotonLuminositiesRestframe", + ]: + input_data["PartType0"][dset] = data["PartType0"][dset] + input_data["PartType0"][dset] = data["PartType0"][dset] + halo_result[ + f"SO/2500_mean/{property_calculator_5x2500mean.radius_name}" + ] = (0.1 * rmax, "Dummy value to force correct behaviour") + input_halo_copy = input_halo.copy() + input_data_copy = input_data.copy() + prop_calc.calculate(input_halo, rmax, input_data, halo_result) + # make sure the calculation does not change the input + assert input_halo_copy == input_halo + assert input_data_copy == input_data + + for prop in prop_calc.property_list.values(): + outputname = prop.name + size = prop.shape + dtype = prop.dtype + unit_string = prop.unit + full_name = f"SO/{SO_name}/{outputname}" + assert full_name in halo_result + result = halo_result[full_name][0] + assert (len(result.shape) == 0 and size == 1) or result.shape[0] == size + assert result.dtype == dtype + unit = unyt.Unit(unit_string, registry=dummy_halos.unit_registry) + assert result.units.same_dimensions_as(unit.units) + + # Check properties were not calculated for filtered halos + if SO_name == "filter_test": + for prop in prop_calc.property_list.values(): + outputname = prop.name + size = prop.shape + full_name = f"SO/{SO_name}/{outputname}" + assert np.all(halo_result[full_name][0].value == np.zeros(size)) + + # Now test the calculation for each property individually, to make sure that + # all properties read all the datasets they require + all_parameters = parameters.get_parameters() + for property in all_parameters["SOProperties"]["properties"]: + print(f"Testing only {property}...") + single_property = dict(all_parameters) + for other_property in all_parameters["SOProperties"]["properties"]: + single_property["SOProperties"]["properties"][other_property] = ( + other_property == property + ) or other_property.startswith("NumberOf") + single_parameters = ParameterFile(parameter_dictionary=single_property) + + property_calculator_50kpc = SOProperties( + dummy_halos.get_cell_grid(), + single_parameters, + gas_filter, + cat_filter, + "basic", + 50.0, + "physical", + ) + property_calculator_2500mean = SOProperties( + dummy_halos.get_cell_grid(), + single_parameters, + gas_filter, + cat_filter, + "basic", + 2500.0, + "mean", + ) + property_calculator_2500crit = SOProperties( + dummy_halos.get_cell_grid(), + single_parameters, + gas_filter, + cat_filter, + "basic", + 2500.0, + "crit", + ) + property_calculator_BN98 = SOProperties( + dummy_halos.get_cell_grid(), + single_parameters, + gas_filter, + cat_filter, + "basic", + 0.0, + "BN98", + ) + property_calculator_5x2500mean = RadiusMultipleSOProperties( + dummy_halos.get_cell_grid(), + single_parameters, + gas_filter, + cat_filter, + "basic", + 2500.0, + 5.0, + "mean", + ) + + halo_result_template = dummy_halos.get_halo_result_template(particle_numbers) + rho_ref = Mtot / (4.0 / 3.0 * np.pi * rmax**3) + + # force the SO radius to be within the search sphere + property_calculator_2500mean.reference_density = 2.0 * rho_ref + property_calculator_2500crit.reference_density = 2.0 * rho_ref + property_calculator_BN98.reference_density = 2.0 * rho_ref + + for SO_name, prop_calc in [ + ("50_kpc", property_calculator_50kpc), + ("2500_mean", property_calculator_2500mean), + ("2500_crit", property_calculator_2500crit), + ("BN98", property_calculator_BN98), + ("5xR_2500_mean", property_calculator_5x2500mean), + ]: + + halo_result = dict(halo_result_template) + # make sure the radius multiple is found this time + if SO_name == "5xR_2500_mean": + halo_result[ + f"SO/2500_mean/{property_calculator_5x2500mean.radius_name}" + ] = (0.1 * rmax, "Dummy value to force correct behaviour") + input_data = {} + for ptype in prop_calc.particle_properties: + if ptype in data: + input_data[ptype] = {} + for dset in prop_calc.particle_properties[ptype]: + input_data[ptype][dset] = data[ptype][dset] + # Adding Restframe luminosties as they are calculated in halo_tasks + if "PartType0" in input_data: + for dset in [ + "XrayLuminositiesRestframe", + "XrayPhotonLuminositiesRestframe", + ]: + input_data["PartType0"][dset] = data["PartType0"][dset] + input_data["PartType0"][dset] = data["PartType0"][dset] + input_halo_copy = input_halo.copy() + input_data_copy = input_data.copy() + prop_calc.calculate(input_halo, rmax, input_data, halo_result) + # make sure the calculation does not change the input + assert input_halo_copy == input_halo + assert input_data_copy == input_data + + for prop in prop_calc.property_list.values(): + outputname = prop.name + if not outputname == property: + continue + size = prop.shape + dtype = prop.dtype + unit_string = prop.unit + physical = prop.output_physical + a_exponent = prop.a_scale_exponent + full_name = f"SO/{SO_name}/{outputname}" + assert full_name in halo_result + result = halo_result[full_name][0] + assert (len(result.shape) == 0 and size == 1) or result.shape[0] == size + assert result.dtype == dtype + unit = unyt.Unit(unit_string, registry=dummy_halos.unit_registry) + if not physical: + unit = ( + unit + * unyt.Unit("a", registry=dummy_halos.unit_registry) + ** a_exponent + ) + assert result.units == unit.units + + dummy_halos.get_cell_grid().snapshot_datasets.print_dataset_log() + + +def calculate_SO_properties_nfw_halo(seed, num_part, c): + """ + Generates a halo with an NFW profile, and calculates SO properties for it + """ + from dummy_halo_generator import DummyHaloGenerator + + dummy_halos = DummyHaloGenerator(seed) + gas_filter = dummy_halos.get_recently_heated_gas_filter() + cat_filter = CategoryFilter(dummy_halos.get_filters({"general": 100})) + parameters = ParameterFile( + parameter_dictionary={ + "aliases": { + "PartType0/ElementMassFractions": "PartType0/SmoothedElementMassFractions", + "PartType4/ElementMassFractions": "PartType4/SmoothedElementMassFractions", + "PartType0/XrayLuminositiesRestframe": "PartType0/XrayLuminositiesRestframe", + "PartType0/XrayPhotonLuminositiesRestframe": "PartType0/XrayPhotonLuminositiesRestframe", + } + } + ) + dummy_halos.get_cell_grid().snapshot_datasets.setup_aliases( + parameters.get_aliases() + ) + parameters.get_halo_type_variations( + "SOProperties", + { + "50_kpc": {"value": 50.0, "type": "physical"}, + "2500_mean": {"value": 2500.0, "type": "mean"}, + "2500_crit": {"value": 2500.0, "type": "crit"}, + "BN98": {"value": 0.0, "type": "BN98"}, + "5xR2500_mean": {"value": 2500.0, "type": "mean", "radius_multiple": 5.0}, + }, + ) + + property_calculator_200crit = SOProperties( + dummy_halos.get_cell_grid(), + parameters, + gas_filter, + cat_filter, + "basic", + 200.0, + "crit", + ) + + (input_halo, data, rmax, Mtot, Npart, particle_numbers) = dummy_halos.gen_nfw_halo( + 100, c, num_part + ) + + halo_result_template = dummy_halos.get_halo_result_template(particle_numbers) + + property_calculator_200crit.cosmology["nu_density"] *= 0 + property_calculator_200crit.calculate(input_halo, rmax, data, halo_result_template) + + return halo_result_template + + +def test_concentration_nfw_halo(): + """ + Test if the calculated concentration is close to the input value. + Only tests halos with for 10000 particles. + Fails due to noise for small particle numbers. + """ + n_part = 10000 + for seed in range(10): + for concentration in [5, 10]: + halo_result = calculate_SO_properties_nfw_halo(seed, n_part, concentration) + calculated = halo_result["SO/200_crit/Concentration"][0] + delta = np.abs(calculated - concentration) / concentration + assert delta < 0.1 + + +if __name__ == "__main__": + """ + Standalone mode for running tests. + """ + print("Calling test_SO_properties_random_halo()...") + test_SO_properties_random_halo() + print("Calling test_concentration_nfw_halo()...") + test_concentration_nfw_halo() + print("Tests passed.") diff --git a/test_SO_radius_calculation.py b/tests/test_SO_radius_calculation.py similarity index 95% rename from test_SO_radius_calculation.py rename to tests/test_SO_radius_calculation.py index 998fddd0..d21c3165 100644 --- a/test_SO_radius_calculation.py +++ b/tests/test_SO_radius_calculation.py @@ -4,9 +4,6 @@ test_SO_radius_calculation.py Unit test for the SO radius calculation. - -We put this in a separate file to avoid cluttering -SO_properties.py even more. """ import numpy as np @@ -16,8 +13,8 @@ matplotlib.use("Agg") import matplotlib.pyplot as pl -from SO_properties import find_SO_radius_and_mass -from halo_properties import SearchRadiusTooSmallError +from SOAP.particle_selection.SO_properties import find_SO_radius_and_mass +from SOAP.particle_selection.halo_properties import SearchRadiusTooSmallError def test_SO_radius_calculation(): @@ -51,9 +48,9 @@ def test_SO_radius_calculation(): ipos = np.argmax(ordered_radius > 0.0) ordered_radius = ordered_radius[ipos:] cumulative_mass = cumulative_mass[ipos:] - density = cumulative_mass / (4.0 * np.pi / 3.0 * ordered_radius ** 3) + density = cumulative_mass / (4.0 * np.pi / 3.0 * ordered_radius**3) - reference_density = 200.0 * Mpart * npart / (4.0 * np.pi / 3.0 * rmax ** 3) + reference_density = 200.0 * Mpart * npart / (4.0 * np.pi / 3.0 * rmax**3) try: SO_r, SO_mass, SO_volume = find_SO_radius_and_mass( @@ -78,7 +75,7 @@ def test_SO_radius_calculation(): ax[1][0].semilogy(ordered_radius, cumulative_mass, "o-") if SO_r >= 0.0 * unyt.kpc: rrange = np.linspace(0.0 * unyt.kpc, 2.0 * SO_r, 100) - Mrange = reference_density * 4.0 * np.pi / 3.0 * rrange ** 3 + Mrange = reference_density * 4.0 * np.pi / 3.0 * rrange**3 rrange.convert_to_units("kpc") Mrange.convert_to_units("Msun") ax[1][0].semilogy(rrange, Mrange, ":", color="C2") @@ -88,7 +85,7 @@ def test_SO_radius_calculation(): ax[0][1].semilogy(ordered_radius[beg:end], density[beg:end], "o-") ax[1][1].semilogy(ordered_radius[beg:end], cumulative_mass[beg:end], "o-") rrange = np.linspace(0.9 * SO_r, 1.1 * SO_r, 100) - Mrange = reference_density * 4.0 * np.pi / 3.0 * rrange ** 3 + Mrange = reference_density * 4.0 * np.pi / 3.0 * rrange**3 rrange.convert_to_units("kpc") Mrange.convert_to_units("Msun") ax[1][1].semilogy(rrange, Mrange, ":", color="C2") diff --git a/tests/test_aperture_properties.py b/tests/test_aperture_properties.py new file mode 100644 index 00000000..fcfae31d --- /dev/null +++ b/tests/test_aperture_properties.py @@ -0,0 +1,249 @@ +import pytest +import numpy as np +import unyt + +from SOAP.core.category_filter import CategoryFilter +from SOAP.core.parameter_file import ParameterFile +from SOAP.property_calculation.stellar_age_calculator import StellarAgeCalculator +from SOAP.particle_filter.cold_dense_gas_filter import ColdDenseGasFilter +from SOAP.particle_filter.recently_heated_gas_filter import RecentlyHeatedGasFilter +from SOAP.particle_selection.halo_properties import SearchRadiusTooSmallError +from SOAP.particle_selection.aperture_properties import ( + ExclusiveSphereProperties, + InclusiveSphereProperties, +) + +from dummy_halo_generator import DummyHaloGenerator + + +def test_aperture_properties(): + """ + Unit test for the aperture property calculations. + + We generate 100 random "dummy" halos and feed them to + ExclusiveSphereProperties::calculate() and + InclusiveSphereProperties::calculate(). We check that the returned values + are present, and have the right units, size and dtype + """ + + # initialise the DummyHaloGenerator with a random seed + dummy_halos = DummyHaloGenerator(3256) + recently_heated_filter = dummy_halos.get_recently_heated_gas_filter() + stellar_age_calculator = StellarAgeCalculator(dummy_halos.get_cell_grid()) + cold_dense_gas_filter = dummy_halos.get_cold_dense_gas_filter() + cat_filter = CategoryFilter( + dummy_halos.get_filters( + {"general": 0, "gas": 0, "dm": 0, "star": 0, "baryon": 0} + ) + ) + parameters = ParameterFile( + parameter_dictionary={ + "aliases": { + "PartType0/ElementMassFractions": "PartType0/SmoothedElementMassFractions", + "PartType4/ElementMassFractions": "PartType4/SmoothedElementMassFractions", + } + } + ) + dummy_halos.get_cell_grid().snapshot_datasets.setup_aliases( + parameters.get_aliases() + ) + parameters.get_halo_type_variations( + "ApertureProperties", + { + "exclusive_50_kpc": {"radius_in_kpc": 50.0, "inclusive": False}, + "inclusive_50_kpc": {"radius_in_kpc": 50.0, "inclusive": True}, + }, + ) + + pc_exclusive = ExclusiveSphereProperties( + dummy_halos.get_cell_grid(), + parameters, + 50.0, + None, + recently_heated_filter, + stellar_age_calculator, + cold_dense_gas_filter, + cat_filter, + "basic", + [50.0], + ) + pc_inclusive = InclusiveSphereProperties( + dummy_halos.get_cell_grid(), + parameters, + 50.0, + None, + recently_heated_filter, + stellar_age_calculator, + cold_dense_gas_filter, + cat_filter, + "basic", + [50.0], + ) + + # Create a filter that no halos will satisfy + fail_filter = CategoryFilter(dummy_halos.get_filters({"general": 10000000})) + pc_filter_test = ExclusiveSphereProperties( + dummy_halos.get_cell_grid(), + parameters, + 50.0, + None, + recently_heated_filter, + stellar_age_calculator, + cold_dense_gas_filter, + fail_filter, + "general", + [50.0], + ) + + # generate 100 random halos + for i in range(100): + input_halo, data, _, _, _, particle_numbers = dummy_halos.get_random_halo( + [1, 10, 100, 1000, 10000] + ) + halo_result_template = dummy_halos.get_halo_result_template(particle_numbers) + + for pc_name, pc_type, pc_calc in [ + ("ExclusiveSphere", "ExclusiveSphere", pc_exclusive), + ("InclusiveSphere", "InclusiveSphere", pc_inclusive), + ("filter_test", "ExclusiveSphere", pc_filter_test), + ]: + input_data = {} + for ptype in pc_calc.particle_properties: + if ptype in data: + input_data[ptype] = {} + for dset in pc_calc.particle_properties[ptype]: + input_data[ptype][dset] = data[ptype][dset] + input_halo_copy = input_halo.copy() + input_data_copy = input_data.copy() + + # Check halo fails if search radius is too small + halo_result = dict(halo_result_template) + if pc_name != "filter_test": + with pytest.raises(SearchRadiusTooSmallError): + pc_calc.calculate( + input_halo, 10 * unyt.kpc, input_data, halo_result + ) + # Skipped halos shouldn't ever require a larger search radius + else: + pc_calc.calculate(input_halo, 10 * unyt.kpc, input_data, halo_result) + + halo_result = dict(halo_result_template) + pc_calc.calculate(input_halo, 100 * unyt.kpc, input_data, halo_result) + assert input_halo == input_halo_copy + assert input_data == input_data_copy + + # check that the calculation returns the correct values + for prop in pc_calc.property_list.values(): + outputname = prop.name + size = prop.shape + dtype = prop.dtype + unit_string = prop.unit + physical = prop.output_physical + a_exponent = prop.a_scale_exponent + full_name = f"{pc_type}/50kpc/{outputname}" + assert full_name in halo_result + result = halo_result[full_name][0] + assert (len(result.shape) == 0 and size == 1) or result.shape[0] == size + assert result.dtype == dtype + unit = unyt.Unit(unit_string, registry=dummy_halos.unit_registry) + if not physical: + unit = ( + unit + * unyt.Unit("a", registry=dummy_halos.unit_registry) + ** a_exponent + ) + assert result.units == unit.units + + # Check properties were not calculated for filtered halos + if pc_name == "filter_test": + for prop in pc_calc.property_list.values(): + outputname = prop.name + size = prop.shape + full_name = f"{pc_type}/50kpc/{outputname}" + assert np.all(halo_result[full_name][0].value == np.zeros(size)) + + # Now test the calculation for each property individually, to make sure that + # all properties read all the datasets they require + # we reuse the last random halo for this + all_parameters = parameters.get_parameters() + for property in all_parameters["ApertureProperties"]["properties"]: + print(f"Testing only {property}...") + single_property = dict(all_parameters) + for other_property in all_parameters["ApertureProperties"]["properties"]: + single_property["ApertureProperties"]["properties"][other_property] = ( + other_property == property + ) or other_property.startswith("NumberOf") + single_parameters = ParameterFile(parameter_dictionary=single_property) + pc_exclusive = ExclusiveSphereProperties( + dummy_halos.get_cell_grid(), + single_parameters, + 50.0, + None, + recently_heated_filter, + stellar_age_calculator, + cold_dense_gas_filter, + cat_filter, + "basic", + [50.0], + ) + pc_inclusive = InclusiveSphereProperties( + dummy_halos.get_cell_grid(), + single_parameters, + 50.0, + None, + recently_heated_filter, + stellar_age_calculator, + cold_dense_gas_filter, + cat_filter, + "basic", + [50.0], + ) + + halo_result_template = dummy_halos.get_halo_result_template(particle_numbers) + + for pc_type, pc_calc in [ + ("ExclusiveSphere", pc_exclusive), + ("InclusiveSphere", pc_inclusive), + ]: + input_data = {} + for ptype in pc_calc.particle_properties: + if ptype in data: + input_data[ptype] = {} + for dset in pc_calc.particle_properties[ptype]: + input_data[ptype][dset] = data[ptype][dset] + input_halo_copy = input_halo.copy() + input_data_copy = input_data.copy() + halo_result = dict(halo_result_template) + pc_calc.calculate(input_halo, 100 * unyt.kpc, input_data, halo_result) + assert input_halo == input_halo_copy + assert input_data == input_data_copy + + # check that the calculation returns the correct values + for prop in pc_calc.property_list.values(): + outputname = prop.name + if not outputname == property: + continue + size = prop.shape + dtype = prop.dtype + unit_string = prop.unit + full_name = f"{pc_type}/50kpc/{outputname}" + assert full_name in halo_result + result = halo_result[full_name][0] + assert (len(result.shape) == 0 and size == 1) or result.shape[0] == size + assert result.dtype == dtype + unit = unyt.Unit(unit_string, registry=dummy_halos.unit_registry) + assert result.units.same_dimensions_as(unit.units) + + dummy_halos.get_cell_grid().snapshot_datasets.print_dataset_log() + + +if __name__ == "__main__": + """ + Standalone version of the program: just run the unit test. + + Note that this can also be achieved by running "pytest *.py" in the folder. + """ + + print("Running test_aperture_properties()...") + test_aperture_properties() + print("Test passed.") diff --git a/tests/test_half_mass_radius.py b/tests/test_half_mass_radius.py new file mode 100644 index 00000000..8b605162 --- /dev/null +++ b/tests/test_half_mass_radius.py @@ -0,0 +1,38 @@ +import numpy as np +import unyt + +from SOAP.property_calculation.half_mass_radius import get_half_mass_radius + + +def test_get_half_mass_radius(): + """ + Unit test for get_half_mass_radius(). + + We generate 1000 random particle distributions and check that the + half mass radius returned by the function contains less than half + the particles in mass. + """ + np.random.seed(203) + + for i in range(1000): + npart = np.random.choice([1, 10, 100, 1000, 10000]) + + radius = np.random.exponential(1.0, npart) * unyt.kpc + + Mpart = 1.0e9 * unyt.Msun + mass = Mpart * (1.0 + 0.2 * (np.random.random(npart) - 0.5)) + + total_mass = mass.sum() + + half_mass_radius = get_half_mass_radius(radius, mass, total_mass) + + mask = radius <= half_mass_radius + Mtest = mass[mask].sum() + assert Mtest <= 0.5 * total_mass + + fail = False + try: + half_mass_radius = get_half_mass_radius(radius, mass, 2.0 * total_mass) + except RuntimeError: + fail = True + assert fail diff --git a/tests/test_projected_aperture_properties.py b/tests/test_projected_aperture_properties.py new file mode 100644 index 00000000..15e7cb34 --- /dev/null +++ b/tests/test_projected_aperture_properties.py @@ -0,0 +1,194 @@ +import pytest +import numpy as np +import unyt + +from SOAP.core.category_filter import CategoryFilter +from SOAP.core.parameter_file import ParameterFile +from SOAP.particle_selection.projected_aperture_properties import ( + ProjectedApertureProperties, +) + +from dummy_halo_generator import DummyHaloGenerator + + +def test_projected_aperture_properties(): + """ + Unit test for the projected aperture calculation. + + Generates 100 random halos and passes them on to + ProjectedApertureProperties::calculate(). + Tests that all expected return values are computed and have the right size, + dtype and units. + """ + + import pytest + from dummy_halo_generator import DummyHaloGenerator + + dummy_halos = DummyHaloGenerator(127) + category_filter = CategoryFilter(dummy_halos.get_filters({"general": 100})) + parameters = ParameterFile( + parameter_dictionary={ + "aliases": { + "PartType0/ElementMassFractions": "PartType0/SmoothedElementMassFractions", + "PartType4/ElementMassFractions": "PartType4/SmoothedElementMassFractions", + } + } + ) + dummy_halos.get_cell_grid().snapshot_datasets.setup_aliases( + parameters.get_aliases() + ) + parameters.get_halo_type_variations( + "ProjectedApertureProperties", {"30_kpc": {"radius_in_kpc": 30.0}} + ) + + pc_projected = ProjectedApertureProperties( + dummy_halos.get_cell_grid(), + parameters, + 30.0, + None, + category_filter, + "basic", + [30.0], + ) + + # Create a filter that no halos will satisfy + fail_filter = CategoryFilter(dummy_halos.get_filters({"general": 10000000})) + pc_filter_test = ProjectedApertureProperties( + dummy_halos.get_cell_grid(), + parameters, + 30.0, + None, + fail_filter, + "general", + [30.0], + ) + + for i in range(100): + input_halo, data, _, _, _, particle_numbers = dummy_halos.get_random_halo( + [1, 10, 100, 1000, 10000] + ) + halo_result_template = dummy_halos.get_halo_result_template(particle_numbers) + + for pc_name, pc_calc in [ + ("ProjectedAperture", pc_projected), + ("filter_test", pc_filter_test), + ]: + input_data = {} + for ptype in pc_calc.particle_properties: + if ptype in data: + input_data[ptype] = {} + for dset in pc_calc.particle_properties[ptype]: + input_data[ptype][dset] = data[ptype][dset] + input_halo_copy = input_halo.copy() + input_data_copy = input_data.copy() + + halo_result = dict(halo_result_template) + pc_calc.calculate(input_halo, 50 * unyt.kpc, input_data, halo_result) + assert input_halo == input_halo_copy + assert input_data == input_data_copy + + for proj in ["projx", "projy", "projz"]: + for prop in pc_calc.property_list.values(): + outputname = prop.name + size = prop.shape + dtype = prop.dtype + unit_string = prop.unit + full_name = f"ProjectedAperture/30kpc/{proj}/{outputname}" + assert full_name in halo_result + result = halo_result[full_name][0] + assert (len(result.shape) == 0 and size == 1) or result.shape[ + 0 + ] == size + assert result.dtype == dtype + unit = unyt.Unit(unit_string, registry=dummy_halos.unit_registry) + assert result.units.same_dimensions_as(unit.units) + + # Check properties were not calculated for filtered halos + if pc_name == "filter_test": + for proj in ["projx", "projy", "projz"]: + for prop in pc_calc.property_list.values(): + outputname = prop.name + size = prop.shape + full_name = f"ProjectedAperture/30kpc/{proj}/{outputname}" + assert np.all(halo_result[full_name][0].value == np.zeros(size)) + + # Now test the calculation for each property individually, to make sure that + # all properties read all the datasets they require + all_parameters = parameters.get_parameters() + for property in all_parameters["ProjectedApertureProperties"]["properties"]: + print(f"Testing only {property}...") + single_property = dict(all_parameters) + for other_property in all_parameters["ProjectedApertureProperties"][ + "properties" + ]: + single_property["ProjectedApertureProperties"]["properties"][ + other_property + ] = (other_property == property) or other_property.startswith("NumberOf") + single_parameters = ParameterFile(parameter_dictionary=single_property) + + property_calculator = ProjectedApertureProperties( + dummy_halos.get_cell_grid(), + single_parameters, + 30.0, + None, + category_filter, + "basic", + [30.0], + ) + + halo_result_template = dummy_halos.get_halo_result_template(particle_numbers) + + input_data = {} + for ptype in property_calculator.particle_properties: + if ptype in data: + input_data[ptype] = {} + for dset in property_calculator.particle_properties[ptype]: + input_data[ptype][dset] = data[ptype][dset] + input_halo_copy = input_halo.copy() + input_data_copy = input_data.copy() + halo_result = dict(halo_result_template) + property_calculator.calculate( + input_halo, 50 * unyt.kpc, input_data, halo_result + ) + assert input_halo == input_halo_copy + assert input_data == input_data_copy + + for proj in ["projx", "projy", "projz"]: + for prop in property_calculator.property_list: + outputname = prop[1] + if not outputname == property: + continue + size = prop.size + dtype = prop.dtype + unit_string = prop.unit + physical = prop.output_physical + a_exponent = prop.a_scale_exponent + full_name = f"ProjectedAperture/30kpc/{proj}/{outputname}" + assert full_name in halo_result + result = halo_result[full_name][0] + assert (len(result.shape) == 0 and size == 1) or result.shape[0] == size + assert result.dtype == dtype + unit = unyt.Unit(unit_string, registry=dummy_halos.unit_registry) + if not physical: + unit = ( + unit + * unyt.Unit("a", registry=dummy_halos.unit_registry) + ** a_exponent + ) + assert result.units == unit.units + + dummy_halos.get_cell_grid().snapshot_datasets.print_dataset_log() + + +if __name__ == "__main__": + """ + Standalone mode: simply run the unit test. + + Note that this can also be achieved by running + python3 -m pytest *.py + in the main folder. + """ + + print("Calling test_projected_aperture_properties()...") + test_projected_aperture_properties() + print("Test passed.") diff --git a/tests/test_read_rockstar.py b/tests/test_read_rockstar.py new file mode 100644 index 00000000..3fbed66d --- /dev/null +++ b/tests/test_read_rockstar.py @@ -0,0 +1,60 @@ +#!/bin/env python +import os +import warnings + +from mpi4py import MPI +import numpy as np +import pytest +import unyt +import virgo.mpi.parallel_sort as psort + +from SOAP.catalogue_readers.read_rockstar import ( + read_rockstar_groupnr, + locate_files, + read_group_file, +) + +comm = MPI.COMM_WORLD +comm_rank = comm.Get_rank() + + +@pytest.mark.mpi +def test_read_rockstar_groupnr(): + """ + Read in rockstar group numbers and compute the number of particles + in each group. This is then compared with the input catalogue as a + sanity check on the group membershp files. + """ + + # Test with FLAMINGO data on cosma8 + test_data_dir = "/cosma8/data/dp004/dc-mcgi1/SOAP/TEST_DATA/ROCKSTAR" + run = "L1000N0900/DMO_FIDUCIAL" + snap_nr = 77 + basename = f"{test_data_dir}/{run}/snapshot_{snap_nr:04d}/halos_{snap_nr:04d}" + # Skip this test if we can't find the data (too large to download) + skip = False + if comm_rank == 0: + if not os.path.exists(os.path.dirname(basename)): + skip = True + skip = comm.bcast(skip) + if skip: + return + + # Catch deprecation warning from VirgoDC + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + _, ids, grnr = read_rockstar_groupnr(basename) + del ids # Don't need the particle IDs + + # Find maximum group number + max_grnr = comm.allreduce(np.amax(grnr), op=MPI.MAX) + nr_groups_from_grnr = max_grnr + 1 + if comm_rank == 0: + print(f"Number of groups from membership files = {nr_groups_from_grnr}") + + # Discard particles in no group + keep = grnr >= 0 + grnr = grnr[keep] + + # Compute group sizes + nbound_from_grnr = psort.parallel_bincount(grnr, comm=comm) diff --git a/tests/test_read_subfind.py b/tests/test_read_subfind.py new file mode 100644 index 00000000..91c07de4 --- /dev/null +++ b/tests/test_read_subfind.py @@ -0,0 +1,90 @@ +#!/bin/env python +import os +import warnings + +from mpi4py import MPI +import numpy as np +import pytest +import virgo.mpi.parallel_sort as psort +import virgo.mpi.parallel_hdf5 as phdf5 + +from SOAP.catalogue_readers.read_subfind import ( + read_gadget4_groupnr, + locate_files, +) + +comm = MPI.COMM_WORLD +comm_rank = comm.Get_rank() + + +@pytest.mark.mpi +def test_read_gadget4_groupnr(): + """ + Read in Gadget-4 group numbers and compute the number of particles + in each group. This is then compared with the input catalogue as a + sanity check on the group membershp files. + """ + + # Test with FLAMINGO data on cosma8 + test_data_dir = "/cosma8/data/dp004/dc-mcgi1/SOAP/TEST_DATA/SubFind" + run = "L1000N1800/DMO_FIDUCIAL" + snap_nr = 77 + basename = f"{test_data_dir}/{run}/snapdir_{snap_nr:03d}/snapshot_{snap_nr:03d}" + # Skip this test if we can't find the data (too large to download) + skip = False + if comm_rank == 0: + print(os.path.dirname(basename)) + if not os.path.exists(os.path.dirname(basename)): + skip = True + skip = comm.bcast(skip) + if skip: + return + + n_halo, ids, grnr = read_gadget4_groupnr(basename) + del ids # Don't need the particle IDs + + # Find maximum group number + max_grnr = comm.allreduce(np.amax(grnr), op=MPI.MAX) + nr_groups_from_grnr = max_grnr + 1 + if comm_rank == 0: + print(f"Number of groups from membership files = {nr_groups_from_grnr}") + + # Discard particles in no group + keep = grnr >= 0 + grnr = grnr[keep] + + # Compute group sizes + nbound_from_grnr = psort.parallel_bincount(grnr, comm=comm) + + # Locate the snapshot and fof_subhalo_tab files + if comm_rank == 0: + snap_format_string, group_format_string = locate_files(basename) + else: + snap_format_string = None + group_format_string = None + snap_format_string, group_format_string = comm.bcast( + (snap_format_string, group_format_string) + ) + + # Read group sizes from the group catalogue + subtab = phdf5.MultiFile(group_format_string, file_nr_attr=("Header", "NumFiles")) + nbound_from_subtab = subtab.read("Subhalo/SubhaloLen") + + # Find number of groups in the subfind output + nr_groups_from_subtab = comm.allreduce(len(nbound_from_subtab)) + if comm_rank == 0: + print(f"Number of groups from fof_subhalo_tab = {nr_groups_from_subtab}") + if nr_groups_from_subtab != nr_groups_from_grnr: + print("Number of groups does not agree!") + comm.Abort(1) + + # Ensure nbound arrays are partitioned the same way + nr_per_rank = comm.allgather(len(nbound_from_subtab)) + nbound_from_grnr = psort.repartition( + nbound_from_grnr, ndesired=nr_per_rank, comm=comm + ) + + # Compare + nr_different = comm.allreduce(np.sum(nbound_from_grnr != nbound_from_subtab)) + if comm_rank == 0: + print(f"Number of group sizes which differ = {nr_different} (should be 0!)") diff --git a/tests/test_read_vr.py b/tests/test_read_vr.py new file mode 100644 index 00000000..124a5b71 --- /dev/null +++ b/tests/test_read_vr.py @@ -0,0 +1,41 @@ +import sys +import warnings + +from mpi4py import MPI +import pytest + +from SOAP.catalogue_readers.read_vr import read_vr_group_sizes + +import helpers + +comm = MPI.COMM_WORLD +comm_rank = comm.Get_rank() + +required_files = [ + "VR_output/vr_018.catalog_groups.0", + "VR_output/vr_018.catalog_particles.0", + "VR_output/vr_018.catalog_particles.unbound.0", + "VR_output/vr_018.properties.0", +] + + +@pytest.mark.mpi +@helpers.requires(required_files, comm=comm) +def test_read_vr(filenames): + + basename = filenames[0].split(".")[0] + suffix = "." + filenames[0].split(".")[-1] + + # Catch deprecation warning from VirgoDC + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + nr_parts_bound, nr_parts_unbound = read_vr_group_sizes(basename, suffix, comm) + + comm.barrier() + nr_halos_total = comm.allreduce(len(nr_parts_bound)) + if comm_rank == 0: + print(f"Read {nr_halos_total} halos") + + +if __name__ == "__main__": + test_read_vr() diff --git a/tests/test_shared_mesh.py b/tests/test_shared_mesh.py new file mode 100644 index 00000000..6a425385 --- /dev/null +++ b/tests/test_shared_mesh.py @@ -0,0 +1,223 @@ +import numpy as np +from mpi4py import MPI +import pytest +import unyt + +from SOAP.core import shared_array, shared_mesh + + +def make_test_dataset(boxsize, total_nr_points, centre, radius, box_wrap, comm): + """ + Make a set of random test points + + boxsize - periodic box size (unyt scalar) + total_nr_points - number of points in the box over all MPI ranks + centre - centre of the particle distribution + radius - half side length of the particle distribution + box_wrap - True if points should be wrapped into the box + comm - MPI communicator to use + + Returns a (total_nr_points,3) SharedArray instance. + """ + comm_size = comm.Get_size() + comm_rank = comm.Get_rank() + + # Determine number of points per rank + nr_points = total_nr_points // comm_size + if comm_rank < (total_nr_points % comm_size): + nr_points += 1 + assert comm.allreduce(nr_points) == total_nr_points + + # Make some test data + pos = shared_array.SharedArray( + local_shape=(nr_points, 3), dtype=np.float64, units=radius.units, comm=comm + ) + if comm_rank == 0: + # Rank 0 initializes all elements to avoid parallel RNG issues + pos.full[:, :] = 2 * radius * np.random.random_sample(pos.full.shape) - radius + pos.full[:, :] += centre[None, :].to(radius.units) + if box_wrap: + pos.full[:, :] = pos.full[:, :] % boxsize + assert np.all((pos.full >= 0.0) & (pos.full < boxsize)) + pos.sync() + comm.barrier() + return pos + + +def _test_periodic_box( + total_nr_points, + centre, + radius, + boxsize, + box_wrap, + nr_queries, + resolution, + max_search_radius, +): + """ + Test case where points fill the periodic box. + + Creates a shared mesh from random points, queries for points near random + centres and checks the results against a simple brute force method. + """ + + comm = MPI.COMM_WORLD + comm_size = comm.Get_size() + comm_rank = comm.Get_rank() + + if comm_rank == 0: + print( + f"Test with {total_nr_points} points, resolution {resolution} and {nr_queries} queries" + ) + print( + f" Boxsize {boxsize}, centre {centre}, radius {radius}, box_wrap {box_wrap}" + ) + + def periodic_distance_squared(pos, centre): + dr = pos - centre[None, :] + dr[dr > 0.5 * boxsize] -= boxsize + dr[dr < -0.5 * boxsize] += boxsize + return np.sum(dr**2, axis=1) + + # Generate random test points + pos = make_test_dataset(boxsize, total_nr_points, centre, radius, box_wrap, comm) + + # Construct the shared mesh + mesh = shared_mesh.SharedMesh(comm, pos, resolution=resolution) + + # Each MPI rank queries random points and verifies the result + nr_failures = 0 + for query_nr in range(nr_queries): + + # Pick a centre and radius + search_centre = (np.random.random_sample((3,)) * 2 * radius) - radius + centre + search_radius = np.random.random_sample(()) * max_search_radius + + # Query the mesh for point indexes + idx = mesh.query_radius_periodic(search_centre, search_radius, pos, boxsize) + + # Check that the indexes are unique + if len(idx) != len(np.unique(idx)): + print( + f" Duplicate IDs for centre={search_centre}, radius={search_radius}" + ) + nr_failures += 1 + else: + # Flag the points in the returned index array + in_idx = np.zeros(pos.full.shape[0], dtype=bool) + in_idx[idx] = True + # Find radii of all points + r2 = periodic_distance_squared(pos.full, search_centre) + # Check for any flagged points outside the radius + if np.any(r2[in_idx] > search_radius * search_radius): + print( + f" Returned point outside radius for centre={search_centre}, radius={search_radius}" + ) + nr_failures += 1 + # Check for any non-flagged points inside the radius + missed = (in_idx == False) & (r2 < search_radius * search_radius) + if np.any(missed): + print(r2[missed]) + print( + f" Missed point inside radius for centre={search_centre}, radius={search_radius}, rank={comm_rank}" + ) + nr_failures += 1 + + # Tidy up before possibly throwing an exception + pos.free() + mesh.free() + + nr_failures = comm.allreduce(nr_failures) + + comm.barrier() + if comm_rank == 0: + if nr_failures == 0: + print(f" OK") + else: + print(f" {nr_failures} of {nr_queries*comm_size} queries FAILED") + comm.Abort(1) + + +@pytest.mark.mpi +def test_shared_mesh(): + + # Use a different, reproducible seed on each rank + comm = MPI.COMM_WORLD + np.random.seed(comm.Get_rank()) + + resolutions = (1, 2, 4, 8, 16, 32) + + # Test a particle distribution which fills the box, searching up to 0.25 box size + for resolution in resolutions: + centre = 0.5 * np.ones(3, dtype=np.float64) * unyt.m + radius = 0.5 * unyt.m + centre, radius = comm.bcast((centre, radius)) + boxsize = 1.0 * unyt.m + _test_periodic_box( + 1000, + centre, + radius, + boxsize, + box_wrap=False, + nr_queries=100, + resolution=resolution, + max_search_radius=0.25 * boxsize, + ) + + # Test populating some random sub-regions, which may extend outside the box or be wrapped back in + nr_regions = 10 + boxsize = 1.0 * unyt.m + for box_wrap in (True, False): + for resolution in resolutions: + for region_nr in range(nr_regions): + centre = np.random.random_sample((3,)) * boxsize + radius = 0.25 * np.random.random_sample(()) * boxsize + centre, radius = comm.bcast((centre, radius)) + _test_periodic_box( + 1000, + centre, + radius, + boxsize, + box_wrap=box_wrap, + nr_queries=10, + resolution=resolution, + max_search_radius=radius, + ) + + # Zero particles in the box + for resolution in resolutions: + centre = 0.5 * np.ones(3, dtype=np.float64) * unyt.m + radius = 0.5 * unyt.m + centre, radius = comm.bcast((centre, radius)) + boxsize = 1.0 * unyt.m + _test_periodic_box( + 0, + centre, + radius, + boxsize, + box_wrap=False, + nr_queries=100, + resolution=resolution, + max_search_radius=0.25 * boxsize, + ) + + # One particle in the box + for resolution in resolutions: + centre = 0.5 * np.ones(3, dtype=np.float64) * unyt.m + radius = 0.5 * unyt.m + centre, radius = comm.bcast((centre, radius)) + boxsize = 1.0 * unyt.m + _test_periodic_box( + 1, + centre, + radius, + boxsize, + box_wrap=False, + nr_queries=100, + resolution=resolution, + max_search_radius=0.25 * boxsize, + ) + + +if __name__ == "__main__": + test_shared_mesh() diff --git a/tests/test_subhalo_properties.py b/tests/test_subhalo_properties.py new file mode 100644 index 00000000..ff0d0d15 --- /dev/null +++ b/tests/test_subhalo_properties.py @@ -0,0 +1,158 @@ +import pytest +import numpy as np +import unyt + +from SOAP.core.category_filter import CategoryFilter +from SOAP.core.parameter_file import ParameterFile +from SOAP.property_calculation.stellar_age_calculator import StellarAgeCalculator +from SOAP.particle_selection.subhalo_properties import SubhaloProperties + +from dummy_halo_generator import DummyHaloGenerator + + +def test_subhalo_properties(): + """ + Unit test for the subhalo property calculations. + + We generate 100 random "dummy" halos and feed them to + SubhaloProperties::calculate(). We check that the returned values + are present, and have the right units, size and dtype + """ + + # initialise the DummyHaloGenerator with a random seed + dummy_halos = DummyHaloGenerator(16902) + cat_filter = CategoryFilter( + dummy_halos.get_filters( + {"general": 100, "gas": 100, "dm": 100, "star": 100, "baryon": 100} + ) + ) + parameters = ParameterFile( + parameter_dictionary={ + "aliases": { + "PartType0/ElementMassFractions": "PartType0/SmoothedElementMassFractions", + "PartType4/ElementMassFractions": "PartType4/SmoothedElementMassFractions", + } + } + ) + dummy_halos.get_cell_grid().snapshot_datasets.setup_aliases( + parameters.get_aliases() + ) + parameters.get_halo_type_variations( + "SubhaloProperties", + {}, + ) + + recently_heated_gas_filter = dummy_halos.get_recently_heated_gas_filter() + stellar_age_calculator = StellarAgeCalculator(dummy_halos.get_cell_grid()) + + property_calculator_bound = SubhaloProperties( + dummy_halos.get_cell_grid(), + parameters, + recently_heated_gas_filter, + stellar_age_calculator, + cat_filter, + ) + # generate 100 random halos + for i in range(100): + input_halo, data, _, _, _, _ = dummy_halos.get_random_halo( + [1, 10, 100, 1000, 10000] + ) + + halo_result = {} + for subhalo_name, prop_calc in [ + ("BoundSubhalo", property_calculator_bound), + ]: + input_data = {} + for ptype in prop_calc.particle_properties: + if ptype in data: + input_data[ptype] = {} + for dset in prop_calc.particle_properties[ptype]: + input_data[ptype][dset] = data[ptype][dset] + input_halo_copy = input_halo.copy() + input_data_copy = input_data.copy() + prop_calc.calculate(input_halo, 0.0 * unyt.kpc, input_data, halo_result) + assert input_halo == input_halo_copy + assert input_data == input_data_copy + + # check that the calculation returns the correct values + for prop in prop_calc.property_list.values(): + outputname = prop.name + size = prop.shape + dtype = prop.dtype + unit_string = prop.unit + physical = prop.output_physical + a_exponent = prop.a_scale_exponent + full_name = f"{subhalo_name}/{outputname}" + assert full_name in halo_result + result = halo_result[full_name][0] + assert (len(result.shape) == 0 and size == 1) or result.shape[0] == size + assert result.dtype == dtype + unit = unyt.Unit(unit_string, registry=dummy_halos.unit_registry) + if not physical: + unit = ( + unit + * unyt.Unit("a", registry=dummy_halos.unit_registry) + ** a_exponent + ) + assert result.units == unit.units + + # Now test the calculation for each property individually, to make sure that + # all properties read all the datasets they require + all_parameters = parameters.get_parameters() + for property in all_parameters["SubhaloProperties"]["properties"]: + print(f"Testing only {property}...") + single_property = dict(all_parameters) + for other_property in all_parameters["SubhaloProperties"]["properties"]: + single_property["SubhaloProperties"]["properties"][other_property] = ( + other_property == property + ) or other_property.startswith("NumberOf") + single_parameters = ParameterFile(parameter_dictionary=single_property) + property_calculator_bound = SubhaloProperties( + dummy_halos.get_cell_grid(), + single_parameters, + recently_heated_gas_filter, + stellar_age_calculator, + cat_filter, + ) + halo_result = {} + for subhalo_name, prop_calc in [("BoundSubhalo", property_calculator_bound)]: + input_data = {} + for ptype in prop_calc.particle_properties: + if ptype in data: + input_data[ptype] = {} + for dset in prop_calc.particle_properties[ptype]: + input_data[ptype][dset] = data[ptype][dset] + input_halo_copy = input_halo.copy() + input_data_copy = input_data.copy() + prop_calc.calculate(input_halo, 0.0 * unyt.kpc, input_data, halo_result) + assert input_halo == input_halo_copy + assert input_data == input_data_copy + + # check that the calculation returns the correct values + for prop in prop_calc.property_list.values(): + outputname = prop.name + if not outputname == property: + continue + size = prop.shape + dtype = prop.dtype + unit_string = prop.unit + full_name = f"{subhalo_name}/{outputname}" + assert full_name in halo_result + result = halo_result[full_name][0] + assert (len(result.shape) == 0 and size == 1) or result.shape[0] == size + assert result.dtype == dtype + unit = unyt.Unit(unit_string, registry=dummy_halos.unit_registry) + assert result.units.same_dimensions_as(unit.units) + + dummy_halos.get_cell_grid().snapshot_datasets.print_dataset_log() + + +if __name__ == "__main__": + """ + Standalone version of the program: just run the unit test. + + Note that this can also be achieved by running "pytest *.py" in the folder. + """ + print("Running test_subhalo_properties()...") + test_subhalo_properties() + print("Test passed.") diff --git a/tests/test_subhalo_rank.py b/tests/test_subhalo_rank.py new file mode 100644 index 00000000..a9a9ea35 --- /dev/null +++ b/tests/test_subhalo_rank.py @@ -0,0 +1,62 @@ +#!/bin/env python + +import numpy as np +import h5py +import pytest +from mpi4py import MPI +import virgo.mpi.parallel_hdf5 as phdf5 + +from SOAP.property_calculation.subhalo_rank import compute_subhalo_rank + +import helpers + +comm = MPI.COMM_WORLD +comm_rank = comm.Get_rank() + + +@pytest.mark.mpi +@helpers.requires("HBT_output/018/SubSnap_018.0.hdf5", comm=comm) +def test_subhalo_rank(filename): + + # Read HBT halos from a small DMO run + with h5py.File(filename, "r", driver="mpio", comm=comm) as file: + sub = phdf5.collective_read(file["Subhalos"], comm=comm) + if comm_rank == 0: + print("Read subhalos") + host_id = sub["HostHaloId"] + subhalo_id = sub["TrackId"] + subhalo_mass = sub["Mbound"] + depth = sub["Depth"] + + field = depth == 0 + host_id[field] = subhalo_id[field] + + # Compute ranking of subhalos + subhalo_rank = compute_subhalo_rank(host_id, subhalo_mass, comm) + if comm_rank == 0: + print("Computed ranks") + + # Find fraction of 'field' halos with rank=0 + nr_field_halos = comm.allreduce(np.sum(field)) + nr_field_rank_nonzero = comm.allreduce(np.sum((field) & (subhalo_rank > 0))) + fraction = nr_field_rank_nonzero / nr_field_halos + if comm_rank == 0: + print(f"Fraction of field halos (hostHaloID<0) with rank>0 is {fraction:.3e}") + + # Sanity check: there should be one instance of each hostHaloID with rank=0 + all_ranks = comm.gather(subhalo_rank) + all_host_ids = comm.gather(host_id) + all_ids = comm.gather(subhalo_id) + if comm_rank == 0: + all_ranks = np.concatenate(all_ranks) + all_host_ids = np.concatenate(all_host_ids) + all_ids = np.concatenate(all_ids) + all_host_ids[all_host_ids < 0] = all_ids[all_host_ids < 0] + rank0 = all_ranks == 0 + rank0_hosts = all_host_ids[rank0] + assert len(rank0_hosts) == len(np.unique(all_host_ids)) + + +if __name__ == "__main__": + # Run test with "mpirun -np 8 python3 -m mpi4py ./subhalo_rank.py" + test_subhalo_rank()