From 65bab935287564e6713822eb3388086a75c0eba0 Mon Sep 17 00:00:00 2001 From: shimwell Date: Thu, 21 Feb 2019 17:06:19 +0000 Subject: [PATCH 01/13] modified existing scripts for JENDL data in endf form --- convert_jendl.py | 139 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 139 insertions(+) create mode 100755 convert_jendl.py diff --git a/convert_jendl.py b/convert_jendl.py new file mode 100755 index 00000000..c9f8bc1e --- /dev/null +++ b/convert_jendl.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 + +import argparse +import glob +import os +import sys +import tarfile +import zipfile +from collections import defaultdict +from string import digits +from urllib.request import urlopen + +import openmc.data +from openmc._utils import download + +description = """ +Download JENDL 4.0 data from JAEA and convert it to a HDF5 library for +use with OpenMC. + +""" + + +class CustomFormatter(argparse.ArgumentDefaultsHelpFormatter, + argparse.RawDescriptionHelpFormatter): + pass + + +parser = argparse.ArgumentParser( + description=description, + formatter_class=CustomFormatter +) +parser.add_argument('-b', '--batch', action='store_true', + help='supresses standard in') +parser.add_argument('-d', '--destination', default=None, + help='Directory to create new library in') +parser.add_argument('--libver', choices=['earliest', 'latest'], + default='latest', help="Output HDF5 versioning. Use " + "'earliest' for backwards compatibility or 'latest' for " + "performance") +parser.add_argument('-r', '--release', choices=['4.0'], + default='4.0', help="The nuclear data library release version. " + "The only option currently supported is 4.0") +args = parser.parse_args() + + + +library_name = 'jendl' #this could be added as an argument to allow different libraries to be downloaded +endf_files_dir = '-'.join([library_name, args.release, 'endf']) +# the destination is decided after the release is known to avoid putting the release in a folder with a misleading name +if args.destination == None: + args.destination = '-'.join([library_name, args.release, 'hdf5']) + +# This dictionary contains all the unique information about each release. This can be exstened to accommodated new releases +release_details = { + '4.0': { + 'base_url': 'https://wwwndc.jaea.go.jp/ftpnd/ftp/JENDL/', + 'files': ['jendl40-or-up_20160106.tar.gz'], + 'neutron_files': os.path.join(endf_files_dir, 'jendl40-or-up_20160106', '*.dat'), + 'metastables': os.path.join(endf_files_dir, 'jendl40-or-up_20160106', '*m.dat'), + 'compressed_file_size': '0.2 GB', + 'uncompressed_file_size': '? GB' + } +} + +download_warning = """ +WARNING: This script will download {} of data. +Extracting and processing the data requires {} of additional free disk space. + +Are you sure you want to continue? ([y]/n) +""".format(release_details[args.release]['compressed_file_size'], + release_details[args.release]['uncompressed_file_size']) + +response = input(download_warning) if not args.batch else 'y' +if response.lower().startswith('n'): + sys.exit() + +block_size = 16384 + +# ============================================================================== +# DOWNLOAD FILES FROM WEBSITE + +files_complete = [] +for f in release_details[args.release]['files']: + # Establish connection to URL + url = release_details[args.release]['base_url'] + f + downloaded_file = download(url) + files_complete.append(downloaded_file) + +# ============================================================================== +# EXTRACT FILES FROM TGZ + +for f in release_details[args.release]['files']: + if f not in files_complete: + continue + + # Extract files + + suffix = '' + with tarfile.open(f, 'r') as tgz: + print('Extracting {0}...'.format(f)) + tgz.extractall(path=os.path.join(endf_files_dir, suffix)) + + +# ============================================================================== +# GENERATE HDF5 LIBRARY -- NEUTRON FILES + +# Get a list of all ACE files +neutron_files = glob.glob(release_details[args.release]['neutron_files']) + +# Create output directory if it doesn't exist +if not os.path.isdir(args.destination): + os.mkdir(args.destination) + +library = openmc.data.DataLibrary() + +for filename in sorted(neutron_files): + + # this is a fix for the TENDL-2017 release where the B10 ACE file which has an error on one of the values + if library_name == 'tendl' and args.release == '2017' and os.path.basename(filename) == 'B010': + text = open(filename, 'r').read() + if text[423:428] == '86843': + print('Manual fix for incorrect value in ACE file') # see OpenMC user group issue for more details + text = ''.join(text[:423])+'86896'+''.join(text[428:]) + open(filename, 'w').write(text) + + print('Converting: ' + filename) + data = openmc.data.IncidentNeutron.from_njoy(filename) + + # Export HDF5 file + h5_file = os.path.join(args.destination, data.name + '.h5') + print('Writing {}...'.format(h5_file)) + data.export_to_hdf5(h5_file, 'w', libver=args.libver) + + # Register with library + library.register_file(h5_file) + +# Write cross_sections.xml +libpath = os.path.join(args.destination, 'cross_sections.xml') +library.export_to_xml(libpath) From 5c3475234ac58139a6ec8cbb124766ced308e215 Mon Sep 17 00:00:00 2001 From: shimwell Date: Sat, 23 Feb 2019 09:16:13 +0000 Subject: [PATCH 02/13] renamed script convert to generate --- convert_jendl.py => generate_jendl.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename convert_jendl.py => generate_jendl.py (100%) diff --git a/convert_jendl.py b/generate_jendl.py similarity index 100% rename from convert_jendl.py rename to generate_jendl.py From 92c17b12c94503e30699d5185bccee798648378a Mon Sep 17 00:00:00 2001 From: shimwell Date: Sat, 23 Feb 2019 09:18:03 +0000 Subject: [PATCH 03/13] removed unused imports --- generate_jendl.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/generate_jendl.py b/generate_jendl.py index c9f8bc1e..bc85dc12 100755 --- a/generate_jendl.py +++ b/generate_jendl.py @@ -5,10 +5,6 @@ import os import sys import tarfile -import zipfile -from collections import defaultdict -from string import digits -from urllib.request import urlopen import openmc.data from openmc._utils import download From bcd8e8358287813deee1ed5f97ebc044f46021ba Mon Sep 17 00:00:00 2001 From: shimwell Date: Sat, 23 Feb 2019 09:19:54 +0000 Subject: [PATCH 04/13] removed unused code --- generate_jendl.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/generate_jendl.py b/generate_jendl.py index bc85dc12..c43d9107 100755 --- a/generate_jendl.py +++ b/generate_jendl.py @@ -43,7 +43,7 @@ class CustomFormatter(argparse.ArgumentDefaultsHelpFormatter, library_name = 'jendl' #this could be added as an argument to allow different libraries to be downloaded endf_files_dir = '-'.join([library_name, args.release, 'endf']) # the destination is decided after the release is known to avoid putting the release in a folder with a misleading name -if args.destination == None: +if args.destination is None: args.destination = '-'.join([library_name, args.release, 'hdf5']) # This dictionary contains all the unique information about each release. This can be exstened to accommodated new releases @@ -70,8 +70,6 @@ class CustomFormatter(argparse.ArgumentDefaultsHelpFormatter, if response.lower().startswith('n'): sys.exit() -block_size = 16384 - # ============================================================================== # DOWNLOAD FILES FROM WEBSITE @@ -111,14 +109,6 @@ class CustomFormatter(argparse.ArgumentDefaultsHelpFormatter, for filename in sorted(neutron_files): - # this is a fix for the TENDL-2017 release where the B10 ACE file which has an error on one of the values - if library_name == 'tendl' and args.release == '2017' and os.path.basename(filename) == 'B010': - text = open(filename, 'r').read() - if text[423:428] == '86843': - print('Manual fix for incorrect value in ACE file') # see OpenMC user group issue for more details - text = ''.join(text[:423])+'86896'+''.join(text[428:]) - open(filename, 'w').write(text) - print('Converting: ' + filename) data = openmc.data.IncidentNeutron.from_njoy(filename) From d0ba342f106db7bf189b6afd2952d55ab9519758 Mon Sep 17 00:00:00 2001 From: shimwell Date: Sat, 23 Feb 2019 09:23:03 +0000 Subject: [PATCH 05/13] Pythonic replaced == None with is None --- convert_fendl.py | 2 +- convert_tendl.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/convert_fendl.py b/convert_fendl.py index 8e9c9aec..e39ac388 100755 --- a/convert_fendl.py +++ b/convert_fendl.py @@ -49,7 +49,7 @@ class CustomFormatter(argparse.ArgumentDefaultsHelpFormatter, library_name = 'fendl' ace_files_dir = '-'.join([library_name, args.release, 'ace']) # the destination is decided after the release is know to avoid putting the release in a folder with a misleading name -if args.destination == None: +if args.destination is None: args.destination = '-'.join([library_name, args.release, 'hdf5']) # This dictionary contains all the unique information about each release. This can be exstened to accommodated new releases diff --git a/convert_tendl.py b/convert_tendl.py index bf76227d..d2ece57d 100755 --- a/convert_tendl.py +++ b/convert_tendl.py @@ -48,7 +48,7 @@ class CustomFormatter(argparse.ArgumentDefaultsHelpFormatter, library_name = 'tendl' #this could be added as an argument to allow different libraries to be downloaded ace_files_dir = '-'.join([library_name, args.release, 'ace']) # the destination is decided after the release is know to avoid putting the release in a folder with a misleading name -if args.destination == None: +if args.destination is None: args.destination = '-'.join([library_name, args.release, 'hdf5']) # This dictionary contains all the unique information about each release. This can be exstened to accommodated new releases From 3bc3f9e65b9e4d3ca233f48b080eeb71e57ba07a Mon Sep 17 00:00:00 2001 From: shimwell Date: Sat, 23 Feb 2019 09:29:57 +0000 Subject: [PATCH 06/13] added the generate word to readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8e838877..c3704e19 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ This repository contains a collection of scripts for generating HDF5 data libraries that can be used with OpenMC. Some of these scripts convert existing -ACE libraries (such as those produced by LANL) whereas others use NJOY to +ACE libraries (such as those produced by LANL) whereas generate scripts use NJOY to process ENDF files directly. Note that unless you are interested in making a customized library, you can find pregenerated HDF5 libraries at https://openmc.mcs.anl.gov. From 07c4bf10e5b124fe4a06f3a6f3a3abdb5ccf970a Mon Sep 17 00:00:00 2001 From: shimwell Date: Sat, 23 Feb 2019 09:30:17 +0000 Subject: [PATCH 07/13] removed some unused imports --- convert_tendl.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/convert_tendl.py b/convert_tendl.py index d2ece57d..982617cb 100755 --- a/convert_tendl.py +++ b/convert_tendl.py @@ -4,11 +4,8 @@ from collections import defaultdict import sys import tarfile -import zipfile import glob import argparse -from string import digits -from urllib.request import urlopen import openmc.data from openmc._utils import download From 3eac9a307219f6d45e5c02ba669610716824bfa5 Mon Sep 17 00:00:00 2001 From: shimwell Date: Sat, 23 Feb 2019 09:30:35 +0000 Subject: [PATCH 08/13] removed some used imports --- convert_fendl.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/convert_fendl.py b/convert_fendl.py index e39ac388..b2b8ecb2 100755 --- a/convert_fendl.py +++ b/convert_fendl.py @@ -3,14 +3,11 @@ import os from collections import defaultdict import sys -import tarfile import zipfile import glob import argparse import ssl import subprocess -from string import digits -from urllib.request import urlopen, Request import openmc.data from openmc._utils import download From d73a1ffcd427180e3c4be7f5e06597fe76aaf17a Mon Sep 17 00:00:00 2001 From: shimwell Date: Sat, 23 Feb 2019 09:34:36 +0000 Subject: [PATCH 09/13] made use of download function in utils to replace code block --- convert_nndc71.py | 40 ++++++---------------------------------- 1 file changed, 6 insertions(+), 34 deletions(-) diff --git a/convert_nndc71.py b/convert_nndc71.py index ff02d3e4..1d779610 100755 --- a/convert_nndc71.py +++ b/convert_nndc71.py @@ -6,17 +6,17 @@ used for OpenMC's regression test suite. """ +import argparse +import glob +import hashlib import os import shutil import subprocess import sys import tarfile -import glob -import hashlib -import argparse -from urllib.request import urlopen import openmc.data +from openmc._utils import download class CustomFormatter(argparse.ArgumentDefaultsHelpFormatter, @@ -51,36 +51,8 @@ class CustomFormatter(argparse.ArgumentDefaultsHelpFormatter, for f in files: # Establish connection to URL url = base_url + f - req = urlopen(url) - - # Get file size from header - file_size = req.length - downloaded = 0 - - # Check if file already downloaded - if os.path.exists(f): - if os.path.getsize(f) == file_size: - print('Skipping ' + f) - files_complete.append(f) - continue - else: - overwrite = input('Overwrite {}? ([y]/n) '.format(f)) - if overwrite.lower().startswith('n'): - continue - - # Copy file to disk - print('Downloading {}... '.format(f), end='') - with open(f, 'wb') as fh: - while True: - chunk = req.read(block_size) - if not chunk: break - fh.write(chunk) - downloaded += len(chunk) - status = '{0:10} [{1:3.2f}%]'.format( - downloaded, downloaded * 100. / file_size) - print(status + chr(8)*len(status), end='') - print('') - files_complete.append(f) + downloaded_file = download(url) + files_complete.append(f) # ============================================================================== # VERIFY MD5 CHECKSUMS From d7db51204bbef21da89c3597733431803531d79e Mon Sep 17 00:00:00 2001 From: shimwell Date: Sat, 23 Feb 2019 09:37:49 +0000 Subject: [PATCH 10/13] made use of download function in utils to replace code block --- convert_jeff32.py | 43 ++++++------------------------------------- 1 file changed, 6 insertions(+), 37 deletions(-) diff --git a/convert_jeff32.py b/convert_jeff32.py index 03b58163..a4d42e25 100755 --- a/convert_jeff32.py +++ b/convert_jeff32.py @@ -1,17 +1,16 @@ #!/usr/bin/env python3 +import argparse +import glob import os -from collections import defaultdict import sys import tarfile import zipfile -import glob -import argparse +from collections import defaultdict from string import digits -from urllib.request import urlopen import openmc.data - +from openmc._utils import download description = """ Download JEFF 3.2 ACE data from OECD/NEA and convert it to a multi-temperature @@ -73,38 +72,8 @@ class CustomFormatter(argparse.ArgumentDefaultsHelpFormatter, for f in files: # Establish connection to URL url = base_url + f - req = urlopen(url) - - # Get file size from header - if sys.version_info[0] < 3: - file_size = int(req.info().getheaders('Content-Length')[0]) - else: - file_size = req.length - downloaded = 0 - - # Check if file already downloaded - if os.path.exists(f): - if os.path.getsize(f) == file_size: - print('Skipping {}, already downloaded'.format(f)) - files_complete.append(f) - continue - else: - overwrite = input('Overwrite {}? ([y]/n) '.format(f)) - if overwrite.lower().startswith('n'): - continue - - # Copy file to disk - print('Downloading {}... '.format(f), end='') - with open(f, 'wb') as fh: - while True: - chunk = req.read(block_size) - if not chunk: break - fh.write(chunk) - downloaded += len(chunk) - status = '{:10} [{:3.2f}%]'.format(downloaded, downloaded * 100. / file_size) - print(status + chr(8)*len(status), end='') - print('') - files_complete.append(f) + downloaded_file = download(url) + files_complete.append(f) # ============================================================================== # EXTRACT FILES FROM TGZ From b1c2eec679bb771cbe27b7173ca989d9b7daf522 Mon Sep 17 00:00:00 2001 From: shimwell Date: Sat, 23 Feb 2019 09:49:00 +0000 Subject: [PATCH 11/13] removed unused blocksize code --- convert_fendl.py | 2 -- convert_jeff32.py | 2 -- convert_nndc71.py | 1 - convert_tendl.py | 2 -- 4 files changed, 7 deletions(-) diff --git a/convert_fendl.py b/convert_fendl.py index b2b8ecb2..4e420a95 100755 --- a/convert_fendl.py +++ b/convert_fendl.py @@ -79,8 +79,6 @@ class CustomFormatter(argparse.ArgumentDefaultsHelpFormatter, if response.lower().startswith('n'): sys.exit() -block_size = 16384 - # ============================================================================== # DOWNLOAD FILES FROM IAEA SITE diff --git a/convert_jeff32.py b/convert_jeff32.py index a4d42e25..f255fda3 100755 --- a/convert_jeff32.py +++ b/convert_jeff32.py @@ -63,8 +63,6 @@ class CustomFormatter(argparse.ArgumentDefaultsHelpFormatter, 'JEFF32-ACE-1800K.tar.gz', 'TSLs.tar.gz'] -block_size = 16384 - # ============================================================================== # DOWNLOAD FILES FROM OECD SITE diff --git a/convert_nndc71.py b/convert_nndc71.py index 1d779610..3fc17985 100755 --- a/convert_nndc71.py +++ b/convert_nndc71.py @@ -42,7 +42,6 @@ class CustomFormatter(argparse.ArgumentDefaultsHelpFormatter, 'ENDF-B-VII.1-tsl.tar.gz'] checksums = ['9729a17eb62b75f285d8a7628ace1449', 'e17d827c92940a30f22f096d910ea186'] -block_size = 16384 # ============================================================================== # DOWNLOAD FILES FROM NNDC SITE diff --git a/convert_tendl.py b/convert_tendl.py index 982617cb..af673325 100755 --- a/convert_tendl.py +++ b/convert_tendl.py @@ -80,8 +80,6 @@ class CustomFormatter(argparse.ArgumentDefaultsHelpFormatter, if response.lower().startswith('n'): sys.exit() -block_size = 16384 - # ============================================================================== # DOWNLOAD FILES FROM WEBSITE From 2f7e226ac64afbd0be547fdb5094695f80d9e062 Mon Sep 17 00:00:00 2001 From: shimwell Date: Sat, 23 Feb 2019 10:05:44 +0000 Subject: [PATCH 12/13] sorted imports --- convert_fendl.py | 10 ++++------ convert_tendl.py | 6 ++---- download_all.sh | 6 ++++++ generate_endf71.py | 7 +++---- 4 files changed, 15 insertions(+), 14 deletions(-) create mode 100644 download_all.sh diff --git a/convert_fendl.py b/convert_fendl.py index 4e420a95..3a7b6461 100755 --- a/convert_fendl.py +++ b/convert_fendl.py @@ -1,18 +1,16 @@ #!/usr/bin/env python3 -import os -from collections import defaultdict -import sys -import zipfile -import glob import argparse +import glob +import os import ssl import subprocess +import sys +import zipfile import openmc.data from openmc._utils import download - description = """ Download FENDL 3.1d or FENDL 3.1c ACE data from the IAEA and convert it to a HDF5 library for use with OpenMC. diff --git a/convert_tendl.py b/convert_tendl.py index af673325..00265ce5 100755 --- a/convert_tendl.py +++ b/convert_tendl.py @@ -1,16 +1,14 @@ #!/usr/bin/env python3 +import argparse +import glob import os -from collections import defaultdict import sys import tarfile -import glob -import argparse import openmc.data from openmc._utils import download - description = """ Download TENDL 2017 or TENDL 2015 ACE data from PSI and convert it to a HDF5 library for use with OpenMC. diff --git a/download_all.sh b/download_all.sh new file mode 100644 index 00000000..d3ea33c1 --- /dev/null +++ b/download_all.sh @@ -0,0 +1,6 @@ +python3 data/convert_fendl.py +python3 data/convert_jeff32.py +python3 data/convert_nndc71.py +python3 data/convert_tendl.py +python3 data/generate_endf71.py +python3 data/generate_jendl.py \ No newline at end of file diff --git a/generate_endf71.py b/generate_endf71.py index 431404f0..a4f24247 100755 --- a/generate_endf71.py +++ b/generate_endf71.py @@ -1,21 +1,20 @@ #!/usr/bin/env python3 import argparse -from multiprocessing import Pool import os -from pathlib import Path import shutil import sys import tarfile import tempfile -from urllib.parse import urljoin import warnings import zipfile +from multiprocessing import Pool +from pathlib import Path +from urllib.parse import urljoin import openmc.data from openmc._utils import download - # Make sure Python version is sufficient assert sys.version_info >= (3, 6), "Python 3.6+ is required" From 81f1114ea63d0108e3c0ae1790f9db5818ce48b8 Mon Sep 17 00:00:00 2001 From: shimwell Date: Sat, 23 Feb 2019 11:07:31 +0000 Subject: [PATCH 13/13] added file size to jendl --- generate_jendl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generate_jendl.py b/generate_jendl.py index c43d9107..75657c00 100755 --- a/generate_jendl.py +++ b/generate_jendl.py @@ -54,7 +54,7 @@ class CustomFormatter(argparse.ArgumentDefaultsHelpFormatter, 'neutron_files': os.path.join(endf_files_dir, 'jendl40-or-up_20160106', '*.dat'), 'metastables': os.path.join(endf_files_dir, 'jendl40-or-up_20160106', '*m.dat'), 'compressed_file_size': '0.2 GB', - 'uncompressed_file_size': '? GB' + 'uncompressed_file_size': '2 GB' } }